From e5be2ca844b1b9badbdcd47131e1c846a999c0ff Mon Sep 17 00:00:00 2001 From: rain Date: Mon, 16 Nov 2026 00:37:54 +0000 Subject: [PATCH] feat(gc): remove-file and path removal (#45) --- .../layerfs-history/src/contract/catalog.rs | 27 +- .../layerfs-history/src/contract/error.rs | 3 + .../layerfs-history/src/contract/mod.rs | 1 + .../layerfs-history/src/contract/path.rs | 154 ++++++ .../layerfs-history/src/contract/records.rs | 47 ++ core/crates/layerfs-history/src/lib.rs | 4 +- core/docs/issues/project-41/gc-rollout.md | 119 +++++ .../layerfs-daemon/src/operator/workspace.rs | 20 +- .../sql/postgresql/schema.sql | 1 + .../sql/sqlite/history.sql | 1 + .../queries/history/commit_insert_commit.sql | 2 +- .../history/layerstack_insert_layer.sql | 2 +- .../queries/history/paths_advance_epoch.sql | 1 + .../sqlite/queries/history/paths_marker.sql | 1 + .../sqlite/queries/history/paths_markers.sql | 1 + .../sqlite/queries/history/paths_upsert.sql | 1 + .../queries/history/position_high_water.sql | 1 + .../queries/history/position_next_serial.sql | 1 + .../queries/history/position_node_serial.sql | 1 + .../history/position_stack_high_water.sql | 1 + .../src/backend/postgresql/schema.rs | 92 +++- .../backend/postgresql/schema_diagnostic.rs | 1 + .../src/backend/sqlite/schema.rs | 168 +++++- .../src/backend/sqlite/session/query.rs | 24 + .../src/history/catalog.rs | 22 + .../layerfs-persistence/src/history/commit.rs | 3 + .../src/history/layerstack.rs | 6 + .../layerfs-persistence/src/history/mod.rs | 2 + .../layerfs-persistence/src/history/paths.rs | 112 ++++ .../src/history/position.rs | 47 ++ .../layerfs-persistence/src/store/open.rs | 1 + .../src/store/postgresql.rs | 1 + .../layerfs-persistence/tests/gc_backfill.rs | 492 ++++++++++++++++++ .../tests/old_format_columns.rs | 15 +- .../tests/old_format_open.rs | 8 +- .../layerfs-persistence/tests/path_removal.rs | 172 ++++++ .../src/history/lookup.rs | 1 + .../layerfs-store-service/src/methods/args.rs | 8 + .../src/methods/dispatch.rs | 1 + .../src/methods/file/glob.rs | 18 +- .../src/methods/file/list.rs | 15 +- .../src/methods/file/read.rs | 12 +- .../src/methods/file/stat.rs | 19 +- .../src/methods/project.rs | 52 +- .../layerfs-store-service/src/read/mod.rs | 1 + .../src/read/snapshot.rs | 7 + .../src/read/visibility.rs | 98 ++++ .../tests/golden/sdk_methods.txt | 10 +- .../tests/sdk_path_removal.rs | 135 +++++ 49 files changed, 1872 insertions(+), 60 deletions(-) create mode 100644 core/crates/layerfs-history/src/contract/path.rs create mode 100644 core/docs/issues/project-41/gc-rollout.md create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_advance_epoch.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_marker.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_markers.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_upsert.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_high_water.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_next_serial.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_node_serial.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_stack_high_water.sql create mode 100644 engine/crates/layerfs-persistence/src/history/paths.rs create mode 100644 engine/crates/layerfs-persistence/src/history/position.rs create mode 100644 engine/crates/layerfs-persistence/tests/gc_backfill.rs create mode 100644 engine/crates/layerfs-persistence/tests/path_removal.rs create mode 100644 engine/crates/layerfs-store-service/src/read/visibility.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_path_removal.rs diff --git a/core/crates/layerfs-history/src/contract/catalog.rs b/core/crates/layerfs-history/src/contract/catalog.rs index fee7e3ffda..431c5563a7 100644 --- a/core/crates/layerfs-history/src/contract/catalog.rs +++ b/core/crates/layerfs-history/src/contract/catalog.rs @@ -21,8 +21,8 @@ use crate::identity::{BranchId, CatalogId, CommitId, HistoryName, LayerId, Layer use crate::records::{ AddLayerOutcome, AddLayerRequest, BranchMergeRequest, BranchRecord, BranchSnapshot, CommitHistoryRequest, CommitOutcome, CommitRecord, CommitRequest, ForkRequest, - LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, Reservation, - ReserveRequest, StackInitialization, + LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, PathRemovalOutcome, + PathRemovalRecord, PathRemovalRequest, Reservation, ReserveRequest, StackInitialization, }; /// Checked inputs that establish one catalog binding. #[derive(Clone, Eq, PartialEq)] @@ -332,6 +332,29 @@ pub trait HistoryCatalog: Send + Sync { )) } + /// Applies one `project.remove_file` marker in one write transaction: + /// reads the Store high-water as the new frontier, returns `UpToDate` + /// when the path's marker already covers every project node, otherwise + /// advances the marker and the project epoch. The default refuses. + fn remove_path(&self, _request: &PathRemovalRequest) -> HistoryResult { + Err(crate::error::HistoryError::Unsupported("path removal")) + } + + /// Every `path_removed` marker of one Stack. The default refuses. + fn path_removals(&self, _stack: LayerStackId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported( + "path removal markers", + )) + } + + /// The `history_serial` a Commit or Layer records, if it exists. The + /// default refuses. + fn node_history_serial(&self, _node: NodeId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported( + "node history position", + )) + } + /// The retained history roots of one Stack. The default refuses. /// /// # Errors diff --git a/core/crates/layerfs-history/src/contract/error.rs b/core/crates/layerfs-history/src/contract/error.rs index d13b476c1d..e4b686b3d0 100644 --- a/core/crates/layerfs-history/src/contract/error.rs +++ b/core/crates/layerfs-history/src/contract/error.rs @@ -147,6 +147,8 @@ pub enum HistoryError { NameTaken(Box), /// A path is hidden by `remove_file`. Removed(Box), + /// A required marker or fence row is missing or contradicts the request. + Conflict(&'static str), /// The Store was replaced or moved under this handle. StaleStore, /// Immediate metadata admission was refused; the caller may retry later. @@ -208,6 +210,7 @@ impl std::fmt::Display for HistoryError { Self::Unsupported(what) => write!(formatter, "unsupported: {what}"), Self::NameTaken(state) => write!(formatter, "name taken: {:?}", state.holder), Self::Removed(state) => write!(formatter, "removed: {} paths", state.paths.len()), + Self::Conflict(what) => write!(formatter, "conflict: {what}"), Self::StaleStore => formatter.write_str("stale Store"), Self::Busy => formatter.write_str("busy"), Self::OwnershipUnavailable => formatter.write_str("ownership unavailable"), diff --git a/core/crates/layerfs-history/src/contract/mod.rs b/core/crates/layerfs-history/src/contract/mod.rs index 37313bcbc2..520aebbe72 100644 --- a/core/crates/layerfs-history/src/contract/mod.rs +++ b/core/crates/layerfs-history/src/contract/mod.rs @@ -2,5 +2,6 @@ pub mod catalog; pub mod error; pub mod identity; +pub mod path; pub mod query; pub mod records; diff --git a/core/crates/layerfs-history/src/contract/path.rs b/core/crates/layerfs-history/src/contract/path.rs new file mode 100644 index 0000000000..dc751b0403 --- /dev/null +++ b/core/crates/layerfs-history/src/contract/path.rs @@ -0,0 +1,154 @@ +//! The `project.remove_file` path grammar and its keys. +//! +//! The call accepts an absolute POSIX path and stores it as the normalized +//! relative path (`/a` and `a` are the same). A path is refused when it is +//! empty, `/`, `//a`, has an empty, `.` or `..` component, a trailing `/`, a +//! `\` or NUL byte, is longer than 4096 bytes, or has more than 256 +//! components. The stored key is `BLAKE3(normalized path)`; the trace key is +//! keyed by the Store's `trace_salt` over the stack and the normalized path. + +use crate::identity::LayerStackId; + +/// The most bytes one normalized path may have. +pub const MAXIMUM_PATH_BYTES: usize = 4096; +/// The most components one normalized path may have. +pub const MAXIMUM_PATH_COMPONENTS: usize = 256; + +/// A checked, normalized relative path. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NormalizedPath { + relative: String, +} + +impl NormalizedPath { + /// Checks `text` and returns its normalized relative path. + /// + /// One leading `/` is stripped; everything after it is the relative path. + /// The returned value never starts or ends with `/` and never holds an + /// empty, `.` or `..` component. + pub fn new(text: &str) -> Result { + if text.is_empty() { + return Err("path is empty"); + } + if text.as_bytes().contains(&0) { + return Err("path contains a NUL byte"); + } + if text.contains('\\') { + return Err("path contains a backslash"); + } + let relative = text.strip_prefix('/').unwrap_or(text); + if relative.is_empty() { + return Err("path is the root"); + } + if relative.starts_with('/') { + return Err("path has an empty component"); + } + let mut components = 0usize; + for component in relative.split('/') { + if component.is_empty() { + return Err("path has an empty component"); + } + if component == "." || component == ".." { + return Err("path has a dot component"); + } + components += 1; + } + if components > MAXIMUM_PATH_COMPONENTS { + return Err("path has too many components"); + } + if relative.len() > MAXIMUM_PATH_BYTES { + return Err("path is too long"); + } + Ok(Self { + relative: relative.to_owned(), + }) + } + + /// The normalized relative path, without a leading `/`. + pub fn as_str(&self) -> &str { + &self.relative + } + + /// The normalized path bytes as stored in the marker. + pub fn as_bytes(&self) -> &[u8] { + self.relative.as_bytes() + } + + /// `BLAKE3(normalized path)`, the marker key. + pub fn key(&self) -> [u8; 32] { + path_key(self.as_bytes()) + } +} + +/// `BLAKE3(path)` over already normalized path bytes, the marker key. +pub fn path_key(path: &[u8]) -> [u8; 32] { + *blake3::hash(path).as_bytes() +} + +/// True when `path` is `marker` or lies below it at a `/` boundary +/// (`p == m OR p begins m ++ b"/"`), compared over bytes, never `||`/`LIKE`. +pub fn governs(marker: &[u8], path: &[u8]) -> bool { + path == marker + || (path.len() > marker.len() && path.starts_with(marker) && path[marker.len()] == b'/') +} + +/// The key a trace hashes for one stack and path: keyed by the Store's +/// 32-byte `trace_salt` over the stack identity and the normalized path. +pub fn trace_key(salt: &[u8; 32], stack: LayerStackId, path: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new_keyed(salt); + hasher.update(stack.as_slice()); + hasher.update(path); + *hasher.finalize().as_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_one_leading_slash_and_relative_paths() { + assert_eq!(NormalizedPath::new("/a").unwrap().as_str(), "a"); + assert_eq!(NormalizedPath::new("a").unwrap().as_str(), "a"); + assert_eq!(NormalizedPath::new("/a/b").unwrap().as_str(), "a/b"); + assert_eq!(NormalizedPath::new("a-b_c.d").unwrap().as_str(), "a-b_c.d"); + } + + #[test] + fn refuses_the_bad_grammar() { + for text in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "/./a", "a\\b", "a\0b", + ] { + assert!(NormalizedPath::new(text).is_err(), "accepted {text:?}"); + } + } + + #[test] + fn refuses_long_paths_and_many_components() { + assert!(NormalizedPath::new(&"a".repeat(MAXIMUM_PATH_BYTES)).is_ok()); + assert!(NormalizedPath::new(&"a".repeat(MAXIMUM_PATH_BYTES + 1)).is_err()); + let many = vec!["a"; MAXIMUM_PATH_COMPONENTS].join("/"); + assert!(NormalizedPath::new(&many).is_ok()); + let more = vec!["a"; MAXIMUM_PATH_COMPONENTS + 1].join("/"); + assert!(NormalizedPath::new(&more).is_err()); + } + + #[test] + fn subtree_matches_only_at_a_slash_boundary() { + assert!(governs(b"a", b"a")); + assert!(governs(b"a", b"a/b")); + assert!(!governs(b"a", b"ab")); + assert!(!governs(b"ab", b"a")); + assert!(governs(b"a/b", b"a/b/c")); + assert!(!governs(b"a/b", b"a/bc")); + } + + #[test] + fn keys_are_stable_and_salted_traces_differ() { + let path = NormalizedPath::new("/a/b").unwrap(); + assert_eq!(path.key(), path_key(b"a/b")); + let stack = LayerStackId::from_authority([7; 16]); + let first = trace_key(&[1; 32], stack, path.as_bytes()); + let second = trace_key(&[2; 32], stack, path.as_bytes()); + assert_ne!(first, second); + } +} diff --git a/core/crates/layerfs-history/src/contract/records.rs b/core/crates/layerfs-history/src/contract/records.rs index b840fdbf00..394ba3de9f 100644 --- a/core/crates/layerfs-history/src/contract/records.rs +++ b/core/crates/layerfs-history/src/contract/records.rs @@ -636,3 +636,50 @@ pub struct InitStatistics { /// Init's wall time in milliseconds, up to the first Branch. pub millis: u64, } + +/// One `path_removed` marker as stored for a project. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalRecord { + /// The normalized relative path the marker names. + pub path: Vec, + /// The Store high-water when the marker last advanced. + pub frontier_serial: u64, + /// When the marker last advanced, in milliseconds since the epoch. + pub removed_ms: i64, +} + +/// Whether a `project.remove_file` call advanced the marker or found it current. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PathRemovalKind { + /// The marker advanced to a new frontier. + Removed, + /// No project node is above the marker's frontier; the stored values are returned. + UpToDate, +} + +/// One `project.remove_file` request: the project, the normalized path and the +/// time the caller observed. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalRequest { + /// Project (LayerStack) the path is removed in. + pub stack: LayerStackId, + /// The normalized relative path bytes; `BLAKE3` keys the marker. + pub path: Vec, + /// When the call was made, in milliseconds since the epoch. + pub removed_ms: i64, +} + +/// One `project.remove_file` result: the stored marker values. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalOutcome { + /// Whether the call advanced the marker or found it current. + pub kind: PathRemovalKind, + /// Project (LayerStack) the marker belongs to. + pub stack: LayerStackId, + /// The stored normalized relative path. + pub path: Vec, + /// The stored frontier serial. + pub frontier_serial: u64, + /// The stored `removed_ms`. + pub removed_ms: i64, +} diff --git a/core/crates/layerfs-history/src/lib.rs b/core/crates/layerfs-history/src/lib.rs index 0134a0e65d..6f01165217 100644 --- a/core/crates/layerfs-history/src/lib.rs +++ b/core/crates/layerfs-history/src/lib.rs @@ -21,6 +21,7 @@ mod contract; pub use contract::catalog; pub use contract::error; pub use contract::identity; +pub use contract::path; pub use contract::query; pub use contract::records; @@ -34,6 +35,7 @@ pub use records::{ BranchSnapshot, CommitHistoryRequest, CommitMeta, CommitOutcome, CommitRecord, CommitRequest, CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, InitStatistics, LayerHistoryRequest, LayerRecord, LayerStackRecord, MergeCommit, MergeHead, MergeSide, Page, - PageResult, RecordKind, Reservation, ReserveRequest, StackInitialization, StoreStatistics, + PageResult, PathRemovalKind, PathRemovalOutcome, PathRemovalRecord, PathRemovalRequest, + RecordKind, Reservation, ReserveRequest, StackInitialization, StoreStatistics, FILE_DIGEST_WINDOW, MAXIMUM_MESSAGE_BYTES, MAXIMUM_PAGE_RECORDS, }; diff --git a/core/docs/issues/project-41/gc-rollout.md b/core/docs/issues/project-41/gc-rollout.md new file mode 100644 index 0000000000..5765c7f3c9 --- /dev/null +++ b/core/docs/issues/project-41/gc-rollout.md @@ -0,0 +1,119 @@ +# GC rollout, rollback and dry run (T1.6) + +> Plan for the frozen `LayerFS-GC-spec.md` (2.0.0; the spec wins, `§N` cites it). This page adds no contract and no option: the four calls take only a resource +> identifier, with no confirm, force or dry run (§1), and are admin-profile only, refused under a folder scope (§10). + +## 1. Rollout order + +| Step | Content | Contract | +|---|---|---| +| T1.x | Additive schema: marker/intent tables, position state, `gc_op`/`gc_dead`, `trace_salt` | §4.1 | +| T3/T4/T5 | Removals `branch.remove`, `project.delete`, `project.remove_file`: marker rows only, no bytes freed | §4 | +| T6 | Admission: `gc_intent`, `MountsActive`/`Busy`/`Unresolved`, epoch fence | §5 | +| T7 | Mark: reachability closure at `G`, `none`/`rebuild` from wholly dead packs | §3, §6 | +| T8 | SQLite rebuild: seal, `visited` scratch, verify, rename install | §7 | +| T9 | PostgreSQL rebuild: Phase M mark, Phase A build, Phase B swap, Phase C cleanup | §8 | +| T10/T11 | Report (§11 fields) and API surface | §9–§11 | +| T12 | Acceptance (§12 cases) and measurement (§13) | §12 | + +Order is forced: a rebuild copies T1.x's tables and cleans T3–T5's marker rows. T8/T9 are independent per backend; no step enables `store.compact` by itself. + +## 2. Expand and contract + +- **Expand (T1.x, T3–T5).** The schema step only adds tables and columns and one idempotent backfill under the current schema identity; no column changes type or meaning, + none is dropped, and `history_meta.next_history_serial` is never lowered (§4.1). Removals add marker rows and bump `project_epoch.epoch` (§4, §5). Both are additive: a + Store that never removed anything and never compacts is a plain 2.0 Store. +- **Old readers.** SQLite takes the step as one more `add_*`; PostgreSQL's open check requires the exact table set and recorded definition, so the step rewrites that + definition for older Stores (§4.1). An older PG binary that predates the step refuses the upgraded Store, and the spec defines no down-migration: **downgrade is not + supported**. Roll forward, or restore a pre-upgrade backup. +- **Contract.** Two API contractions are version cuts, not toggles: `project.remove` leaves the call table (an old client gets `InvalidInput`, never a silent hard delete) + and `project.init` returns `NameTaken` instead of `Exists` (§4, §10). Both land with T1.x and are asserted in the error matrix. +- **Flags.** None are added. `store.compact` is opt-in by the caller invoking it: no automatic collection, no online sweep (§1), and the spec defines no global + enable/disable switch. Do not invent one; the only "enable" is the release cut plus §7, and an owner may simply not call it. + +## 3. Dry run + +`store.compact` has no dry-run option (§1). Two non-mutating paths exist and neither is a new call: + +1. **Copy rehearsal.** `backup.create`, `backup.restore` into a scratch Store/home, run `store.compact` against the *scratch* name, compare. §9 names `backup.create` + before compaction as the recovery path for a mark bug; a copy is the closest honest rehearsal. +2. **`none` outcome.** Below 10% wholly dead packs the path is `none`: mark only, nothing written, generation unchanged, `deadBytesRefined == deadBytesBound` (§3, §11). + It cannot be requested, but on a Store expected to be `none` it is a real non-mutating run. + +Compare, before → after (both backends; §11): + +| Observation | `none` | `rebuild` | +|---|---|---| +| `store.status` | no op before/after; phase visible during | same; phases `mark`/`build`/`verify`/`install` (SQLite) or `mark`/`build`/`swap`/`cleanup` (PG) | +| `generation {before, after}` | equal, `G` | `G → G + 1` | +| `deadBytesBound` ≥ `deadBytesRefined` | equal | refined ≤ bound, from the payload sweep | +| `logicalDiscarded` | 0 with a `zeroReason` | collected history `logical` bytes | +| `keptWaste` | measured, unchanged by the rehearsal | `{deadInRetainedPacksBytes, deltaBases, inodeBlocksBytes}` | +| `store_totals` | unchanged | `logical` unchanged; `canonical`/`stored`/`metadata` = old − `dead_*` (§7 step 4, §8 step 9) | +| `residue` | empty | empty after settle; nonzero is explained | + +Also compare `physicalShrink` and `backendReusable` (§11). A rehearsal that disagrees with the live Store is a fixture difference, not a different path: both obey the +same §3 rule. + +## 4. Rollback + +- **Schema is additive.** The T1.x step is the only schema change and re-running it is a no-op; there is no down-migration. Do not un-create the tables by hand: without + `gc_intent`/`gc_op` the Store cannot admit or recover an operation, and PostgreSQL's open check (exact table set, recorded definition) refuses it (§4.1). +- **Markers are not undone by dropping tables.** Dropping `branch_retired`, `layer_stack_retired` or `path_removed` would resurrect names and paths whose history a + rebuild may already have collected (a deleted project's rows are dropped by the rebuild, §7/§8), and §1 promises no undo. The only supported re-creation is a Commit + **after** the frontier that writes the path again: a post-frontier re-creation is visible, and the marker hides only inherited content (§4.3). A Branch or project name + is re-created after a rebuild frees it; the history is gone (§4.2, §12 "Names"). +- **A failed compaction keeps `G`.** A definite failure before the rename aborts and deletes the artifact set + `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}` (SQLite) or the recorded `lfs_gc_` scratch schema (PostgreSQL); the old Store is unchanged + apart from step 1's checkpoint (§7 step 8, §8, §9). Keep `G` and re-run later. +- **Installed-but-uncertain.** A failure of the rename/fsync itself, or a lost `COMMIT` reply, is not pre-install: keep the new Store at `G + 1`, settle the op `unknown` + and report it; never delete the artifact as if nothing happened (§7 step 8, §8). On PostgreSQL resolve a lost reply by proving the backend ended (`pid`, + `backend_start`, `usename`), then read the incarnation (§8). +- **Artifacts belong to the next `store.compact`.** Only a compaction that names them from its op removes them; a staged name no record names is `Exists` and is not + deleted (§9). Leave crash leftovers alone by hand. + +## 5. Verification gates + +| Phase | Tests | `store.status` / report | Doctor | +|---|---|---|---| +| T1.x | converted 1.0/2.0 Stores open; backfill (`history_serial`, `project_epoch`, `trace_salt`, never-lowered serial); `tools/store_blocks.py` | unchanged, no op row | full, before and after | +| T3–T5 | marker transaction, §4.2 visibility guard, admission races, whiteout on merge/mount/publish | no op left; `Removed`/`NoSuch*` shapes | full | +| T6 | intent lifetime, `machine_stopped`, `Unresolved` ownership, `holders` | `busy`/`unresolved` only while owned | full | +| T7 | closure vs reference walk, `live_heads_digest`, "no live object lost" (§12) | phase `mark`; path chosen | full on fixtures | +| T8 | §7 failpoints (mark/build/verify/install/WAL switch/artifact removal/op settle), exact artifact set, `StaleStore` | both paths' §11 fields; `physicalShrink`, `backendReusable` | `validate` before rename; full after | +| T9 | §8 failpoints (mark/build/swap/cleanup/scratch drop), `ACCESS EXCLUSIVE` duration, standby, reply loss, privileges | PG zeros (`backendReusable = 0`); resume from `gc_op` | full after cleanup | +| T10/T11 | §11 snake_case keys, `zeroReason` vocabulary, capped `cap`, scope refusals (§10) | phase and `op` projection | full | +| T12 | whole §12 matrix; independent `diagnostic/scan.rs` walk; 3 runs per arm (§13) | every field with unit and sampling point | full on fixtures | + +`quick_check` and the full doctor stay separate commands, not part of install (§7 step 6). A gate is green only on both backends, or on the rows §12 assigns. + +## 6. Failure handling + +- **Crash.** The Store is wholly old or wholly new (§9). The next `store.compact` judges the recorded `G`: at `G` the artifact is removed (not installed); at `G + 1` the + outcome is recorded and Phase C resumes. A PG crash in Phase M/A leaves only the scratch schema, which the op names; Phase B is atomic. +- **`Busy`.** A live mount, long-lived client or operation holds the scope. The call does not wait, retry or replay; prepare inactivity and call once later (§1, §5). +- **`Unresolved`.** A dead, uncertain owner's op. A non-owning call observes it; only the owning call (the next `store.compact`) adopts or clears it (§5, §10). An + unreachable machine reports `unknown`, never clean (§9). +- **Lost reply.** The outcome stays uncertain: read `store.status`, the op and the generation before deciding; never replay blind, and never delete a sidecar or an + artifact the op names. +- **Standby.** PostgreSQL waits only when `synchronous_commit` is at least `remote_write` and `synchronous_standby_names` is non-empty; the check runs before Phase M and + again inside the swap. A missing standby abandons with `Busy` (standby list); a standby dying after the check blocks the `COMMIT` unbounded (§8). SQLite has no such + gate. +- **`Capacity` / disk full.** Aborts, deletes the artifact, leaves the old Store unchanged apart from the checkpoint; `Capacity` is the real refusal path, not a codec + bound (§7). +- **`StaleStore`.** A connection that bypassed admission is caught at its next transaction; an in-flight transaction at install is a documented limit (§7, §10). + +## 7. Go/no-go: enabling `store.compact` on a shared Store + +1. `backup.create` succeeds and a `backup.restore` drill of this Store was done (§9). +2. `store.status` is clean: no `holders`, `state` not `busy`/`unresolved`, no live `gc_intent` row, no existing `.compact*` set or `lfs_gc_` schema. +3. All mounts, daemons and long-lived clients of the Store are gone, and no Init/merge row names it (§5); one owner knows nobody may start one during the window. +4. The backend's T8 or T9 gate and the §12 rows touching it are green; the full doctor is clean on a fixture of the same schema layout (§7 step 6, §12). +5. Disk headroom covers peak temp: old Store + new Store + journal/WAL + `visited` (SQLite); old tables + new heap/TOAST + indexes + live-bytes WAL and `pg_wal` headroom + (PG) (§7 cost, §8). +6. PostgreSQL only: `CREATE` on the database, `TEMP`, `USAGE`/`CREATE` on the Store schema, table ownership, `pg_read_all_stats`, else `Unsupported`; a direct or + session-pooled connection (transaction pooling is `Unsupported`); standby replication at the intended level (§5.2, §8). +7. The expected path is known: wholly dead bytes below 10% means `none` and no shrink — decide in advance whether the run is still wanted (§3). +8. The call is made under the MCP admin profile, not a folder scope (§10). +9. Rollback is understood: keep `G`, let only the next `store.compact` remove named artifacts, restore from backup if a mark bug is suspected (§9). +10. One attempt, no replay: a refusal from §6 is inspected before any second call (§1, §9). diff --git a/engine/crates/layerfs-daemon/src/operator/workspace.rs b/engine/crates/layerfs-daemon/src/operator/workspace.rs index ad7218a6c2..19d5cb8a64 100644 --- a/engine/crates/layerfs-daemon/src/operator/workspace.rs +++ b/engine/crates/layerfs-daemon/src/operator/workspace.rs @@ -155,15 +155,27 @@ fn scan( ) -> Result<()> { let binding = workspace.snapshot().map_err(args::failed)?; let root = binding.effective_root; + let node = binding.branch.head_commit.map_or( + layerfs_history::NodeId::Layer(binding.branch.base_layer), + layerfs_history::NodeId::Commit, + ); + // The base node `history_serial`, as every `Snap` carries it for the + // `remove_file` visibility rule; 0 is the same unknown fallback that + // `snapshot::resolve` takes. + let serial = match service + .store + .published(|history| history.node_history_serial(node)) + { + Ok(Some(serial)) => serial, + _ => 0, + }; let snap = Snap { kind: Kind::Commit, label: "Workspace base".into(), - node: binding.branch.head_commit.map_or( - layerfs_history::NodeId::Layer(binding.branch.base_layer), - layerfs_history::NodeId::Commit, - ), + node, root, tree: TreeTable::open(op.ports(), FilesystemRootId(root)).map_err(args::content)?, + serial, }; let pending = workspace .pending_conflicts() diff --git a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql index 5b15c1bb69..8687c0371c 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql @@ -70,6 +70,7 @@ CREATE TABLE history_meta ( binding_key bytea NOT NULL CHECK (octet_length(binding_key) BETWEEN 1 AND 128), identity_format bigint NOT NULL CHECK (identity_format = 3), schema_version bigint NOT NULL CHECK (schema_version = 2), + next_history_serial bigint NOT NULL DEFAULT 1, schema_source text NOT NULL, schema_definition text NOT NULL ); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/history.sql b/engine/crates/layerfs-persistence/sql/sqlite/history.sql index a1f6e85f6a..7c3ec88f1c 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/history.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/history.sql @@ -6,6 +6,7 @@ CREATE TABLE history_meta ( binding_key BLOB NOT NULL CHECK (length(binding_key) BETWEEN 1 AND 128), identity_format INTEGER NOT NULL CHECK (identity_format = 3), schema_version INTEGER NOT NULL CHECK(schema_version=1), + next_history_serial INTEGER NOT NULL DEFAULT 1, schema_source TEXT NOT NULL, schema_definition TEXT NOT NULL ) STRICT; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/commit_insert_commit.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/commit_insert_commit.sql index 48d2df4768..9af2640f05 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/commit_insert_commit.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/commit_insert_commit.sql @@ -1 +1 @@ -INSERT INTO "commit" (commit_id, layer_stack_id, root_id, parent_commit_id, base_layer_id, second_parent_id, branch_id, via, message, merged_from_id, created_ms, logical, new_canonical, new_stored, new_metadata, lww, displaced_id) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) ON CONFLICT (commit_id) DO NOTHING RETURNING 1; +INSERT INTO "commit" (commit_id, layer_stack_id, root_id, parent_commit_id, base_layer_id, second_parent_id, branch_id, via, message, merged_from_id, created_ms, logical, new_canonical, new_stored, new_metadata, lww, displaced_id, history_serial) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18) ON CONFLICT (commit_id) DO NOTHING RETURNING 1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_insert_layer.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_insert_layer.sql index c483c0caf3..7d908b00d7 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_insert_layer.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/layerstack_insert_layer.sql @@ -1 +1 @@ -INSERT INTO layer (layer_id,layer_stack_id,parent_layer_id,root_id,source_branch_id,source_commit_id,logical, overwrote) VALUES ($1,$2,$3,$4,$5,$6,$7, $8) RETURNING 1; +INSERT INTO layer (layer_id,layer_stack_id,parent_layer_id,root_id,source_branch_id,source_commit_id,logical, overwrote, history_serial) VALUES ($1,$2,$3,$4,$5,$6,$7, $8, $9) RETURNING 1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_advance_epoch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_advance_epoch.sql new file mode 100644 index 0000000000..15fa42d851 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_advance_epoch.sql @@ -0,0 +1 @@ +UPDATE project_epoch SET epoch=epoch+1 WHERE layer_stack_id=$1 RETURNING 1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_marker.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_marker.sql new file mode 100644 index 0000000000..049788375c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_marker.sql @@ -0,0 +1 @@ +SELECT path,frontier_serial,removed_ms FROM path_removed WHERE layer_stack_id=$1 AND path_key=$2; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_markers.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_markers.sql new file mode 100644 index 0000000000..d3bd1e7c8a --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_markers.sql @@ -0,0 +1 @@ +SELECT path,frontier_serial,removed_ms FROM path_removed WHERE layer_stack_id=$1 ORDER BY path; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_upsert.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_upsert.sql new file mode 100644 index 0000000000..f9171232e1 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/paths_upsert.sql @@ -0,0 +1 @@ +INSERT INTO path_removed(layer_stack_id,path_key,path,frontier_serial,removed_ms) VALUES($1,$2,$3,$4,$5) ON CONFLICT(layer_stack_id,path_key) DO UPDATE SET path=excluded.path, frontier_serial=excluded.frontier_serial, removed_ms=excluded.removed_ms; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_high_water.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_high_water.sql new file mode 100644 index 0000000000..c76bf13a1e --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_high_water.sql @@ -0,0 +1 @@ +SELECT next_history_serial - 1 FROM history_meta WHERE id=1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_next_serial.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_next_serial.sql new file mode 100644 index 0000000000..a7f0c03ba9 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_next_serial.sql @@ -0,0 +1 @@ +UPDATE history_meta SET next_history_serial=next_history_serial+1 WHERE id=1 RETURNING next_history_serial-1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_node_serial.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_node_serial.sql new file mode 100644 index 0000000000..5a3ff1e15c --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_node_serial.sql @@ -0,0 +1 @@ +SELECT history_serial FROM "commit" WHERE commit_id=$1 UNION ALL SELECT history_serial FROM layer WHERE layer_id=$1 LIMIT 1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_stack_high_water.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_stack_high_water.sql new file mode 100644 index 0000000000..fab8fefa98 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/position_stack_high_water.sql @@ -0,0 +1 @@ +SELECT COALESCE(MAX(history_serial),0) FROM (SELECT history_serial FROM "commit" WHERE layer_stack_id=$1 UNION ALL SELECT history_serial FROM layer WHERE layer_stack_id=$1) AS serials; diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs index 3fb0af5afa..fa3115e6ba 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs @@ -33,6 +33,9 @@ pub(crate) const SCHEMA: &str = concat!( ); /// The Store tables a schema had before the GC tables ([`GC`]) were appended. const PRE_GC_SCHEMA: &str = include_str!("../../../sql/postgresql/schema.sql"); +/// The `history_meta` position column as `schema.sql` writes it; it sits on +/// its own line, so removing the line leaves the text an older Store recorded. +const NEXT_SERIAL: &str = " next_history_serial bigint NOT NULL DEFAULT 1,\n"; /// The GC marker, position and admission tables; a Store written before them /// gains them at open ([`add_gc`]). const GC: &str = include_str!("../../../sql/postgresql/gc.sql"); @@ -127,6 +130,16 @@ fn pre_gc_source(acquisition: bool) -> String { } } +/// The current source before [`NEXT_SERIAL`] was written. +fn source_without_next_serial(acquisition: bool) -> String { + source(acquisition).replacen(NEXT_SERIAL, "", 1) +} + +/// The pre-GC source before [`NEXT_SERIAL`] was written. +fn pre_gc_source_without_next_serial(acquisition: bool) -> String { + pre_gc_source(acquisition).replacen(NEXT_SERIAL, "", 1) +} + /// Creates the new schema `tx.schema` with the Store and acquisition tables, /// its policy row and its catalog row, inside the caller's transaction. A /// schema of that name that already exists is refused before anything is @@ -211,10 +224,16 @@ pub(crate) fn add_gc(session: &Session) -> Result<(), PersistenceError> { return Err(BackendError::Integrity.into()); }; let recorded_source: String = row.get(0)?; - let acquisition = match (recorded_source == pre_gc_source(false), recorded_source == pre_gc_source(true)) { - (true, _) => false, - (false, true) => true, - (false, false) => return Err(BackendError::Integrity.into()), + let acquisition = if recorded_source == pre_gc_source(false) + || recorded_source == pre_gc_source_without_next_serial(false) + { + false + } else if recorded_source == pre_gc_source(true) + || recorded_source == pre_gc_source_without_next_serial(true) + { + true + } else { + return Err(BackendError::Integrity.into()); }; let tables = tx.query(TABLE_NAMES, vec![])?; let actual: Vec = tables.iter().map(|r| r.get(0)).collect::>()?; @@ -239,6 +258,71 @@ pub(crate) fn add_gc(session: &Session) -> Result<(), PersistenceError> { }) } +/// Adds the position column `history_meta.next_history_serial` (T1.4) to a +/// Store written before it, then records the current schema: an additive +/// migration under the same conditions as [`add_gc`]. A schema that already +/// has the column is left alone; a recorded source or definition that is not +/// one of the earlier schemas is refused, never migrated. PostgreSQL has no +/// `commit`/`layer` serial columns yet, so only the column and one +/// `project_epoch` row per project are recorded here; the serial backfill is +/// SQLite's. +pub(crate) fn add_position_state(session: &Session) -> Result<(), PersistenceError> { + // A Store that already has the column is left alone: one statement, no + // transaction and no lane wait. + let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { + let sql = format!( + "SELECT 1 FROM information_schema.columns WHERE table_schema='{}' AND table_name='history_meta' AND column_name='next_history_serial'", + tx.schema + ); + Ok(!tx.query(&sql, vec![])?.is_empty()) + })?; + if present { + return Ok(()); + } + session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { + // Re-check under the lane: another process may have migrated first. + let present = tx.query( + "SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='history_meta' AND column_name='next_history_serial'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let [row] = rows.as_slice() else { + return Err(BackendError::Integrity.into()); + }; + let recorded_source: String = row.get(0)?; + let acquisition = if recorded_source == source(false) + || recorded_source == source_without_next_serial(false) + { + false + } else if recorded_source == source(true) + || recorded_source == source_without_next_serial(true) + { + true + } else { + return Err(BackendError::Integrity.into()); + }; + if row.get::(1)? != definition(tx)? { + return Err(BackendError::Integrity.into()); + } + tx.batch("ALTER TABLE history_meta ADD COLUMN next_history_serial bigint NOT NULL DEFAULT 1;")?; + tx.batch( + "INSERT INTO project_epoch(layer_stack_id, epoch) SELECT s.layer_stack_id, 0 FROM layer_stack s WHERE NOT EXISTS (SELECT 1 FROM project_epoch e WHERE e.layer_stack_id = s.layer_stack_id);", + )?; + let definition = definition(tx)?; + tx.query( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", + vec![Param::Text(source(acquisition)), Param::Text(definition)], + )?; + Ok(()) + }) +} + /// What an open's check read: the catalog's incarnation, whether the schema /// has the acquisition tables, the recorded binding, the frozen policy and /// the server's version. diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs index 8f4a225948..dc4eb5ea1f 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema_diagnostic.rs @@ -89,6 +89,7 @@ history_meta catalog_incarnation bigint NO history_meta binding_key bytea NO history_meta identity_format bigint NO history_meta schema_version bigint NO +history_meta next_history_serial bigint NO history_meta schema_source text NO history_meta schema_definition text NO init_entry operation_id bigint NO diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs index e9ce857540..188dcb1a9b 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs @@ -64,6 +64,17 @@ pub(crate) fn definition(c: &Connection, w: &RefCell) -> Result &'static str { + let start = HISTORY + .find(" next_history_serial INTEGER NOT NULL DEFAULT 1") + .expect("history schema holds the next serial column"); + let end = start + HISTORY[start..].find('\n').expect("column line ends") + 1; + &HISTORY[start..end] +} + /// The `commit` and `layer` position column as `history.sql` writes it; it /// appears once in each of the two tables. fn history_serial_column() -> &'static str { @@ -83,14 +94,16 @@ fn trace_salt_column() -> &'static str { &METADATA[start..end] } -/// `source` as a Store written before the three position columns recorded it: -/// the two `history_serial` columns and `store_policy.trace_salt` removed. The -/// table migrations compare against this so a Store from before those columns -/// still matches its recorded source. -fn without_gc_columns(source: String) -> String { +/// `source` as a Store written before the position columns recorded it: the +/// two `history_serial` columns, `store_policy.trace_salt` and +/// `history_meta.next_history_serial` removed. The table migrations compare +/// against this so a Store from before those columns still matches its +/// recorded source. +fn without_position_columns(source: String) -> String { source .replacen(history_serial_column(), "", 2) .replacen(trace_salt_column(), "", 1) + .replacen(next_history_serial_column(), "", 1) } /// The ALTERs and salt backfill an old Store needs; existing rows keep serial @@ -114,6 +127,93 @@ fn add_position_columns(tx: &Transaction<'_>) -> Result<(), BackendError> { )?; Ok(()) } + +/// The serials and epochs a Store written before T1.4 needs: Commits take the +/// next Store serials in `(created_ms, commit_id)` order, then each project's +/// genesis Layer takes the next one in stack-id order; every other Layer +/// republishes its source Commit and carries that Commit's serial. Existing +/// serials are kept, `project_epoch` gets one zero row per project and +/// `next_history_serial` never falls below `MAX(serial) + 1`. +fn backfill_position_state(tx: &Transaction<'_>) -> Result<(), BackendError> { + let next: i64 = tx + .query( + "SELECT next_history_serial FROM history_meta WHERE id=1", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let commit_max: i64 = tx + .query( + "SELECT COALESCE(MAX(history_serial),0) FROM \"commit\"", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let layer_max: i64 = tx + .query("SELECT COALESCE(MAX(history_serial),0) FROM layer", vec![])? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + let base = next.max(commit_max.max(layer_max) + 1); + tx.query( + &format!( + "WITH ordered(commit_id,serial) AS ( \ + SELECT commit_id,{base}+ROW_NUMBER() OVER (ORDER BY created_ms,commit_id)-1 \ + FROM \"commit\" WHERE history_serial=0) \ + UPDATE \"commit\" SET history_serial=( \ + SELECT ordered.serial FROM ordered \ + WHERE ordered.commit_id=\"commit\".commit_id) \ + WHERE history_serial=0" + ), + vec![], + )?; + let after_commits: i64 = tx + .query( + "SELECT COALESCE(MAX(history_serial),0)+1 FROM \"commit\"", + vec![], + )? + .first() + .ok_or(BackendError::Integrity)? + .get(0)?; + tx.query( + &format!( + "WITH ordered(layer_id,serial) AS ( \ + SELECT layer_id,{after_commits}+ROW_NUMBER() OVER ( \ + ORDER BY layer_stack_id,layer_id)-1 \ + FROM layer WHERE history_serial=0 AND parent_layer_id IS NULL) \ + UPDATE layer SET history_serial=( \ + SELECT ordered.serial FROM ordered \ + WHERE ordered.layer_id=layer.layer_id) \ + WHERE history_serial=0 AND parent_layer_id IS NULL" + ), + vec![], + )?; + tx.query( + "UPDATE layer SET history_serial=( \ + SELECT \"commit\".history_serial FROM \"commit\" \ + WHERE \"commit\".layer_stack_id=layer.layer_stack_id \ + AND \"commit\".commit_id=layer.source_commit_id) \ + WHERE history_serial=0 AND source_commit_id IS NOT NULL", + vec![], + )?; + tx.query( + "INSERT INTO project_epoch(layer_stack_id,epoch) \ + SELECT s.layer_stack_id,0 FROM layer_stack s \ + WHERE NOT EXISTS ( \ + SELECT 1 FROM project_epoch e WHERE e.layer_stack_id=s.layer_stack_id)", + vec![], + )?; + tx.query( + "UPDATE history_meta SET next_history_serial=MAX(next_history_serial,( \ + SELECT COALESCE(MAX(history_serial),0)+1 FROM ( \ + SELECT history_serial FROM \"commit\" \ + UNION ALL SELECT history_serial FROM layer))) WHERE id=1", + vec![], + )?; + Ok(()) +} pub(crate) fn check( c: &Connection, w: &RefCell, @@ -227,8 +327,8 @@ impl super::session::connection::Session { let Some(row) = rows.first() else { return Ok(()); }; - let current = - without_gc_columns(source((self.layout, self.acquisition))).replacen(GC, "", 1); + let current = without_position_columns(source((self.layout, self.acquisition))) + .replacen(GC, "", 1); if row.get::(0)? != current || row.get::(1)? != definition(tx.connection, tx.work)? { @@ -271,7 +371,7 @@ impl super::session::connection::Session { }; // The schema this migration records is the one before the // lineage and merge tables; later migrations add those. - let current = without_gc_columns(source((self.layout, self.acquisition))) + let current = without_position_columns(source((self.layout, self.acquisition))) .replacen(DIGESTS, "", 1) .replacen(SCRATCH, "", 1) .replacen(LINEAGE, "", 1) @@ -319,7 +419,7 @@ impl super::session::connection::Session { let Some(row) = rows.first() else { return Ok(()); }; - let current = without_gc_columns(source((self.layout, self.acquisition))) + let current = without_position_columns(source((self.layout, self.acquisition))) .replacen(DIGESTS, "", 1) .replacen(SCRATCH, "", 1) .replacen(GC, "", 1); @@ -357,7 +457,7 @@ impl super::session::connection::Session { let Some(row) = rows.first() else { return Ok(()); }; - let current = without_gc_columns(source((self.layout, self.acquisition))) + let current = without_position_columns(source((self.layout, self.acquisition))) .replacen(DIGESTS, "", 1) .replacen(GC, "", 1); if row.get::(0)? != current.replacen(SCRATCH, "", 1) @@ -394,8 +494,8 @@ impl super::session::connection::Session { let Some(row) = rows.first() else { return Ok(()); }; - let current = - without_gc_columns(source((self.layout, self.acquisition))).replacen(GC, "", 1); + let current = without_position_columns(source((self.layout, self.acquisition))) + .replacen(GC, "", 1); if row.get::(0)? != current.replacen(DIGESTS, "", 1) || row.get::(1)? != definition(tx.connection, tx.work)? { @@ -444,6 +544,50 @@ impl super::session::connection::Session { Ok(()) }) } + /// Adds the position state T1.4 introduced (`history_meta.next_history_serial`, + /// appended NOT NULL DEFAULT 1) and backfills the `commit`/`layer` serials + /// and `project_epoch` rows in the same write transaction: an additive + /// migration (REBUILD.md kind 2) under the same recorded-schema gate as + /// [`Self::add_gc_columns`]. `recorded == current` also matches a Store an + /// earlier step of this same open just brought forward, because those steps + /// record the current source before this column is appended. + pub(crate) fn add_position_state(&self) -> Result<(), BackendError> { + self.run::<_, BackendError>(true, |tx| { + let present = tx.query( + "SELECT 1 FROM pragma_table_info('history_meta') WHERE name='next_history_serial'", + vec![], + )?; + if !present.is_empty() { + return Ok(()); + } + let rows = tx.query( + "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", + vec![], + )?; + let Some(row) = rows.first() else { + return Ok(()); + }; + let current = source((self.layout, self.acquisition)); + let before = current.replacen(next_history_serial_column(), "", 1); + let recorded = row.get::(0)?; + if (recorded != before && recorded != current) + || row.get::(1)? != definition(tx.connection, tx.work)? + { + return Ok(()); + } + tx.query( + "ALTER TABLE history_meta ADD COLUMN next_history_serial INTEGER NOT NULL DEFAULT 1", + vec![], + )?; + backfill_position_state(tx)?; + let definition = definition(tx.connection, tx.work)?; + tx.query( + "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", + vec![Param::Text(current), Param::Text(definition)], + )?; + Ok(()) + }) + } pub(crate) fn validate( &self, catalog_id: layerfs_history::CatalogId, diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index 5e0f2f7aed..345873410b 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -183,6 +183,30 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "removal_stack" => Ok(include_str!( "../../../../sql/sqlite/queries/history/removal_stack.sql" )), + "position_high_water" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/position_high_water.sql" + )), + "position_stack_high_water" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/position_stack_high_water.sql" + )), + "position_next_serial" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/position_next_serial.sql" + )), + "position_node_serial" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/position_node_serial.sql" + )), + "paths_marker" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/paths_marker.sql" + )), + "paths_markers" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/paths_markers.sql" + )), + "paths_upsert" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/paths_upsert.sql" + )), + "paths_advance_epoch" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/paths_advance_epoch.sql" + )), _ => Err(BackendError::Integrity), } } diff --git a/engine/crates/layerfs-persistence/src/history/catalog.rs b/engine/crates/layerfs-persistence/src/history/catalog.rs index 1a5432ca47..498ac93606 100644 --- a/engine/crates/layerfs-persistence/src/history/catalog.rs +++ b/engine/crates/layerfs-persistence/src/history/catalog.rs @@ -209,4 +209,26 @@ impl HistoryCatalog for HistoryProvider { fn reserve_inodes(&self, request: &ReserveRequest) -> HistoryResult { self.write(|tx| allocation::reserve_inodes(tx, self.catalog_id, request)) } + + fn mark_path_removed( + &self, + stack: LayerStackId, + path: &[u8], + frontier_serial: u64, + removed_ms: i64, + ) -> HistoryResult<()> { + self.write(|tx| super::paths::mark(tx, stack, path, frontier_serial, removed_ms)) + } + + fn remove_path(&self, request: &PathRemovalRequest) -> HistoryResult { + self.write(|tx| super::paths::remove_path(tx, request)) + } + + fn path_removals(&self, stack: LayerStackId) -> HistoryResult> { + self.read(|tx| super::paths::markers(tx, stack)) + } + + fn node_history_serial(&self, node: NodeId) -> HistoryResult> { + self.read_statement(|tx| super::position::node_serial(tx, node)) + } } diff --git a/engine/crates/layerfs-persistence/src/history/commit.rs b/engine/crates/layerfs-persistence/src/history/commit.rs index 6c2cda501a..fc7120f3bc 100644 --- a/engine/crates/layerfs-persistence/src/history/commit.rs +++ b/engine/crates/layerfs-persistence/src/history/commit.rs @@ -206,6 +206,8 @@ pub(crate) fn new_commit( second_parent, meta, }; + // Every new Commit takes the next Store serial in this transaction. + let serial = super::position::next_serial(tx)?; // The insert names a new identity; an existing one is then verified. let inserted = tx .execute( @@ -232,6 +234,7 @@ pub(crate) fn new_commit( .map_err(|_| HistoryError::Capacity("metadata bytes"))?, super::facts::encode(&record.meta.lww, &record.meta.units)?, record.meta.displaced.map(|id| id.to_bytes().to_vec()), + i64::try_from(serial).map_err(|_| HistoryError::Capacity("history serials"))?, ], ) .map_err(sql)?; diff --git a/engine/crates/layerfs-persistence/src/history/layerstack.rs b/engine/crates/layerfs-persistence/src/history/layerstack.rs index 505c071114..661f911ac5 100644 --- a/engine/crates/layerfs-persistence/src/history/layerstack.rs +++ b/engine/crates/layerfs-persistence/src/history/layerstack.rs @@ -67,6 +67,7 @@ pub(crate) fn initialize_layerstack( return Err(HistoryError::Integrity("LayerStack name")); } let genesis = LayerId::derive(request.stack, None, request.genesis_root); + let serial = super::position::next_serial(tx)?; tx.execute( INSERT_STACK, params![ @@ -91,6 +92,7 @@ pub(crate) fn initialize_layerstack( Option::>::None, i64::try_from(request.logical).map_err(|_| HistoryError::Capacity("logical bytes"))?, Vec::::new(), + i64::try_from(serial).map_err(|_| HistoryError::Capacity("history serials"))?, ], ) .map_err(sql)?; @@ -382,6 +384,9 @@ pub(crate) fn add_layer( source_commit: Some(request.commit), logical: commit.meta.statistics.logical, }; + // A Layer that republishes a Commit carries that Commit's serial. + let serial = + super::position::node_serial(tx, layerfs_history::NodeId::Commit(commit.id))?.unwrap_or(0); if !verify_layer(tx, &layer)? { tx.execute( INSERT_LAYER, @@ -396,6 +401,7 @@ pub(crate) fn add_layer( // Overwritten Layers are addressed by height (`overwritten`); // only Layers published before #353 list them here. Vec::::new(), + i64::try_from(serial).map_err(|_| HistoryError::Capacity("history serials"))?, ], ) .map_err(sql)?; diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index 966992d594..865402ca69 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -11,6 +11,8 @@ mod layerstack; mod lineage; mod lineage_backfill; mod merge; +mod paths; +mod position; mod provider; mod removal; mod rows; diff --git a/engine/crates/layerfs-persistence/src/history/paths.rs b/engine/crates/layerfs-persistence/src/history/paths.rs new file mode 100644 index 0000000000..6ab1ff46c7 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/paths.rs @@ -0,0 +1,112 @@ +//! The `project.remove_file` marker: one `path_removed` row per project and +//! path key, advanced in one write transaction with the project's epoch. + +use super::position; +use super::rows::{cell, many, one, unsigned}; +use super::transaction::Transaction; +use super::{layerstack, params}; +use crate::backend::records::Record as Row; +use layerfs_history::error::{HistoryError, HistoryResult, Missing}; +use layerfs_history::identity::LayerStackId; +use layerfs_history::path::path_key; +use layerfs_history::records::{ + PathRemovalKind, PathRemovalOutcome, PathRemovalRecord, PathRemovalRequest, +}; + +const MARKER: &str = "paths_marker"; + +const MARKERS: &str = "paths_markers"; + +const UPSERT: &str = "paths_upsert"; + +const ADVANCE_EPOCH: &str = "paths_advance_epoch"; + +/// One marker row decoded from the Store. +fn marker(row: &Row) -> HistoryResult { + Ok(PathRemovalRecord { + path: cell(row, 0)?, + frontier_serial: unsigned(row, 1)?, + removed_ms: cell::(row, 2)?, + }) +} + +/// Inserts or advances one marker, without the UpToDate decision. +pub(crate) fn mark( + tx: &Transaction<'_>, + stack: LayerStackId, + path: &[u8], + frontier_serial: u64, + removed_ms: i64, +) -> HistoryResult<()> { + if path.is_empty() { + return Err(HistoryError::InvalidInput("removed path")); + } + let key = path_key(path); + let frontier = + i64::try_from(frontier_serial).map_err(|_| HistoryError::Capacity("path frontier"))?; + tx.execute( + UPSERT, + params![stack.as_slice(), key.as_slice(), path, frontier, removed_ms], + )?; + Ok(()) +} + +/// Applies one `project.remove_file` marker, or reports it current. +pub(crate) fn remove_path( + tx: &Transaction<'_>, + request: &PathRemovalRequest, +) -> HistoryResult { + if layerstack::layer_stack(tx, request.stack)?.is_none() { + return Err(HistoryError::Missing(Missing::LayerStack)); + } + let key = path_key(&request.path); + let stored = one( + tx, + MARKER, + [request.stack.as_slice(), key.as_slice()], + marker, + )?; + let stack_high = position::stack_high_water(tx, request.stack)?; + if let Some(stored) = &stored { + if stored.path != request.path { + return Err(HistoryError::Integrity("path marker key")); + } + if stack_high <= stored.frontier_serial { + return Ok(PathRemovalOutcome { + kind: PathRemovalKind::UpToDate, + stack: request.stack, + path: stored.path.clone(), + frontier_serial: stored.frontier_serial, + removed_ms: stored.removed_ms, + }); + } + } + // The new frontier is the Store high-water, read in this transaction. + let frontier = position::high_water(tx)?; + mark( + tx, + request.stack, + &request.path, + frontier, + request.removed_ms, + )?; + let advanced = tx.execute(ADVANCE_EPOCH, [request.stack.as_slice()])?; + if advanced != 1 { + return Err(HistoryError::Conflict("project epoch")); + } + Ok(PathRemovalOutcome { + kind: PathRemovalKind::Removed, + stack: request.stack, + path: request.path.clone(), + frontier_serial: frontier, + removed_ms: request.removed_ms, + }) +} + +/// Every marker of one Stack. +pub(crate) fn markers( + tx: &Transaction<'_>, + stack: LayerStackId, +) -> HistoryResult> { + many(tx, MARKERS, [stack.as_slice()], marker) +} diff --git a/engine/crates/layerfs-persistence/src/history/position.rs b/engine/crates/layerfs-persistence/src/history/position.rs new file mode 100644 index 0000000000..2f78dad153 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/position.rs @@ -0,0 +1,47 @@ +//! The Store history position: the `history_meta.next_history_serial` counter +//! a `remove_file` frontier names, and the serial each Commit and Layer +//! records. +//! +//! Every new Commit takes `next_history_serial` and advances the counter in +//! the same write transaction; a Layer that republishes a Commit carries that +//! Commit's serial. The high-water a marker stores is `next_history_serial - +//! 1`, read in the marker transaction, so a node created concurrently is +//! ordered by its serial and never by a later read. + +use super::params; +use super::rows::{one, unsigned}; +use super::transaction::Transaction; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::identity::{LayerStackId, NodeId}; + +const HIGH_WATER: &str = "position_high_water"; + +const NEXT_SERIAL: &str = "position_next_serial"; + +const STACK_HIGH_WATER: &str = "position_stack_high_water"; + +const NODE_SERIAL: &str = "position_node_serial"; + +/// The Store high-water: `next_history_serial - 1`. +pub(crate) fn high_water(tx: &Transaction<'_>) -> HistoryResult { + one(tx, HIGH_WATER, params![], |row| unsigned(row, 0))? + .ok_or(HistoryError::Integrity("history position")) +} + +/// Takes the next Store serial, advancing `next_history_serial` in this +/// transaction. +pub(crate) fn next_serial(tx: &Transaction<'_>) -> HistoryResult { + one(tx, NEXT_SERIAL, params![], |row| unsigned(row, 0))? + .ok_or(HistoryError::Integrity("history position")) +} + +/// The largest `history_serial` in one Stack, for the `UpToDate` test. +pub(crate) fn stack_high_water(tx: &Transaction<'_>, stack: LayerStackId) -> HistoryResult { + one(tx, STACK_HIGH_WATER, [stack.as_slice()], |row| unsigned(row, 0))? + .ok_or(HistoryError::Integrity("history position")) +} + +/// The `history_serial` one Commit or Layer records, when it exists. +pub(crate) fn node_serial(tx: &Transaction<'_>, node: NodeId) -> HistoryResult> { + one(tx, NODE_SERIAL, [node.as_slice()], |row| unsigned(row, 0)) +} diff --git a/engine/crates/layerfs-persistence/src/store/open.rs b/engine/crates/layerfs-persistence/src/store/open.rs index fd559148fd..213707bcc7 100644 --- a/engine/crates/layerfs-persistence/src/store/open.rs +++ b/engine/crates/layerfs-persistence/src/store/open.rs @@ -108,6 +108,7 @@ impl SqliteHandles { // before `add_gc` records the full schema. session.add_gc_columns()?; session.add_gc()?; + session.add_position_state()?; } let handles = Self::validated(session, directory, binding, cursor_key)?; if writable { diff --git a/engine/crates/layerfs-persistence/src/store/postgresql.rs b/engine/crates/layerfs-persistence/src/store/postgresql.rs index c9926202be..85741559e2 100644 --- a/engine/crates/layerfs-persistence/src/store/postgresql.rs +++ b/engine/crates/layerfs-persistence/src/store/postgresql.rs @@ -181,6 +181,7 @@ impl PostgresHandles { let session = Arc::new(Session::connect(selection, writable)?); if writable { schema::add_gc(&session)?; + schema::add_position_state(&session)?; } Self::validated(session, selection.check, Some(binding), cursor_key) .map(|(_, handles)| handles) diff --git a/engine/crates/layerfs-persistence/tests/gc_backfill.rs b/engine/crates/layerfs-persistence/tests/gc_backfill.rs new file mode 100644 index 0000000000..aa65bd1518 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/gc_backfill.rs @@ -0,0 +1,492 @@ +//! T1.4: a Store written before the position state opens writable and gains +//! `history_meta.next_history_serial`, the backfilled `commit`/`layer` +//! serials and one `project_epoch` row per project. A Store that already has +//! the column is left alone, and a serial already recorded is never lowered. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRecord, + CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, + LayerRecord, LayerStackId, StackInitialization, +}; +use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; +use layerfs_storage::StoragePolicy; +use support::{branch_id, name, root, stack, Temp, BINDING}; + +/// The `history_meta` position column as `history.sql` writes it. +const NEXT_SERIAL: &str = " next_history_serial INTEGER NOT NULL DEFAULT 1,\n"; + +fn config(path: &std::path::Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn create(path: &std::path::Path) -> SqliteHandles { + SqliteHandles::create( + config(path), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .expect("Store creation") +} + +fn writable( + path: &std::path::Path, +) -> Result { + SqliteHandles::open_writable(config(path), BINDING, [71; 32]) +} + +fn definition(sql: &rusqlite::Connection) -> String { + let mut statement = sql + .prepare("SELECT type,name,sql FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' ORDER BY type,name") + .unwrap(); + let rows = statement + .query_map([], |r| Ok([r.get::<_, String>(0)?, r.get(1)?, r.get(2)?])) + .unwrap(); + rows.map(|r| r.unwrap().map(|v| v + "\n").concat()) + .collect() +} + +/// Rewrites a fresh Store into the format written before T1.4: no +/// `history_meta.next_history_serial`, every Commit and Layer at serial 0 and +/// no `project_epoch` row, recorded as that revision wrote it. Returns the +/// fresh source string it replaced so a test can assert the migrated Store +/// records the current schema again. +fn strip_position_state(path: &std::path::Path) -> String { + let sql = rusqlite::Connection::open(path).unwrap(); + let fresh: String = sql + .query_row("SELECT schema_source FROM history_meta", [], |r| r.get(0)) + .unwrap(); + sql.execute_batch( + "ALTER TABLE history_meta DROP COLUMN next_history_serial; \ + UPDATE \"commit\" SET history_serial=0; \ + UPDATE layer SET history_serial=0; \ + DELETE FROM project_epoch;", + ) + .unwrap(); + let old = fresh.replacen(NEXT_SERIAL, "", 1); + assert_ne!(old, fresh, "a fresh Store holds the position column"); + sql.execute( + "UPDATE history_meta SET schema_source=?1,schema_definition=?2", + rusqlite::params![old, definition(&sql)], + ) + .unwrap(); + fresh +} + +fn initialize(handles: &SqliteHandles, tag: u8) -> layerfs_history::LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: stack(tag), + name: name(&format!("project-{tag}")), + scope: root(200), + profile: root(201), + genesis_root: root(1), + }) + .unwrap() +} + +fn fork( + handles: &SqliteHandles, + stack: LayerStackId, + branch: BranchId, + tag: u8, + source: ForkSource, +) { + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: name(&format!("branch-{tag}")), + source, + }) + .unwrap(); +} + +fn commit(handles: &SqliteHandles, branch: BranchId, tag: u8) -> CommitRecord { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root: root(tag), + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +fn column_default(sql: &rusqlite::Connection, table: &str, column: &str) -> (i64, Option) { + sql.query_row( + "SELECT \"notnull\",dflt_value FROM pragma_table_info(?1) WHERE name=?2", + rusqlite::params![table, column], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap() +} + +fn next_serial(sql: &rusqlite::Connection) -> i64 { + sql.query_row("SELECT next_history_serial FROM history_meta", [], |r| { + r.get(0) + }) + .unwrap() +} + +fn epochs(sql: &rusqlite::Connection) -> Vec<(Vec, i64)> { + let mut statement = sql + .prepare("SELECT layer_stack_id,epoch FROM project_epoch ORDER BY layer_stack_id") + .unwrap(); + statement + .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap() + .map(|r| r.unwrap()) + .collect() +} + +#[test] +fn an_empty_store_keeps_its_next_serial_and_gains_no_epoch() { + let t = Temp::new("gc-backfill-empty"); + let path = t.join("db"); + drop(create(&path)); + let fresh = strip_position_state(&path); + + writable(&path).expect("a pre-T1.4 Store opens writable"); + drop(writable(&path).expect("an upgraded Store reopens")); + + let sql = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + column_default(&sql, "history_meta", "next_history_serial"), + (1, Some("1".to_owned())), + "the column is appended NOT NULL DEFAULT 1" + ); + assert_eq!(next_serial(&sql), 1, "no history keeps the serial at 1"); + assert!( + epochs(&sql).is_empty(), + "a Store with no project has no epoch" + ); + let (source, recorded): (String, String) = sql + .query_row( + "SELECT schema_source,schema_definition FROM history_meta", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(source, fresh, "the Store records the current schema"); + assert_eq!( + recorded, + definition(&sql), + "the recorded definition is the live one" + ); + drop(sql); + support::reopen(&path); +} + +#[test] +fn commits_layers_and_epochs_are_backfilled_in_deterministic_order() { + let t = Temp::new("gc-backfill-graph"); + let path = t.join("db"); + let fresh; + let (stack_one, stack_two); + { + let handles = create(&path); + let one = initialize(&handles, 1); + let two = initialize(&handles, 2); + let a = branch_id(1); + fork(&handles, one.id, a, 1, ForkSource::Layer(one.head_layer)); + commit(&handles, a, 10); + publish(&handles, one.id, a); + commit(&handles, a, 11); + publish(&handles, one.id, a); + + let b = branch_id(2); + fork(&handles, two.id, b, 2, ForkSource::Layer(two.head_layer)); + commit(&handles, b, 20); + publish(&handles, two.id, b); + stack_one = one.id; + stack_two = two.id; + drop(handles); + fresh = strip_position_state(&path); + } + + writable(&path).expect("a pre-T1.4 Store opens writable"); + let sql = rusqlite::Connection::open(&path).unwrap(); + + // Commits take consecutive Store serials in (created_ms, commit_id) order. + let mut commits: Vec<(Vec, i64, i64)> = { + let mut statement = sql + .prepare("SELECT commit_id,created_ms,history_serial FROM \"commit\"") + .unwrap(); + let rows = statement + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap(); + rows.map(|r| r.unwrap()).collect() + }; + commits.sort_by(|a, b| (a.1, &a.0).cmp(&(b.1, &b.0))); + let ordered: Vec = commits.iter().map(|c| c.2).collect(); + assert_eq!( + ordered, + (1..=ordered.len() as i64).collect::>(), + "serials are dense and follow the deterministic order" + ); + + // Per project the genesis Layer takes the next serial, in stack-id order. + let genesis_one: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE parent_layer_id IS NULL AND layer_stack_id=?1", + rusqlite::params![stack_one.as_slice()], + |r| r.get(0), + ) + .unwrap(); + let genesis_two: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE parent_layer_id IS NULL AND layer_stack_id=?1", + rusqlite::params![stack_two.as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(genesis_one.min(genesis_two), commits.len() as i64 + 1); + assert_eq!(genesis_one.max(genesis_two), commits.len() as i64 + 2); + + // Every republished tree carries the serial of the Commit it publishes. + for (commit_id, _, serial) in &commits { + let layer: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE source_commit_id=?1", + rusqlite::params![commit_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(layer, *serial, "a Layer carries its source Commit serial"); + } + assert_eq!( + next_serial(&sql), + commits.len() as i64 + 3, + "the next serial follows the highest assigned one" + ); + + // One epoch row per project, both zero. + let rows = epochs(&sql); + assert_eq!(rows.len(), 2); + let mut expected = vec![stack_one, stack_two]; + expected.sort_by(|a, b| a.as_slice().cmp(b.as_slice())); + for (row, expected) in rows.iter().zip(expected.iter()) { + assert_eq!(row.0, expected.as_slice()); + assert_eq!(row.1, 0); + } + + let before = next_serial(&sql); + drop(sql); + drop(writable(&path).expect("reopening changes nothing")); + let sql = rusqlite::Connection::open(&path).unwrap(); + assert_eq!(next_serial(&sql), before, "reopening keeps the serials"); + assert_eq!(epochs(&sql).len(), 2); + let (source, recorded): (String, String) = sql + .query_row( + "SELECT schema_source,schema_definition FROM history_meta", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(source, fresh); + assert_eq!(recorded, definition(&sql)); +} + +#[test] +fn republishing_layers_share_their_source_commit_serial() { + let t = Temp::new("gc-backfill-republish"); + let path = t.join("db"); + let (shared_commit, first_layer, second_layer); + { + let handles = create(&path); + let one = initialize(&handles, 1); + let a = branch_id(1); + fork(&handles, one.id, a, 1, ForkSource::Layer(one.head_layer)); + commit(&handles, a, 10); + let first = publish(&handles, one.id, a); + // Advance the stack so publishing the older Commit again is not the + // no-change shortcut. + commit(&handles, a, 11); + let advanced = publish(&handles, one.id, a); + + let b = branch_id(2); + fork( + &handles, + one.id, + b, + 2, + ForkSource::Commit { + branch: a, + commit: first.source_commit.unwrap(), + }, + ); + // Re-publish the Commit that already has a Layer on branch `a`. + let second = publish(&handles, one.id, b); + assert_ne!(first.id, second.id); + assert_ne!(advanced.id, second.id); + assert_eq!(first.source_commit, second.source_commit); + shared_commit = first.source_commit.unwrap(); + first_layer = first.id; + second_layer = second.id; + drop(handles); + strip_position_state(&path); + } + + writable(&path).expect("a pre-T1.4 Store opens writable"); + let sql = rusqlite::Connection::open(&path).unwrap(); + let serial: i64 = sql + .query_row( + "SELECT history_serial FROM \"commit\" WHERE commit_id=?1", + rusqlite::params![shared_commit.as_slice()], + |r| r.get(0), + ) + .unwrap(); + for layer in [first_layer, second_layer] { + let layer_serial: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE layer_id=?1", + rusqlite::params![layer.as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + layer_serial, serial, + "a republishing Layer carries its source Commit serial" + ); + } + assert_eq!(epochs(&sql).len(), 1); + drop(sql); +} + +#[test] +fn existing_serials_are_kept_and_new_ones_start_above_the_maximum() { + let t = Temp::new("gc-backfill-monotonic"); + let path = t.join("db"); + let (kept_commit, other_commit); + { + let handles = create(&path); + let one = initialize(&handles, 1); + let a = branch_id(1); + fork(&handles, one.id, a, 1, ForkSource::Layer(one.head_layer)); + let kept = commit(&handles, a, 10); + publish(&handles, one.id, a); + let other = commit(&handles, a, 11); + publish(&handles, one.id, a); + kept_commit = kept.id; + other_commit = other.id; + drop(handles); + strip_position_state(&path); + } + + // A Store brought part-way forward may already hold serials; they are kept + // and the backfill starts above the highest one. + { + let sql = rusqlite::Connection::open(&path).unwrap(); + sql.execute( + "UPDATE \"commit\" SET history_serial=100 WHERE commit_id=?1", + rusqlite::params![kept_commit.as_slice()], + ) + .unwrap(); + // A Layer that already holds a serial is not rewritten either. + sql.execute( + "UPDATE layer SET history_serial=7 WHERE source_commit_id=?1", + rusqlite::params![kept_commit.as_slice()], + ) + .unwrap(); + drop(sql); + } + + writable(&path).expect("a part-way Store opens writable"); + let sql = rusqlite::Connection::open(&path).unwrap(); + let kept: i64 = sql + .query_row( + "SELECT history_serial FROM \"commit\" WHERE commit_id=?1", + rusqlite::params![kept_commit.as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(kept, 100, "an assigned Commit serial is never rewritten"); + let assigned: i64 = sql + .query_row( + "SELECT history_serial FROM \"commit\" WHERE commit_id=?1", + rusqlite::params![other_commit.as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(assigned, 101, "new serials start above the maximum"); + let existing_layer: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE source_commit_id=?1", + rusqlite::params![kept_commit.as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + existing_layer, 7, + "an assigned Layer serial is never rewritten" + ); + let genesis: i64 = sql + .query_row( + "SELECT history_serial FROM layer WHERE parent_layer_id IS NULL", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(genesis, 102, "a genesis Layer follows the commits"); + assert_eq!(next_serial(&sql), 103); + + // A high-water written later is never lowered by another open. + sql.execute("UPDATE history_meta SET next_history_serial=500", []) + .unwrap(); + drop(sql); + drop(writable(&path).expect("a forward Store reopens")); + let sql = rusqlite::Connection::open(&path).unwrap(); + assert_eq!(next_serial(&sql), 500); + drop(sql); +} diff --git a/engine/crates/layerfs-persistence/tests/old_format_columns.rs b/engine/crates/layerfs-persistence/tests/old_format_columns.rs index 877d5412f1..ff4da595c7 100644 --- a/engine/crates/layerfs-persistence/tests/old_format_columns.rs +++ b/engine/crates/layerfs-persistence/tests/old_format_columns.rs @@ -1,8 +1,8 @@ -//! A Store written before `commit`/`layer.history_serial` and -//! `store_policy.trace_salt` existed opens writable and gains them (T1.2b): -//! the ALTER step matches the recorded schema source and definition, gives -//! existing rows serial 0 and the Store a random salt, and records the -//! current schema. +//! A Store written before `commit`/`layer.history_serial`, +//! `store_policy.trace_salt` and `history_meta.next_history_serial` existed +//! opens writable and gains them (T1.2b, T1.4): the additive steps match the +//! recorded schema source and definition, give existing rows serial 0, the +//! Store a random salt and the next serial 1, and record the current schema. mod support; use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; use support::*; @@ -13,6 +13,8 @@ const GC: &str = include_str!("../sql/sqlite/gc.sql"); const HISTORY_SERIAL: &str = " history_serial INTEGER NOT NULL DEFAULT 0,\n"; /// The `store_policy` column as `metadata.sql` writes it. const TRACE_SALT: &str = " trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)\n"; +/// The `history_meta` next serial column as `history.sql` writes it (T1.4). +const NEXT_SERIAL: &str = " next_history_serial INTEGER NOT NULL DEFAULT 1,\n"; fn definition(sql: &rusqlite::Connection) -> String { let mut statement = sql @@ -34,6 +36,7 @@ fn make_old(path: &std::path::Path) { "ALTER TABLE \"commit\" DROP COLUMN history_serial; \ ALTER TABLE layer DROP COLUMN history_serial; \ ALTER TABLE store_policy DROP COLUMN trace_salt; \ + ALTER TABLE history_meta DROP COLUMN next_history_serial; \ DROP TABLE branch_retired; DROP TABLE layer_stack_retired; DROP TABLE path_removed; \ DROP TABLE project_epoch; DROP TABLE gc_intent; DROP TABLE gc_op; DROP TABLE gc_dead;", ) @@ -43,9 +46,11 @@ fn make_old(path: &std::path::Path) { .unwrap(); assert_eq!(source.matches(HISTORY_SERIAL).count(), 2); assert_eq!(source.matches(TRACE_SALT).count(), 1); + assert_eq!(source.matches(NEXT_SERIAL).count(), 1); let old = source .replacen(HISTORY_SERIAL, "", 2) .replacen(TRACE_SALT, "", 1) + .replacen(NEXT_SERIAL, "", 1) .replacen(GC, "", 1); assert_ne!(old, source, "the current schema holds the columns and GC"); sql.execute( diff --git a/engine/crates/layerfs-persistence/tests/old_format_open.rs b/engine/crates/layerfs-persistence/tests/old_format_open.rs index 733a8d88b3..d5fba96983 100644 --- a/engine/crates/layerfs-persistence/tests/old_format_open.rs +++ b/engine/crates/layerfs-persistence/tests/old_format_open.rs @@ -16,6 +16,8 @@ const GC: &str = include_str!("../sql/sqlite/gc.sql"); /// format this fixture builds lacks them. const HISTORY_SERIAL: &str = " history_serial INTEGER NOT NULL DEFAULT 0,\n"; const TRACE_SALT: &str = " trace_salt BLOB NOT NULL DEFAULT (X'0000000000000000000000000000000000000000000000000000000000000000') CHECK(length(trace_salt)=32)\n"; +/// The `history_meta` next serial column as `history.sql` writes it (T1.4). +const NEXT_SERIAL: &str = " next_history_serial INTEGER NOT NULL DEFAULT 1,\n"; /// The table's definition as the history schema writes it, with its trailing /// blank line: the text an old Store's recorded schema source lacks. @@ -38,14 +40,15 @@ fn definition(sql: &rusqlite::Connection) -> String { /// Rewrites a fresh Store into the format the daemons before `49499abac` /// wrote: no `binding_hint` table (nor the later #353 lineage and #355 merge -/// tables), no position columns, and the schema source and definition they -/// recorded. +/// tables), no position columns (including T1.4's next serial), and the +/// schema source and definition they recorded. fn make_old(path: &std::path::Path) { let sql = rusqlite::Connection::open(path).unwrap(); sql.execute_batch( "ALTER TABLE \"commit\" DROP COLUMN history_serial; \ ALTER TABLE layer DROP COLUMN history_serial; \ ALTER TABLE store_policy DROP COLUMN trace_salt; \ + ALTER TABLE history_meta DROP COLUMN next_history_serial; \ DROP TABLE binding_hint_block; DROP TABLE layer_lineage; DROP TABLE commit_lineage; \ DROP TABLE absorbed; DROP TABLE reach; DROP TABLE merge_operation; \ DROP TABLE merge_record; DROP TABLE file_digest; \ @@ -59,6 +62,7 @@ fn make_old(path: &std::path::Path) { let old = source .replacen(HISTORY_SERIAL, "", 2) .replacen(TRACE_SALT, "", 1) + .replacen(NEXT_SERIAL, "", 1) .replacen(&hint_table(), "", 1) .replacen(LINEAGE, "", 1) .replacen(SCRATCH, "", 1) diff --git a/engine/crates/layerfs-persistence/tests/path_removal.rs b/engine/crates/layerfs-persistence/tests/path_removal.rs new file mode 100644 index 0000000000..6074771ff2 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/path_removal.rs @@ -0,0 +1,172 @@ +//! `project.remove_file`: one marker, its Store-high-water frontier, the +//! `UpToDate` rule, the `/`-boundary subtree, and the per-project scope. +mod support; + +use layerfs_history::error::Missing; +use layerfs_history::path::NormalizedPath; +use layerfs_history::records::{ + CommitMeta, CommitOutcome, CommitRecord, CommitRequest, CommitVia, ForkRequest, ForkSource, + PathRemovalKind, PathRemovalRequest, StackInitialization, +}; +use layerfs_history::{BranchId, HistoryCatalog, HistoryError, LayerStackId, NodeId}; +use support::{branch_id, name, root, stack, Temp}; + +const REMOVED_AT: i64 = 1_700_000_000_000; + +fn project(history: &dyn HistoryCatalog, n: u8, text: &str) -> (LayerStackId, BranchId) { + let record = history + .initialize_layerstack(&StackInitialization { + logical: 0, + stack: stack(n), + name: name(text), + scope: root(200), + profile: root(201), + genesis_root: root(n), + }) + .unwrap(); + let branch = branch_id(n); + history + .fork(&ForkRequest { + stack: stack(n), + branch, + name: name("work"), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); + (stack(n), branch) +} + +fn commit(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRecord { + let record = history.branch(branch).unwrap().unwrap(); + let current = match record.head_commit { + Some(head) => history.commit(head).unwrap().unwrap().root, + None => history.layer(record.base_layer).unwrap().unwrap().root, + }; + let outcome = history + .commit_branch(&CommitRequest { + branch, + expected_head: record.head_commit, + expected_base: record.base_layer, + expected_root: current, + candidate_root: root(tag), + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap(); + let CommitOutcome::Committed { record, .. } = outcome else { + panic!("{outcome:?}"); + }; + *record +} + +fn remove( + history: &dyn HistoryCatalog, + stack: LayerStackId, + path: &str, + at: i64, +) -> layerfs_history::PathRemovalOutcome { + history + .remove_path(&PathRemovalRequest { + stack, + path: NormalizedPath::new(path).unwrap().as_bytes().to_vec(), + removed_ms: at, + }) + .unwrap() +} + +#[test] +fn marker_frontier_and_up_to_date_follow_the_store_position() { + let temp = Temp::new("path-removal"); + let history = support::create(&temp.join("store")); + let (shop, branch) = project(&history, 1, "shop"); + let genesis = history.layer_stack(shop).unwrap().unwrap().head_layer; + assert_eq!( + history.node_history_serial(NodeId::Layer(genesis)).unwrap(), + Some(1) + ); + let first = commit(&history, branch, 5); + assert_eq!( + history + .node_history_serial(NodeId::Commit(first.id)) + .unwrap(), + Some(2) + ); + let _second = commit(&history, branch, 6); + + let removed = remove(&history, shop, "/a", REMOVED_AT); + assert_eq!(removed.kind, PathRemovalKind::Removed); + assert_eq!(removed.stack, shop); + assert_eq!(removed.path, b"a"); + assert_eq!( + removed.frontier_serial, 3, + "frontier is next_history_serial - 1" + ); + assert_eq!(removed.removed_ms, REMOVED_AT); + + // No project node is above the marker frontier: the repeat is UpToDate + // and keeps the stored values. + let repeat = remove(&history, shop, "/a", REMOVED_AT + 1); + assert_eq!(repeat.kind, PathRemovalKind::UpToDate); + assert_eq!(repeat.frontier_serial, 3); + assert_eq!(repeat.removed_ms, REMOVED_AT); + + // A commit in another project raises the Store high-water but leaves + // this path UpToDate: it is not this project's node. + let (_, api) = project(&history, 2, "api"); + let _api_commit = commit(&history, api, 7); + let still = remove(&history, shop, "/a", REMOVED_AT + 2); + assert_eq!(still.kind, PathRemovalKind::UpToDate); + assert_eq!(still.frontier_serial, 3); + + // A new commit in this project advances the path. + let _third = commit(&history, branch, 8); + let advanced = remove(&history, shop, "/a", REMOVED_AT + 3); + assert_eq!(advanced.kind, PathRemovalKind::Removed); + assert_eq!(advanced.frontier_serial, 6); + + // One marker per (stack, path key); `a` and `ab` are different paths. + let ab = remove(&history, shop, "/ab", REMOVED_AT + 4); + assert_eq!(ab.kind, PathRemovalKind::Removed); + assert_eq!(ab.frontier_serial, 6); + let markers = history.path_removals(shop).unwrap(); + let paths: Vec<_> = markers.iter().map(|marker| marker.path.clone()).collect(); + assert_eq!(paths, vec![b"a".to_vec(), b"ab".to_vec()]); + assert_eq!(markers[0].frontier_serial, 6); + assert_eq!(markers[1].frontier_serial, 6); + // `ab` is not governed by the `a` marker. + assert!(!layerfs_history::path::governs(b"a", b"ab")); +} + +#[test] +fn a_raw_marker_frontier_also_yields_up_to_date() { + let temp = Temp::new("path-removal-raw"); + let history = support::create(&temp.join("store")); + let (shop, _branch) = project(&history, 1, "shop"); + history + .mark_path_removed(shop, b"old", 42, REMOVED_AT) + .unwrap(); + let stored = history.path_removals(shop).unwrap(); + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].frontier_serial, 42); + let current = remove(&history, shop, "old", REMOVED_AT + 1); + assert_eq!(current.kind, PathRemovalKind::UpToDate); + assert_eq!(current.frontier_serial, 42); + assert_eq!(current.removed_ms, REMOVED_AT); +} + +#[test] +fn an_unknown_project_is_missing() { + let temp = Temp::new("path-removal-missing"); + let history = support::create(&temp.join("store")); + assert_eq!( + history.remove_path(&PathRemovalRequest { + stack: stack(9), + path: b"a".to_vec(), + removed_ms: REMOVED_AT, + }), + Err(HistoryError::Missing(Missing::LayerStack)) + ); +} diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index 450bc135f6..7f28a16058 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -53,6 +53,7 @@ pub fn history(error: HistoryError) -> SdkRefusal { HistoryError::Capacity(_) => "Capacity", HistoryError::NameTaken(_) => "NameTaken", HistoryError::Removed(_) => "Removed", + HistoryError::Conflict(_) => "Conflict", HistoryError::StaleStore => "StaleStore", HistoryError::Unsupported(_) => "Unsupported", HistoryError::UnknownOutcome => "Unknown", diff --git a/engine/crates/layerfs-store-service/src/methods/args.rs b/engine/crates/layerfs-store-service/src/methods/args.rs index c6ee3bbc8d..ce5e687878 100644 --- a/engine/crates/layerfs-store-service/src/methods/args.rs +++ b/engine/crates/layerfs-store-service/src/methods/args.rs @@ -30,6 +30,14 @@ pub fn snapshot( Ok((reader, snap)) } +/// The project, its reader and the snapshot a read names, in that order. +pub fn snapshot_full( + service: &StoreService, + args: &SdkValue, +) -> Result<(LayerStackRecord, Arc, Snap), SdkRefusal> { + crate::operator::args::snapshot(&service.store, args) +} + /// The required absolute path argument `name`, checked. pub fn path(args: &SdkValue, name: &str) -> Result { files::path(text(args, name)?) diff --git a/engine/crates/layerfs-store-service/src/methods/dispatch.rs b/engine/crates/layerfs-store-service/src/methods/dispatch.rs index d71c543d94..73e1cf034a 100644 --- a/engine/crates/layerfs-store-service/src/methods/dispatch.rs +++ b/engine/crates/layerfs-store-service/src/methods/dispatch.rs @@ -22,6 +22,7 @@ pub fn call(service: &StoreService, call: &SdkCall) -> Answer { "store.stats" => store::stats(service), "project.stats" => project::stats(service, args), "project.list_branches" => project::list_branches(service, args), + "project.remove_file" => project::remove_file(service, args), "branch.fork" => branch::fork(service, args), "branch.list_commits" => branch::list_commits(service, args), "branch.commits" => branch::commits(service, args), diff --git a/engine/crates/layerfs-store-service/src/methods/file/glob.rs b/engine/crates/layerfs-store-service/src/methods/file/glob.rs index c4d978cd46..c7b21f7548 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/glob.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/glob.rs @@ -1,16 +1,22 @@ //! `fs.glob`. -use crate::methods::args::{optional_path, paging, snapshot, text, Answer}; +use crate::methods::args::{optional_path, paging, snapshot_full, text, Answer}; use crate::{ - read::{paged_reads, query, reads}, + read::{paged_reads, query, reads, snapshot, visibility}, StoreService, }; use layerfs_bridge::control::SdkValue; /// `glob(pattern, { branch, commit?, path?, pageSize?, pagePos? })`: paths matching a `/.lww`-syntax pattern. pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; + let (stack, reader, snap) = snapshot_full(service, args)?; let pattern = text(args, "pattern")?; let path = optional_path(args)?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); + if snapshot::entry(&reader, &snap.tree, &path)?.is_some() { + if let Some(refusal) = visibility::refuse(&path, &seen) { + return Err(refusal); + } + } if args.get("page").is_some() { return paged_reads::pin( &service.store, @@ -22,13 +28,15 @@ pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { pattern, &path, query::result_page(args)?, - ), + ) + .map(|value| visibility::filter(value, &seen)), ); } let (limit, pos) = paging(args)?; paged_reads::pin( &service.store, &snap, - reads::glob(&reader, &snap, pattern, &path, limit, pos), + reads::glob(&reader, &snap, pattern, &path, limit, pos) + .map(|value| visibility::filter(value, &seen)), ) } diff --git a/engine/crates/layerfs-store-service/src/methods/file/list.rs b/engine/crates/layerfs-store-service/src/methods/file/list.rs index f3cedc8e50..e983ab9080 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/list.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/list.rs @@ -1,15 +1,21 @@ //! `fs.list`. -use crate::methods::args::{optional_path, paging, snapshot, Answer}; +use crate::methods::args::{optional_path, paging, snapshot_full, Answer}; use crate::{ - read::{paged_reads, query, reads}, + read::{paged_reads, query, reads, snapshot, visibility}, StoreService, }; use layerfs_bridge::control::SdkValue; /// `list(path?, { branch, commit?, pageSize?, pagePos? })`: one window of a directory. pub fn list(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; + let (stack, reader, snap) = snapshot_full(service, args)?; let path = optional_path(args)?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); + if snapshot::entry(&reader, &snap.tree, &path)?.is_some() { + if let Some(refusal) = visibility::refuse(&path, &seen) { + return Err(refusal); + } + } let answer = if args.get("page").is_some() { paged_reads::list( &service.store, @@ -20,6 +26,7 @@ pub fn list(service: &StoreService, args: &SdkValue) -> Answer { ) } else { reads::list(&reader, &snap, &path, paging(args)) - }; + } + .map(|value| visibility::filter(value, &seen)); paged_reads::pin(&service.store, &snap, answer) } diff --git a/engine/crates/layerfs-store-service/src/methods/file/read.rs b/engine/crates/layerfs-store-service/src/methods/file/read.rs index eb7f29dcd2..41532e9989 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/read.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/read.rs @@ -1,15 +1,21 @@ //! `fs.read`. -use crate::methods::args::{line, path, snapshot, Answer}; +use crate::methods::args::{line, path, snapshot_full, Answer}; use crate::{ - read::{paged_reads, reads}, + read::{paged_reads, reads, snapshot, visibility}, StoreService, }; use layerfs_bridge::control::SdkValue; /// `read(path, { branch, commit?, startLine?, endLine? })`: one window of a file at a Branch, Commit or Layer. pub fn read(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; + let (stack, reader, snap) = snapshot_full(service, args)?; let path = path(args, "path")?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); + if snapshot::entry(&reader, &snap.tree, &path)?.is_some() { + if let Some(refusal) = visibility::refuse(&path, &seen) { + return Err(refusal); + } + } let answer = reads::read( &reader, &snap, diff --git a/engine/crates/layerfs-store-service/src/methods/file/stat.rs b/engine/crates/layerfs-store-service/src/methods/file/stat.rs index be76fc4e46..6fb48d0332 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/stat.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/stat.rs @@ -1,17 +1,20 @@ //! `fs.stat`. -use crate::methods::args::{path, snapshot, Answer}; +use crate::methods::args::{path, snapshot_full, Answer}; use crate::{ - read::{paged_reads, reads}, + read::{paged_reads, reads, snapshot, visibility}, StoreService, }; use layerfs_bridge::control::SdkValue; /// `stat(path, { branch, commit? })`: type, inode, size and mode of one path. pub fn stat(service: &StoreService, args: &SdkValue) -> Answer { - let (reader, snap) = snapshot(service, args)?; - paged_reads::pin( - &service.store, - &snap, - reads::stat(&reader, &snap, &path(args, "path")?), - ) + let (stack, reader, snap) = snapshot_full(service, args)?; + let path = path(args, "path")?; + let seen = visibility::Visibility::load(&service.store, &stack, &snap); + if snapshot::entry(&reader, &snap.tree, &path)?.is_some() { + if let Some(refusal) = visibility::refuse(&path, &seen) { + return Err(refusal); + } + } + paged_reads::pin(&service.store, &snap, reads::stat(&reader, &snap, &path)) } diff --git a/engine/crates/layerfs-store-service/src/methods/project.rs b/engine/crates/layerfs-store-service/src/methods/project.rs index 98af26cd67..b42521d620 100644 --- a/engine/crates/layerfs-store-service/src/methods/project.rs +++ b/engine/crates/layerfs-store-service/src/methods/project.rs @@ -1,7 +1,11 @@ -//! `project.list_branches`. -use crate::methods::args::{invalid, paging, project, Answer}; -use crate::{history::branch, StoreService}; +//! `project.list_branches`, `project.stats` and `project.remove_file`. +use crate::methods::args::{invalid, paging, project, text, Answer}; +use crate::{ + history::{branch, lookup}, + StoreService, +}; use layerfs_bridge::control::SdkValue; +use layerfs_history::{PathRemovalKind, PathRemovalRequest}; /// `list_branches({ name?, pageSize?, pagePos? })`: every Branch with its id, base and head, and main, in name order. pub fn list_branches(service: &StoreService, args: &SdkValue) -> Answer { @@ -78,3 +82,45 @@ pub fn stats(service: &StoreService, args: &SdkValue) -> Answer { crate::methods::args::text(args, "project")?, ) } + +/// `project.remove_file({ project, path })`: hide one path from every node at +/// or below the frontier. Visibility only; no byte is freed in 2.0. +pub fn remove_file(service: &StoreService, args: &SdkValue) -> Answer { + let stack = project(service, args)?; + let requested = text(args, "path")?; + let normalized = layerfs_history::path::NormalizedPath::new(requested) + .map_err(|why| invalid(format!("path: {why}")))?; + let removed_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }); + let outcome = service + .store + .history() + .remove_path(&PathRemovalRequest { + stack: stack.id, + path: normalized.as_bytes().to_vec(), + removed_ms, + }) + .map_err(lookup::history)?; + Ok(SdkValue::map([ + ( + "kind", + SdkValue::text(match outcome.kind { + PathRemovalKind::Removed => "Removed", + PathRemovalKind::UpToDate => "UpToDate", + }), + ), + ("project", SdkValue::text(text(args, "project")?)), + ( + "path", + SdkValue::text(String::from_utf8_lossy(&outcome.path).as_ref()), + ), + ( + "frontier_serial", + SdkValue::Int(outcome.frontier_serial as i64), + ), + ("removed_ms", SdkValue::Int(outcome.removed_ms)), + ])) +} diff --git a/engine/crates/layerfs-store-service/src/read/mod.rs b/engine/crates/layerfs-store-service/src/read/mod.rs index 221476651a..3885404fe2 100644 --- a/engine/crates/layerfs-store-service/src/read/mod.rs +++ b/engine/crates/layerfs-store-service/src/read/mod.rs @@ -22,5 +22,6 @@ pub(crate) mod revision_pages; pub mod search; pub mod snapshot; pub mod tree_stats; +pub mod visibility; pub mod walk; pub mod window; diff --git a/engine/crates/layerfs-store-service/src/read/snapshot.rs b/engine/crates/layerfs-store-service/src/read/snapshot.rs index 53e4a0e2e0..5c7a3ece9d 100644 --- a/engine/crates/layerfs-store-service/src/read/snapshot.rs +++ b/engine/crates/layerfs-store-service/src/read/snapshot.rs @@ -35,6 +35,8 @@ pub struct Snap { pub node: NodeId, pub root: ObjectId, pub tree: TreeTable, + /// The node's `history_serial`, for the `remove_file` visibility rule. + pub serial: u64, } /// The stack's object reader. @@ -89,12 +91,17 @@ pub fn resolve( } let root = args::root(store, node)?; let tree = TreeTable::open(reader, FilesystemRootId(root)).map_err(args::content)?; + let serial = match store.published(|history| history.node_history_serial(node)) { + Ok(Some(serial)) => serial, + _ => 0, + }; Ok(Snap { kind, label, node, root, tree, + serial, }) } diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs new file mode 100644 index 0000000000..756d417b2f --- /dev/null +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -0,0 +1,98 @@ +//! The `project.remove_file` visibility filter over one resolved snapshot. +//! +//! A path is hidden at node `N` when its entry exists at `N` and the marker +//! frontier, the maximum over the path and its `/`-boundary ancestors, is at +//! least `N`'s `history_serial` (`§4.3`). A marked-but-absent path hides +//! nothing, so callers check existence first. Reads and stats refuse with one +//! `Removed` detail; listings and globs omit hidden entries and report one +//! top-level `removed_count`. + +use crate::read::snapshot::Snap; +use crate::store::Store; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::error::{RemovedPath, RemovedState}; +use layerfs_history::{HistoryError, LayerStackRecord, PathRemovalRecord}; +use std::sync::Arc; + +/// The markers and node position one snapshot's reads filter against. +pub struct Visibility { + serial: u64, + markers: Vec, +} + +impl Visibility { + /// Loads the snapshot's serial and its project's markers. A provider that + /// does not implement the marker reads (a test double) sees none. + pub fn load(store: &Arc, stack: &LayerStackRecord, snap: &Snap) -> Self { + let serial = snap.serial; + let markers = match store.published(|history| history.path_removals(stack.id)) { + Ok(markers) => markers, + Err(_) => Vec::new(), + }; + Self { serial, markers } + } + + /// The governing frontier for a path: the maximum marker frontier over the + /// path and its `/`-boundary ancestors, if any marker applies. + fn frontier(&self, path: &str) -> Option { + let relative = path.strip_prefix('/').unwrap_or(path); + if relative.is_empty() { + return None; + } + self.markers + .iter() + .filter(|marker| layerfs_history::path::governs(&marker.path, relative.as_bytes())) + .map(|marker| marker.frontier_serial) + .max() + } + + /// The governing frontier when the entry at `path` is hidden at this node. + pub fn hidden(&self, path: &str) -> Option { + let frontier = self.frontier(path)?; + (frontier >= self.serial).then_some(frontier) + } + + /// The refusal for one hidden read or stat, `{paths: [{path, frontier_serial}]}`. + pub fn refusal(path: &str, frontier_serial: u64) -> SdkRefusal { + crate::history::lookup::history(HistoryError::Removed(Box::new(RemovedState::new(vec![ + RemovedPath::new(path, frontier_serial), + ])))) + } +} + +/// The `Removed` refusal a hidden read or stat reports. +pub fn refuse(path: &str, visibility: &Visibility) -> Option { + visibility + .hidden(path) + .map(|frontier| Visibility::refusal(path, frontier)) +} + +/// Removes hidden entries from a list or glob answer and adds `removed_count`. +/// Entries themselves exist, so only marked-and-present paths are counted. +pub fn filter(mut value: SdkValue, visibility: &Visibility) -> SdkValue { + let mut removed = 0i64; + if let SdkValue::Map(fields) = &mut value { + if let Some((_, SdkValue::List(items))) = fields.iter_mut().find(|(k, _)| k == "items") { + items.retain(|item| { + let path = match item { + SdkValue::Text(path) => path.clone(), + other => other + .get_text("path") + .map(str::to_owned) + .unwrap_or_default(), + }; + if path.is_empty() { + return true; + } + if visibility.hidden(&path).is_some() { + removed += 1; + false + } else { + true + } + }); + } + fields.push(("removed_count".into(), SdkValue::Int(removed))); + } + value +} diff --git a/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt b/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt index d5f472bdac..0fb8409998 100644 --- a/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt +++ b/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt @@ -34,16 +34,16 @@ fs.read -> refused NoSuchPath "/nope does not exist at feat@" {path: "/nope fs.read -> refused NoSuchBranch "Branch ghost does not exist" null fs.read -> refused InvalidInput "missing text argument `path`" null fs.read -> refused Binary "text contains NUL or invalid UTF-8" null -fs.list -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "applied-dir", path: "/applied-dir", type: "dir", inode: 201}, {name: "applied.txt", path: "/applied.txt", type: "file", inode: 200}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "made", path: "/made", type: "dir", inode: 72}, {name: "moved.txt", path: "/moved.txt", type: "file", inode: 137}, {name: "new.txt", path: "/new.txt", type: "file", inode: 8}], hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} -fs.list -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/dir", items: [{name: "b.txt", path: "/dir/b.txt", type: "file", inode: 5}], hasMore: true, nextPagePos: 1, at: {id: "", kind: "Commit", branch: "feat", root: ""}} +fs.list -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "applied-dir", path: "/applied-dir", type: "dir", inode: 201}, {name: "applied.txt", path: "/applied.txt", type: "file", inode: 200}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "made", path: "/made", type: "dir", inode: 72}, {name: "moved.txt", path: "/moved.txt", type: "file", inode: 137}, {name: "new.txt", path: "/new.txt", type: "file", inode: 8}], hasMore: false, nextPagePos: null, removed_count: 0, at: {id: "", kind: "Commit", branch: "feat", root: ""}} +fs.list -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/dir", items: [{name: "b.txt", path: "/dir/b.txt", type: "file", inode: 5}], hasMore: true, nextPagePos: 1, removed_count: 0, at: {id: "", kind: "Commit", branch: "feat", root: ""}} fs.list -> refused NotDirectory "expected a directory" null fs.stat -> ok {kind: "ok", snapshot: "main@", root: "", path: "/a.txt", type: "file", inode: 2, size: 14, mode: 420, at: {id: "", kind: "Layer", branch: "main", root: ""}} fs.stat -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/dir", type: "dir", inode: 4, size: 0, mode: 493, at: {id: "", kind: "Commit", branch: "feat", root: ""}} fs.grep -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: [], skipped: 1, hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} fs.grep -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/dir", items: [{path: "/dir/b.txt", line: 1, text: "bee\n", lineTruncated: false}], skipped: 0, hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} fs.grep -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: [{path: "/dir/sub/c.rs", line: 1, text: "fn main() {}\n", lineTruncated: false}], skipped: 1, hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} -fs.glob -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: ["/a.txt", "/applied.txt", "/dir/b.txt", "/moved.txt", "/new.txt"], hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} -fs.glob -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: ["/dir/b.txt", "/dir/sub", "/dir/sub/c.rs"], hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "feat", root: ""}} +fs.glob -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: ["/a.txt", "/applied.txt", "/dir/b.txt", "/moved.txt", "/new.txt"], hasMore: false, nextPagePos: null, removed_count: 0, at: {id: "", kind: "Commit", branch: "feat", root: ""}} +fs.glob -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/", items: ["/dir/b.txt", "/dir/sub", "/dir/sub/c.rs"], hasMore: false, nextPagePos: null, removed_count: 0, at: {id: "", kind: "Commit", branch: "feat", root: ""}} fs.blame -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/a.txt", at: {id: "", kind: "Commit", branch: "feat", root: ""}, coverage: {status: "complete", comparison: "exact", reasons: []}, lines: [{n: 1, owner: {ref: {id: "", kind: "Layer", branch: "main", root: ""}, message: null}, text: "one", lineTruncated: false, clipped: false, commit: "", message: null}, {n: 2, owner: {ref: {id: "", kind: "Commit", branch: "feat", root: ""}, message: "Apply filesystem changes"}, text: "TWO", lineTruncated: false, clipped: false, commit: "", message: "Apply filesystem changes"}, {n: 3, owner: {ref: {id: "", kind: "Layer", branch: "main", root: ""}, message: null}, text: "three", lineTruncated: false, clipped: false, commit: "", message: null}], origins: [{ref: {id: "", kind: "Commit", branch: "feat", root: ""}, message: "Apply filesystem changes"}, {ref: {id: "", kind: "Layer", branch: "main", root: ""}, message: null}], versions: 8, comparisonCoarse: false, nextLine: null} fs.blame -> ok {kind: "ok", snapshot: "feat@", root: "", path: "/a.txt", at: {id: "", kind: "Commit", branch: "feat", root: ""}, coverage: {status: "complete", comparison: "exact", reasons: []}, lines: [{n: 2, owner: {ref: {id: "", kind: "Commit", branch: "feat", root: ""}, message: "Apply filesystem changes"}, text: "TWO", lineTruncated: false, clipped: false, commit: "", message: "Apply filesystem changes"}, {n: 3, owner: {ref: {id: "", kind: "Layer", branch: "main", root: ""}, message: null}, text: "three", lineTruncated: false, clipped: false, commit: "", message: null}], origins: [{ref: {id: "", kind: "Commit", branch: "feat", root: ""}, message: "Apply filesystem changes"}, {ref: {id: "", kind: "Layer", branch: "main", root: ""}, message: null}], versions: 8, comparisonCoarse: false, nextLine: null} fs.diff_files -> ok {kind: "Text", left: {path: "/a.txt", snapshot: "main@", inode: 2}, right: {path: "/a.txt", snapshot: "feat@", inode: 2}, contentEqual: false, sameInode: true, hunksOmitted: false, hunks: [{op: " ", text: "one"}, {op: "-", text: "two"}, {op: "+", text: "TWO"}, {op: " ", text: "three"}, {op: " ", text: ""}], leftRef: {id: "", kind: "Layer", branch: "main", root: ""}, rightRef: {id: "", kind: "Commit", branch: "feat", root: ""}, before: {type: "file", inode: 2, size: 14, mode: 420}, after: {type: "file", inode: 2, size: 14, mode: 420}, changeKind: "Modified", changes: {content: true, metadata: true, identity: false, attribution: null}, contentCoverage: {status: "complete", comparison: "exact", reasons: []}, attributionCoverage: {status: "not_requested", comparison: null, reasons: []}, omissionReason: null, rows: [], page: 1, pageSize: 50, hasMore: false} @@ -67,7 +67,7 @@ fs.write -> ok {kind: "Committed", branch: "left", head: "", commit: {id: " branch.fork -> ok {kind: "Forked", branch: "right", base: "", head: null} fs.write -> ok {kind: "Committed", branch: "right", head: "", commit: {id: "", branch: "right", parent: null, secondParent: null, base: "", root: "", via: "fs", message: "Apply filesystem changes", mergedFrom: null, time: , logical: 42, newCanonical: 958, newStored: , newMetadata: , displaced: null, lww: [], lwwTruncated: false}} branch.merge -> ok {kind: "Merged", fastForward: false, branch: "both", head: "", root: "", sources: [{ref: "left", node: ""}, {ref: "right", node: ""}], mergeBase: "", conflicts: [], lww: [], conflictsTruncated: false, lwwTruncated: false, units: [], unitsTruncated: false} -fs.list -> ok {kind: "ok", snapshot: "both@", root: "", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "left.txt", path: "/left.txt", type: "file", inode: 328}, {name: "right.txt", path: "/right.txt", type: "file", inode: 392}], hasMore: false, nextPagePos: null, at: {id: "", kind: "Commit", branch: "both", root: ""}} +fs.list -> ok {kind: "ok", snapshot: "both@", root: "", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "left.txt", path: "/left.txt", type: "file", inode: 328}, {name: "right.txt", path: "/right.txt", type: "file", inode: 392}], hasMore: false, nextPagePos: null, removed_count: 0, at: {id: "", kind: "Commit", branch: "both", root: ""}} branch.merge -> refused NameTaken "Branch both already exists" null branch.merge -> refused InvalidInput "merge takes two different sources" null branch.merge -> refused ReservedName "\"main\" names the project's main line" null diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_removal.rs b/engine/crates/layerfs-store-service/tests/sdk_path_removal.rs new file mode 100644 index 0000000000..de94729385 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_removal.rs @@ -0,0 +1,135 @@ +//! `project.remove_file` over the public Store method table: one marker hides +//! a read and a stat with a `Removed` detail, a listing and a glob omit the +//! entry and count it, the subtree matches only at `/`, and a post-frontier +//! re-creation is visible again. The path grammar is refused as `InvalidInput`. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::SdkValue; +use std::fs; +use support::{paths, write, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text("work")), ("path", text(path))], + ) +} + +#[test] +fn a_removed_path_is_hidden_in_reads_stat_list_and_glob_until_recreated() { + let fixture = Fixture::new("sdk-path-removal", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("ab"), "ab\n").unwrap(); + 2 + }); + fixture.fork("work"); + fixture.apply("work", vec![write("/x", "x\n")]); + + let removed = remove(&fixture, "/a"); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!(removed.get_text("project"), Some("installed")); + assert_eq!( + removed.get_text("path"), + Some("a"), + "the stored normalized path" + ); + let frontier = removed.get_int("frontier_serial").unwrap(); + assert!(frontier >= 2, "the Store position covers the commits"); + let removed_ms = removed.get_int("removed_ms").unwrap(); + + // Reads and stats refuse with the one detail shape. + let refusal = read(&fixture, "/a").expect_err("hidden read"); + assert_eq!(refusal.code, "Removed"); + let Some(SdkValue::List(hidden)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + assert_eq!(hidden.len(), 1); + assert_eq!(hidden[0].get_text("path"), Some("/a")); + assert_eq!(hidden[0].get_int("frontier_serial"), Some(frontier)); + let stat = fixture + .try_call( + "fs.stat", + vec![("branch", text("work")), ("path", text("/a"))], + ) + .expect_err("hidden stat"); + assert_eq!(stat.code, "Removed"); + + // The subtree matches only at a `/` boundary. + let sibling = read(&fixture, "/ab").expect("ab stays visible"); + assert_eq!(sibling.get_text("text"), Some("ab\n")); + + // A listing omits the entry and counts it; a glob does the same. + let listed = fixture.call( + "fs.list", + vec![("branch", text("work")), ("path", text("/"))], + ); + let names = paths(&listed); + assert!(names.contains(&"/ab"), "{names:?}"); + assert!(names.contains(&"/x"), "{names:?}"); + assert!(!names.contains(&"/a"), "{names:?}"); + assert_eq!(listed.get_int("removed_count"), Some(1)); + let globbed = fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text("/*")), + ("path", text("/")), + ], + ); + let matched: Vec<&str> = match globbed.get("items") { + Some(SdkValue::List(items)) => items + .iter() + .filter_map(|item| match item { + SdkValue::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect(), + other => panic!("no glob items: {other:?}"), + }; + assert!(matched.contains(&"/ab"), "{matched:?}"); + assert!(!matched.contains(&"/a"), "{matched:?}"); + assert_eq!(globbed.get_int("removed_count"), Some(1)); + + // No newer project node: the repeat is UpToDate and keeps the stored values. + let repeat = remove(&fixture, "/a"); + assert_eq!(repeat.get_text("kind"), Some("UpToDate")); + assert_eq!(repeat.get_int("frontier_serial"), Some(frontier)); + assert_eq!(repeat.get_int("removed_ms"), Some(removed_ms)); + + // A post-frontier re-creation is an ordinary visible write. + fixture.apply("work", vec![write("/a", "again\n")]); + let again = read(&fixture, "/a").expect("the re-creation is visible"); + assert_eq!(again.get_text("text"), Some("again\n")); + fixture.cleanup(); +} + +#[test] +fn the_remove_file_path_grammar_is_refused() { + let fixture = Fixture::new("sdk-path-grammar", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + 1 + }); + fixture.fork("work"); + for bad in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "a/b/", "a/../b", + ] { + let refusal = fixture + .try_call("project.remove_file", vec![("path", text(bad))]) + .unwrap_err(); + assert_eq!(refusal.code, "InvalidInput", "{bad:?} was accepted"); + } + fixture.cleanup(); +}