From 40e6305682cc92cc622cbc2c95624245b3cbee7c Mon Sep 17 00:00:00 2001 From: rain Date: Sun, 15 Nov 2026 23:54:22 +0000 Subject: [PATCH] feat(gc): SQLite and PostgreSQL compaction (#48-#49) --- .../layerfs-sdk-observe/references/tracing.md | 10 +- api/crates/layerfs-api/src/branch/mod.rs | 4 +- api/crates/layerfs-api/src/branch/remove.rs | 28 + api/crates/layerfs-api/src/calls/mod.rs | 1 + api/crates/layerfs-api/src/calls/render.rs | 97 +- api/crates/layerfs-api/src/calls/scope.rs | 5 + api/crates/layerfs-api/src/lib.rs | 2 +- api/crates/layerfs-api/src/project/delete.rs | 24 + api/crates/layerfs-api/src/project/mod.rs | 6 +- .../layerfs-api/src/project/remove_file.rs | 29 + api/crates/layerfs-api/src/store/compact.rs | 24 + api/crates/layerfs-api/src/store/mod.rs | 4 +- api/crates/layerfs-api/tests/calls_scope.rs | 10 + api/crates/layerfs-api/tests/calls_table.rs | 35 +- api/crates/layerfs-api/tests/gc_calls.rs | 324 +++ api/crates/layerfs-api/tests/gc_trace.rs | 309 +++ api/crates/layerfs-api/tests/logs.rs | 44 +- api/crates/layerfs-cli/tests/gc.rs | 169 ++ control/crates/layerfs-control/src/store.rs | 6 + .../layerfs-control/src/worker_server.rs | 19 +- .../tests/project_delete_worker.rs | 550 +++++ .../crates/layerfs-store/src/calls/branch.rs | 26 + control/crates/layerfs-store/src/calls/op.rs | 10 + .../crates/layerfs-store/src/calls/project.rs | 22 + control/crates/layerfs-store/src/compact.rs | 150 ++ control/crates/layerfs-store/src/lib.rs | 2 + .../layerfs-store/src/project/delete.rs | 420 +++- .../crates/layerfs-store/src/project/frame.rs | 112 +- control/crates/layerfs-store/src/report.rs | 116 + .../layerfs-store/src/sqlite/transfer/step.rs | 7 + .../src/sqlite/transfer/steps.rs | 16 + control/crates/layerfs-store/src/status.rs | 55 +- .../tests/compact_report_status.rs | 212 ++ .../layerfs-store/tests/project_delete.rs | 157 +- .../tests/project_delete_frames.rs | 75 +- .../tests/project_delete_store.rs | 238 ++ .../layerfs-history/src/contract/error.rs | 3 + .../docs/issues/project-41/LayerFS-GC-spec.md | 37 +- .../issues/project-41/gc-t10-report-status.md | 96 + .../gc-t12-sqlite-compaction-fixtures.md | 88 + .../issues/project-41/gc-t8-sqlite-rebuild.md | 138 ++ .../project-41/gc-t9-postgresql-rebuild.md | 115 + docs/architecture/api/calls/README.md | 10 +- docs/architecture/control/home/README.md | 8 +- .../queries/history/gc_epoch_read.sql | 1 + .../sqlite/queries/history/gc_epoch_read.sql | 1 + .../sqlite/queries/history/gc_tuple_read.sql | 6 + .../src/backend/postgresql/compact.rs | 1413 ++++++++++++ .../backend/postgresql/history/statements.rs | 3 + .../src/backend/postgresql/mod.rs | 1 + .../src/backend/records.rs | 4 + .../src/backend/sqlite/compact.rs | 1967 +++++++++++++++++ .../src/backend/sqlite/lane/lane.rs | 11 + .../src/backend/sqlite/lifecycle/identity.rs | 51 + .../src/backend/sqlite/lifecycle/mod.rs | 1 + .../src/backend/sqlite/lifecycle/seal.rs | 135 +- .../src/backend/sqlite/mod.rs | 1 + .../src/backend/sqlite/session/checkpoint.rs | 34 + .../src/backend/sqlite/session/connection.rs | 22 + .../src/backend/sqlite/session/query.rs | 6 + .../src/backend/sqlite/session/transaction.rs | 3 + .../layerfs-persistence/src/history/commit.rs | 4 + .../layerfs-persistence/src/history/epoch.rs | 69 + .../src/history/layerstack.rs | 4 + .../layerfs-persistence/src/history/merge.rs | 9 +- .../layerfs-persistence/src/history/mod.rs | 3 +- .../src/history/objects.rs | 71 + engine/crates/layerfs-persistence/src/lib.rs | 8 + .../crates/layerfs-persistence/src/report.rs | 124 ++ .../layerfs-persistence/tests/epoch_fence.rs | 442 ++++ .../tests/fixtures/sqlite_compact_golden.json | 407 ++++ .../tests/postgresql_compact.rs | 582 +++++ .../tests/sqlite_compact.rs | 865 ++++++++ .../tests/sqlite_compact_crash.rs | 577 +++++ .../tests/sqlite_compact_fixtures.rs | 1597 +++++++++++++ .../src/history/admission.rs | 10 +- .../src/history/deletion.rs | 19 +- .../src/history/lookup.rs | 1 + .../src/methods/branch/mod.rs | 2 + .../src/methods/branch/remove.rs | 10 + .../src/methods/dispatch.rs | 1 + .../src/methods/file/glob.rs | 2 + .../src/read/visibility.rs | 17 +- .../layerfs-store-service/src/write/files.rs | 13 + .../tests/epoch_fence.rs | 77 + .../tests/sdk_path_hidden_semantics.rs | 290 +++ 86 files changed, 12499 insertions(+), 178 deletions(-) create mode 100644 api/crates/layerfs-api/src/branch/remove.rs create mode 100644 api/crates/layerfs-api/src/project/delete.rs create mode 100644 api/crates/layerfs-api/src/project/remove_file.rs create mode 100644 api/crates/layerfs-api/src/store/compact.rs create mode 100644 api/crates/layerfs-api/tests/gc_calls.rs create mode 100644 api/crates/layerfs-api/tests/gc_trace.rs create mode 100644 api/crates/layerfs-cli/tests/gc.rs create mode 100644 control/crates/layerfs-control/tests/project_delete_worker.rs create mode 100644 control/crates/layerfs-store/src/compact.rs create mode 100644 control/crates/layerfs-store/src/report.rs create mode 100644 control/crates/layerfs-store/tests/compact_report_status.rs create mode 100644 control/crates/layerfs-store/tests/project_delete_store.rs create mode 100644 core/docs/issues/project-41/gc-t10-report-status.md create mode 100644 core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md create mode 100644 core/docs/issues/project-41/gc-t8-sqlite-rebuild.md create mode 100644 core/docs/issues/project-41/gc-t9-postgresql-rebuild.md create mode 100644 engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql create mode 100644 engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs create mode 100644 engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs create mode 100644 engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs create mode 100644 engine/crates/layerfs-persistence/src/history/epoch.rs create mode 100644 engine/crates/layerfs-persistence/src/report.rs create mode 100644 engine/crates/layerfs-persistence/tests/epoch_fence.rs create mode 100644 engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_compact.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs create mode 100644 engine/crates/layerfs-store-service/src/methods/branch/remove.rs create mode 100644 engine/crates/layerfs-store-service/tests/epoch_fence.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs diff --git a/.agents/skills/layerfs-sdk-observe/references/tracing.md b/.agents/skills/layerfs-sdk-observe/references/tracing.md index 316686abfa..5f3534511c 100644 --- a/.agents/skills/layerfs-sdk-observe/references/tracing.md +++ b/.agents/skills/layerfs-sdk-observe/references/tracing.md @@ -19,12 +19,16 @@ Python, direct `layerfs_api::call`) appends one line to and `LAYERFS_TRACE` / `LAYERFS_TRACE_STEP` do not affect it. ```json -{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","/index.ts","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} +{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","<8 bytes>","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} ``` It keeps 1.1.0's redaction rule (values cut to 200 characters; contents, -JSON, commands, messages, edit strings, passwords and URLs as byte counts) -and drops the daemon telemetry: no `sizes`, `scope` or `daemon`. `argv` is +JSON, commands, messages, edit strings, passwords and URLs as byte counts; +a path argument as the byte count of its normalized form, one leading `/` +stripped, so `/a` and `a` are both `<1 bytes>`; a glob or grep `pattern` by +its own bytes) and drops the daemon telemetry: no `sizes`, `scope` or +`daemon`. A refusal is logged as its code only: its message and `detail` are +never traced, so a `Removed` path stays out of the line. `argv` is the call as CLI words (`argv[0]`/`argv[1]` are the family and call), `outcome` is `"ok"` or the refusal code, `kind` and `commit`/`layer` come from the result, `client` is `cli`/`node`/`python`/`rust`, and `tag` is the diff --git a/api/crates/layerfs-api/src/branch/mod.rs b/api/crates/layerfs-api/src/branch/mod.rs index d4e4632dd6..bc4b6b5b7c 100644 --- a/api/crates/layerfs-api/src/branch/mod.rs +++ b/api/crates/layerfs-api/src/branch/mod.rs @@ -1,4 +1,4 @@ -//! The `branch` family: 8 calls, one file each. +//! The `branch` family: 9 calls, one file each. mod commits; mod diff; mod fork; @@ -7,6 +7,7 @@ mod get_commit; mod list; mod merge; mod publish; +mod remove; use crate::calls::Calls; @@ -21,5 +22,6 @@ pub(crate) fn calls() -> Calls { merge::CALL, publish::CALL, diff::CALL, + remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/branch/remove.rs b/api/crates/layerfs-api/src/branch/remove.rs new file mode 100644 index 0000000000..e9279cbd84 --- /dev/null +++ b/api/crates/layerfs-api/src/branch/remove.rs @@ -0,0 +1,28 @@ +//! `branch.remove`: soft-remove one Branch through its Store marker. +use crate::calls::table::Kind::{Name, Project}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::branch::{remove, BranchRemove}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "branch", + "remove", + &[a("project", Project), a("branch", Name)], + &[], + "soft-remove one Branch; its name stays taken until compaction", + run, +); + +/// Runs `branch.remove`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = BranchRemove { + project: text("project").unwrap_or_default(), + branch: text("branch").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/calls/mod.rs b/api/crates/layerfs-api/src/calls/mod.rs index 45c32f6583..ef74ed4009 100644 --- a/api/crates/layerfs-api/src/calls/mod.rs +++ b/api/crates/layerfs-api/src/calls/mod.rs @@ -18,6 +18,7 @@ pub use check::size; pub use context::Context; pub use dispatch::{call, call_scoped, home, Client}; pub use layerfs_home::json; +pub use render::REDACTED; pub use scope::Scope; pub use table::{find, table_json, Arg, Call, CallRef, Calls, Families, Family, Kind, Run}; pub use text::call_json; diff --git a/api/crates/layerfs-api/src/calls/render.rs b/api/crates/layerfs-api/src/calls/render.rs index 87d904246a..c82fc1054d 100644 --- a/api/crates/layerfs-api/src/calls/render.rs +++ b/api/crates/layerfs-api/src/calls/render.rs @@ -9,9 +9,46 @@ use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::outcome::Outcome; /// The `Text` arguments logged only as ``: exec command text, -/// Commit messages, edit strings, Store passwords and connection URLs that -/// may embed `user:pass@`. A new call with such an argument adds its name. -pub const REDACTED: [&str; 6] = ["command", "message", "old", "new", "password", "url"]; +/// Commit messages, edit strings, Store passwords, connection URLs that may +/// embed `user:pass@`, every path argument (`path`, `left_path`, +/// `right_path`, `from_path`, `to_path`, `path_prefix`) and a glob or grep +/// `pattern`. A path is counted in its normalized form ([`PATHS`]), so a +/// `project.remove_file` or `file.*` path never enters the call log (#41 +/// §4.3). A new call with such an argument adds its name. +pub const REDACTED: [&str; 13] = [ + "command", + "message", + "old", + "new", + "password", + "url", + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", +]; + +/// The arguments that name a path: they log the length of the normalized +/// path, one leading `/` stripped, so `/a` and `a` count the same and a root +/// `/` counts as 0 (#41 §4.3). A `pattern` is not a path and keeps every +/// byte it has. +const PATHS: [&str; 6] = [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", +]; + +/// A path argument's logged byte length: the §4.3 normalization (strip one +/// leading `/`) applied before counting. +fn path_len(text: &str) -> usize { + text.strip_prefix('/').unwrap_or(text).len() +} /// The most characters a logged value keeps. const CUT: usize = 200; @@ -46,6 +83,7 @@ fn word(name: &str, kind: Kind, value: &SdkValue) -> Option { (Kind::Content, SdkValue::Text(text)) => bytes(text.len()), (Kind::Content, SdkValue::Bytes(raw)) => bytes(raw.len()), (Kind::Content | Kind::Json, value) => bytes(layerfs_home::json::print(value).len()), + (Kind::Text, SdkValue::Text(text)) if PATHS.contains(&name) => bytes(path_len(text)), (Kind::Text, SdkValue::Text(text)) if REDACTED.contains(&name) => bytes(text.len()), (_, SdkValue::Text(text)) => SdkValue::Text(cut(text)), (_, SdkValue::Int(n)) => SdkValue::Text(n.to_string()), @@ -187,3 +225,56 @@ pub fn line( } SdkValue::Map(fields) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_path_argument_name_is_redacted() { + for name in [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", + ] { + assert!(REDACTED.contains(&name), "{name} is not redacted"); + } + for name in ["project", "branch", "name", "commit", "content", "changes"] { + assert!(!REDACTED.contains(&name), "{name} is redacted"); + } + } + + #[test] + fn a_path_logs_the_byte_length_of_its_normalized_form() { + // `/a` and `a` are the same path (§4.3), so they log the same count; + // the root `/` normalizes to the empty relative path. + for (text, len) in [ + ("/base.txt", 8), + ("base.txt", 8), + ("/a", 1), + ("a", 1), + ("/", 0), + ] { + let value = SdkValue::text(text); + assert_eq!( + word("path", Kind::Text, &value), + Some(SdkValue::text(format!("<{len} bytes>"))), + "{text}" + ); + } + // A glob pattern is not a path and keeps every byte it has. + assert_eq!( + word("pattern", Kind::Text, &SdkValue::text("*.txt")), + Some(SdkValue::text("<5 bytes>")) + ); + // Text that is not a path is still its own (cut) text. + assert_eq!( + word("branch", Kind::Text, &SdkValue::text("/main")), + Some(SdkValue::text("/main")) + ); + } +} diff --git a/api/crates/layerfs-api/src/calls/scope.rs b/api/crates/layerfs-api/src/calls/scope.rs index c7c81b6b2c..bdb139bf60 100644 --- a/api/crates/layerfs-api/src/calls/scope.rs +++ b/api/crates/layerfs-api/src/calls/scope.rs @@ -151,6 +151,11 @@ impl Scope { | ("file", "revision") => { return Err(out("history")); } + // GC retires or compacts a whole entity, never one folder: + // a scope refuses these instead of widening them (§10). + ("branch", "remove") | ("project", "delete" | "remove_file") | ("store", "compact") => { + return Err(out(name)); + } ("file", _) => { bound(&args)?; let paths: &[&str] = match call { diff --git a/api/crates/layerfs-api/src/lib.rs b/api/crates/layerfs-api/src/lib.rs index 0d2b610fe9..0f67fadce2 100644 --- a/api/crates/layerfs-api/src/lib.rs +++ b/api/crates/layerfs-api/src/lib.rs @@ -71,7 +71,7 @@ pub use layerfs_sandbox::{ BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi, }; -/// The ten families, in the order help lists them: 73 calls, the same +/// The ten families, in the order help lists them: 77 calls, the same /// from the `layerfs` binary, Node and Python. pub fn families() -> Families { &[ diff --git a/api/crates/layerfs-api/src/project/delete.rs b/api/crates/layerfs-api/src/project/delete.rs new file mode 100644 index 0000000000..9afba64e3f --- /dev/null +++ b/api/crates/layerfs-api/src/project/delete.rs @@ -0,0 +1,24 @@ +//! `project.delete`: retire one project in its Store and remove its home entry. +use crate::calls::table::Kind::Project; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::names::ProjectRef; +use layerfs_home::outcome::Outcome; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "project", + "delete", + &[a("project", Project)], + &[], + "retire one project in its Store and remove its home entry", + run, +); + +/// Runs `project.delete`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let project = ProjectRef::parse(args.get_text("project").unwrap_or(""))?; + let (home, events) = context.parts(); + layerfs_control::project::delete(home, events, &project) +} diff --git a/api/crates/layerfs-api/src/project/mod.rs b/api/crates/layerfs-api/src/project/mod.rs index 03b79a3597..7a65fd924f 100644 --- a/api/crates/layerfs-api/src/project/mod.rs +++ b/api/crates/layerfs-api/src/project/mod.rs @@ -1,9 +1,11 @@ -//! The `project` family: 6 calls, one file each. +//! The `project` family: 8 calls, one file each. +mod delete; mod export; mod get; mod init; mod list; mod remove; +mod remove_file; mod stats; use crate::calls::Calls; @@ -16,6 +18,8 @@ pub(crate) fn calls() -> Calls { get::CALL, stats::CALL, export::CALL, + delete::CALL, + remove_file::CALL, remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/project/remove_file.rs b/api/crates/layerfs-api/src/project/remove_file.rs new file mode 100644 index 0000000000..3c8d4f2176 --- /dev/null +++ b/api/crates/layerfs-api/src/project/remove_file.rs @@ -0,0 +1,29 @@ +//! `project.remove_file`: hide one absolute path from every node at or below +//! its marker frontier. +use crate::calls::table::Kind::{Project, Text}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::project::{remove_file, ProjectRemoveFile}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "project", + "remove_file", + &[a("project", Project), a("path", Text)], + &[], + "hide one absolute path from every node at or below its marker frontier", + run, +); + +/// Runs `project.remove_file`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = ProjectRemoveFile { + project: text("project").unwrap_or_default(), + path: text("path").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove_file(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/store/compact.rs b/api/crates/layerfs-api/src/store/compact.rs new file mode 100644 index 0000000000..dc8a044d3b --- /dev/null +++ b/api/crates/layerfs-api/src/store/compact.rs @@ -0,0 +1,24 @@ +//! `store.compact`: reclaim dead bytes from one sqlite Store. +use crate::calls::table::Kind::Name; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::names::StoreName; +use layerfs_home::outcome::Outcome; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "store", + "compact", + &[a("name", Name)], + &[], + "reclaim dead bytes from one sqlite Store", + run, +); + +/// Runs `store.compact`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let name = StoreName::parse(args.get_text("name").unwrap_or(""))?; + let (home, events) = context.parts(); + layerfs_control::store::compact(home, events, &name) +} diff --git a/api/crates/layerfs-api/src/store/mod.rs b/api/crates/layerfs-api/src/store/mod.rs index 8c0df7b57c..2142e041ea 100644 --- a/api/crates/layerfs-api/src/store/mod.rs +++ b/api/crates/layerfs-api/src/store/mod.rs @@ -1,6 +1,7 @@ -//! The `store` family: 12 calls, one file each (H1: push and pull replace +//! The `store` family: 13 calls, one file each (H1: push and pull replace //! move; attach and detach supersede #536 C4). mod attach; +mod compact; mod connect; mod create; mod detach; @@ -29,6 +30,7 @@ pub(crate) fn calls() -> Calls { push::CALL, pull::CALL, remove::CALL, + compact::CALL, disconnect::CALL, ] } diff --git a/api/crates/layerfs-api/tests/calls_scope.rs b/api/crates/layerfs-api/tests/calls_scope.rs index 11d9b26286..5b324435e0 100644 --- a/api/crates/layerfs-api/tests/calls_scope.rs +++ b/api/crates/layerfs-api/tests/calls_scope.rs @@ -115,6 +115,16 @@ fn a_scope_refuses_every_call_it_does_not_cover() { ("store.remove", &[("name", t("shop"))]), ("store.list", &[]), ("project.remove", &[("project", t("shop/api"))]), + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("shop/api"))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", t("/web/a"))], + ), + ("store.compact", &[("name", t("shop"))]), ( "project.init", &[("project", t("shop/x")), ("folder", t("/tmp"))], diff --git a/api/crates/layerfs-api/tests/calls_table.rs b/api/crates/layerfs-api/tests/calls_table.rs index 39cdc1df3e..2ba77897cd 100644 --- a/api/crates/layerfs-api/tests/calls_table.rs +++ b/api/crates/layerfs-api/tests/calls_table.rs @@ -1,4 +1,4 @@ -//! The call table and its dispatcher (#533 step 2): 73 calls in ten +//! The call table and its dispatcher (#533 step 2): 77 calls in ten //! families, arguments checked before any effect, one test per family that //! a valid call reaches its family function and a wrong one is refused. use layerfs_api::{families, json, keys, size, Call, Kind}; @@ -51,9 +51,9 @@ fn code(name: &str, args: SdkValue) -> String { } #[test] -fn the_table_has_ten_families_and_73_calls() { +fn the_table_has_ten_families_and_77_calls() { let all: Vec<&Call> = layerfs_api::calls(); - assert_eq!(all.len(), 73); + assert_eq!(all.len(), 77); // each family holds only its own calls, and has at least one for family in families() { assert!(!family.calls().is_empty(), "{}", family.name); @@ -66,7 +66,7 @@ fn the_table_has_ten_families_and_73_calls() { let mut names: Vec<_> = all.iter().map(|c| c.name()).collect(); names.sort(); names.dedup(); - assert_eq!(names.len(), 73, "call names are unique"); + assert_eq!(names.len(), 77, "call names are unique"); // an optional positional is only ever followed by optional ones (D7) for c in &all { let first_optional = c.args.iter().position(|a| a.optional); @@ -75,7 +75,7 @@ fn the_table_has_ten_families_and_73_calls() { } } let counts: Vec = families().iter().map(|f| f.calls().len()).collect(); - assert_eq!(counts, [6, 8, 3, 15, 4, 6, 12, 12, 5, 2]); + assert_eq!(counts, [8, 9, 3, 15, 4, 6, 12, 13, 5, 2]); } #[test] @@ -444,3 +444,28 @@ fn sandbox_images_are_named_by_the_image_kind() { assert!(prune.args[1].optional); assert_eq!(prune.options[0].name, "unused"); } + +#[test] +fn the_four_gc_calls_take_only_ids() { + // T11.1: the four GC calls, each one resource identifier and no option. + use layerfs_api::Kind; + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name}"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name}"); + assert!(spec.options.is_empty(), "{name} takes no option"); + } +} diff --git a/api/crates/layerfs-api/tests/gc_calls.rs b/api/crates/layerfs-api/tests/gc_calls.rs new file mode 100644 index 0000000000..c7d37f5d24 --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_calls.rs @@ -0,0 +1,324 @@ +//! T11.1: the four GC calls in the API call table — their ids-only arguments, +//! typed refusals before any effect, and the result shapes of one reachable +//! refusal per call plus the three removal results and the §11 report. +use layerfs_api::Kind; +use layerfs_bridge::control::SdkValue; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-calls-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call( + home: &Path, + name: &str, + fields: &[(&str, SdkValue)], +) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) + .map_err(|refusal| (refusal.code, refusal.message)) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> (String, String) { + call(home, name, fields).expect_err("the call is refused") +} + +/// A home with Store `shop` and project `shop/api` imported from a folder. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +#[test] +fn the_four_gc_calls_take_ids_only() { + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name} arguments"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name} required"); + assert!(spec.options.is_empty(), "{name} has no options"); + } +} + +#[test] +fn wrong_kinds_and_unknown_options_are_refused_before_any_effect() { + let home = tempdir("types"); + layerfs_api::home::open(&home).unwrap(); + let wrong: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", SdkValue::Int(1))], + ), + ("project.delete", &[("project", SdkValue::Int(1))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", SdkValue::Int(1))], + ), + ("store.compact", &[("name", SdkValue::Int(1))]), + ]; + for &(name, fields) in wrong { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.starts_with(name), "{name}: {message}"); + } + let unknown: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[ + ("project", t("shop/api")), + ("branch", t("main")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "project.delete", + &[("project", t("shop/api")), ("force", SdkValue::Bool(true))], + ), + ( + "project.remove_file", + &[ + ("project", t("shop/api")), + ("path", t("/base.txt")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "store.compact", + &[("name", t("shop")), ("force", SdkValue::Bool(true))], + ), + ]; + for &(name, fields) in unknown { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!( + message.contains("unknown argument `force`"), + "{name}: {message}" + ); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn branch_remove_refuses_reserved_and_absent_branches() { + let (home, folder) = project(); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("main"))]); + assert_eq!(code, "ReservedName", "{message}"); + // nothing ran: main is still there + ok(&home, "branch.get", &[p(), ("branch", t("main"))]); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("ghost"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + // a live Branch is hidden by the marker and reported as `Removed` + ok( + &home, + "branch.fork", + &[p(), ("name", t("work")), ("from_branch", t("main"))], + ); + let removed = ok(&home, "branch.remove", &[p(), ("branch", t("work"))]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the project is the caller's /, not the Store's inner name + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert_eq!(removed.get_text("branch"), Some("work")); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + let (code, message) = refused(&home, "branch.get", &[p(), ("branch", t("work"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_remove_file_checks_its_path_grammar_then_reports_markers() { + let (home, folder) = project(); + for bad in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "\\", + ] { + let (code, message) = refused(&home, "project.remove_file", &[p(), ("path", t(bad))]); + assert_eq!(code, "InvalidInput", "{bad:?}: {message}"); + } + // no effect: the file the bad calls named is still readable + let read = ok( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.get_text("text"), Some("hello\n")); + + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the same full project name the caller passed + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("frontier_serial").is_some()); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + // a repeat at the same frontier is UpToDate + let repeat = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(repeat.get_text("kind"), Some("UpToDate")); + assert_eq!( + repeat.get_int("frontier_serial"), + removed.get_int("frontier_serial") + ); + // the marker hides it from reads + let (code, message) = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(code, "Removed", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_delete_reports_removal_and_retires_the_entry() { + let (home, folder) = project(); + let (code, message) = refused(&home, "project.delete", &[("project", t("shop/ghost"))]); + assert_eq!(code, "NoSuchProject", "{message}"); + let removed = ok(&home, "project.delete", &[p()]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("deleted_ms").is_some_and(|ms| ms >= 0)); + assert!(removed.get_bool("home_entry_removed")); + let (code, _) = refused(&home, "project.get", &[p()]); + assert_eq!(code, "NoSuchProject"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn store_compact_refuses_an_unknown_store_and_reports_a_run() { + let home = tempdir("compact-none"); + layerfs_api::home::open(&home).unwrap(); + let (code, message) = refused(&home, "store.compact", &[("name", t("ghost"))]); + assert_eq!(code, "NoSuchStore", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + let path = report.get_text("path").expect("path"); + assert!(matches!(path, "none" | "rebuild"), "{path}"); + let generation = report.get("generation").expect("generation"); + let before = generation.get_int("before").unwrap(); + let after = generation.get_int("after").unwrap(); + if path == "none" { + assert_eq!(before, after); + } else { + assert_eq!(after, before + 1); + } + for field in [ + "dead_bytes_bound", + "dead_bytes_refined", + "logical_discarded", + "backend_reusable", + "physical_shrink", + "peak_temp_bytes", + "estimate", + "zero_reasons", + "kept_waste", + "residue", + ] { + assert!(report.get(field).is_some(), "missing {field}"); + } + let kept = report.get("kept_waste").expect("kept_waste"); + for field in [ + "dead_in_retained_packs_bytes", + "delta_bases", + "inode_blocks_bytes", + ] { + assert!(kept.get(field).is_some(), "missing kept_waste.{field}"); + } + let residue = report.get("residue").expect("residue"); + assert!(residue.get("bytes").is_some()); + assert!(residue.get("artifacts").is_some()); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn a_bare_project_name_is_refused_by_every_project_call() { + let home = tempdir("bare-names"); + layerfs_api::home::open(&home).unwrap(); + let bare: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("api"))]), + ( + "project.remove_file", + &[("project", t("api")), ("path", t("/base.txt"))], + ), + ]; + for &(name, fields) in bare { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.contains("with its Store"), "{name}: {message}"); + } + // store.compact takes a Store name; a project path is not one + let (code, message) = refused(&home, "store.compact", &[("name", t("shop/api"))]); + assert_eq!(code, "InvalidInput", "{message}"); + assert!(message.contains("store name"), "{message}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/gc_trace.rs b/api/crates/layerfs-api/tests/gc_trace.rs new file mode 100644 index 0000000000..a3cb2144c2 --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_trace.rs @@ -0,0 +1,309 @@ +//! T-G2: the call log redacts every path argument (#41 §4.3). A +//! `project.remove_file` or a hidden `file.read`/`stat`/`list`/`glob` path is +//! logged as `` of its normalized form (one leading `/` stripped), +//! the `Removed` refusal still echoes the request path, the salted trace key +//! changes with the salt, and a `store.compact` report's artifact names never +//! reach the log. +use layerfs_api::calls::REDACTED; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::path::{path_key, trace_key}; +use layerfs_history::LayerStackId; +use layerfs_home::home::paths; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard}; + +static ENV: Mutex<()> = Mutex::new(()); +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// The tests share the process environment, so each one holds this lock. +fn serial() -> MutexGuard<'static, ()> { + let guard = ENV.lock().unwrap_or_else(|poison| poison.into_inner()); + std::env::remove_var("LAYERFS_LOGS"); + std::env::remove_var("LAYERFS_LOGS_TAG"); + guard +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-trace-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkRefusal { + call(home, name, fields).expect_err("the call is refused") +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +/// A home with Store `shop` and project `shop/api` imported from a folder +/// holding `base.txt`. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +/// Every line of the call log, oldest first. +fn lines(home: &Path) -> Vec { + match std::fs::read_to_string(paths::calls_file(home)) { + Ok(text) => text + .lines() + .map(|line| layerfs_api::json::parse(line).unwrap()) + .collect(), + Err(_) => Vec::new(), + } +} + +fn last(home: &Path) -> SdkValue { + lines(home).pop().expect("a line") +} + +fn log_text(home: &Path) -> String { + std::fs::read_to_string(paths::calls_file(home)).unwrap_or_default() +} + +fn argv(line: &SdkValue) -> Vec { + match line.get("argv") { + Some(SdkValue::List(words)) => words + .iter() + .map(|word| match word { + SdkValue::Text(text) => text.clone(), + other => panic!("argv word {other:?}"), + }) + .collect(), + other => panic!("no argv: {other:?}"), + } +} + +fn bytes(n: usize) -> String { + format!("<{n} bytes>") +} + +/// The one path a `Removed` refusal details. +fn removed_path(refusal: &SdkRefusal) -> &str { + let Some(SdkValue::List(paths)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + paths + .first() + .and_then(|path| path.get_text("path")) + .expect("one path") +} + +#[test] +fn path_arguments_are_byte_counts_in_the_call_log() { + let _env = serial(); + let (home, folder) = project(); + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!( + argv(&last(&home)), + [ + "project".to_owned(), + "remove-file".into(), + "shop/api".into(), + bytes("base.txt".len()), + ] + ); + + // Hidden reads, stats, listings and globs: a path is a byte count. + let read = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.code, "Removed"); + assert_eq!( + argv(&last(&home)), + [ + "file".to_owned(), + "read".into(), + "shop/api".into(), + "main".into(), + bytes("base.txt".len()), + ] + ); + let stat = refused( + &home, + "file.stat", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(stat.code, "Removed"); + assert_eq!(argv(&last(&home))[4], bytes("base.txt".len())); + + let listed = ok( + &home, + "file.list", + &[p(), ("branch", t("main")), ("path", t("/"))], + ); + assert_eq!(listed.get_int("removed_count"), Some(1)); + assert_eq!(argv(&last(&home))[4], bytes(0)); + + let globbed = ok( + &home, + "file.glob", + &[ + p(), + ("branch", t("main")), + ("path", t("/")), + ("pattern", t("*.txt")), + ], + ); + assert_eq!(globbed.get_int("removed_count"), Some(1)); + let words = argv(&last(&home)); + assert_eq!(words[4], bytes(0)); + assert_eq!(words[5], bytes("*.txt".len())); + + let text = log_text(&home); + assert!(!text.contains("/base.txt"), "raw path in the log:\n{text}"); + assert!(!text.contains("base.txt"), "raw path in the log:\n{text}"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn a_removed_refusal_echoes_the_path_the_log_does_not() { + let _env = serial(); + let (home, folder) = project(); + ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + let refusal = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(refusal.code, "Removed"); + assert_eq!(removed_path(&refusal), "/base.txt"); + let line = last(&home); + assert_eq!(line.get_text("outcome"), Some("Removed")); + assert!(line.get("detail").is_none(), "a detail reached the log"); + let text = log_text(&home); + assert!(!text.contains("/base.txt")); + assert!(!text.contains("base.txt")); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn the_redacted_set_covers_every_path_bearing_call() { + for name in [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", + ] { + assert!(REDACTED.contains(&name), "{name} is not redacted"); + } + for (name, args) in [ + ("project.remove_file", &["path"][..]), + ("file.read", &["path"][..]), + ("file.stat", &["path"][..]), + ("file.list", &["path"][..]), + ("file.glob", &["path", "pattern"][..]), + ("file.diff", &["left_path", "right_path"][..]), + ("file.cp", &["from_path", "to_path"][..]), + ("file.mv", &["from_path", "to_path"][..]), + ("branch.diff", &["path_prefix"][..]), + ] { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + for arg in args { + assert!(spec.arg(arg).is_some(), "{name} has no {arg}"); + assert!(REDACTED.contains(arg), "{name}'s {arg} is not redacted"); + } + } + // text that is not a path stays visible + for name in ["project", "branch", "name", "commit", "content", "changes"] { + assert!(!REDACTED.contains(&name), "{name} is redacted"); + } +} + +#[test] +fn the_trace_key_is_salted_by_the_store_policy_salt() { + let _env = serial(); + let stack = LayerStackId::from_authority([7; 16]); + let path = b"a/b"; + let first = trace_key(&[1; 32], stack, path); + assert_eq!(first, trace_key(&[1; 32], stack, path), "stable salt"); + let second = trace_key(&[2; 32], stack, path); + assert_ne!(first, second, "a different salt changes the keyed hash"); + assert_eq!(first.len(), 32); + // the keyed hash is not the marker's plain BLAKE3(normalized path) + assert_ne!(first, path_key(path)); + // the stack is part of the key + let other = LayerStackId::from_authority([9; 16]); + assert_ne!(first, trace_key(&[1; 32], other, path)); +} + +#[test] +fn store_compact_artifact_names_never_reach_the_log() { + let _env = serial(); + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + assert_eq!( + argv(&last(&home)), + ["store".to_owned(), "compact".into(), "shop".into()] + ); + let text = log_text(&home); + assert!( + !text.contains(".compact"), + "an artifact name reached the log:\n{text}" + ); + let store = paths::store_file(&home, "shop"); + let name = store.file_name().unwrap().to_string_lossy().into_owned(); + assert!(!text.contains(&name), "the Store file name reached the log"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/logs.rs b/api/crates/layerfs-api/tests/logs.rs index 5fdbf68dbf..cbb0f9cc44 100644 --- a/api/crates/layerfs-api/tests/logs.rs +++ b/api/crates/layerfs-api/tests/logs.rs @@ -181,21 +181,19 @@ fn argv_has_positionals_options_flags_and_the_scope_before_it_rewrites() { ("page_size", SdkValue::Int(5)), ], ); - assert_eq!( - argv(&last(&home)), - [ - "file", - "diff", - "shop/api", - "main", - "/README.md", - "main", - "/README.md", - "--allow-missing", - "--page-size", - "5" - ] - ); + let want: Vec = vec![ + "file".into(), + "diff".into(), + "shop/api".into(), + "main".into(), + bytes("README.md".len()), + "main".into(), + bytes("README.md".len()), + "--allow-missing".into(), + "--page-size".into(), + "5".into(), + ]; + assert_eq!(argv(&last(&home)), want); // optional positionals stop at the first absent one let _ = call(&home, "backup.list", &[]); assert_eq!(argv(&last(&home)), ["backup", "list"]); @@ -266,7 +264,7 @@ fn free_text_credentials_content_and_json_are_byte_counts() { "write".into(), "shop/api".into(), "dev".into(), - "/a.ts".into(), + bytes("a.ts".len()), bytes(5321), "--message".into(), bytes(5) @@ -283,16 +281,22 @@ fn free_text_credentials_content_and_json_are_byte_counts() { ], ); assert_eq!(argv(&last(&home))[5], bytes(7)); - // text is cut at 200 characters + // a path argument is logged as its byte length, never its text let long = format!("/{}", "é".repeat(300)); let _ = call( &home, "file.stat", &[p(), ("branch", t("main")), ("path", t(&long))], ); - let path = &argv(&last(&home))[4]; - assert_eq!(path.chars().count(), 201); - assert!(path.ends_with('…')); + // a path logs the length of its normalized form: one leading `/` is + // stripped, so this path counts 600 bytes, not 601 + assert_eq!(argv(&last(&home))[4], bytes(long.len() - 1)); + // text that is not redacted is cut at 200 characters + let long = "n".repeat(300); + let _ = call(&home, "store.get", &[("name", t(&long))]); + let name = &argv(&last(&home))[2]; + assert_eq!(name.chars().count(), 201); + assert!(name.ends_with('…')); // each named text argument let _ = call( &home, diff --git a/api/crates/layerfs-cli/tests/gc.rs b/api/crates/layerfs-cli/tests/gc.rs new file mode 100644 index 0000000000..36d04aeb68 --- /dev/null +++ b/api/crates/layerfs-cli/tests/gc.rs @@ -0,0 +1,169 @@ +//! T11.2: the four GC calls on the shell. Each has one command, the call +//! that it names and the arguments the table gives it: a wrong arity or an +//! option the call does not take is exit 2 before any call, a well-formed +//! command line reaches the library, and the help is the table's. +mod common; +use common::{layerfs, run, tempdir, text}; +use std::path::Path; + +fn run_code(home: &Path, words: &[&str]) -> (Option, String, String) { + let out = run(&mut layerfs(home, words)); + (out.status.code(), text(&out.stdout), text(&out.stderr)) +} + +/// The help line of each call, as the table renders it: remove-file is the +/// shell spelling of remove_file. +const CALLS: &[(&[&str], &str, &str)] = &[ + ( + &["branch", "remove"], + "layerfs branch remove ", + "soft-remove one Branch", + ), + ( + &["project", "delete"], + "layerfs project delete ", + "retire one project", + ), + ( + &["project", "remove-file"], + "layerfs project remove-file ", + "hide one absolute path", + ), + ( + &["store", "compact"], + "layerfs store compact ", + "reclaim dead bytes", + ), +]; + +#[test] +fn help_lists_each_gc_call_with_its_usage() { + let home = tempdir("gc-help"); + let (code, out, err) = run_code(&home, &["help"]); + assert_eq!(code, Some(0), "{err}"); + assert!(err.is_empty(), "{err}"); + let line = |family: &str| { + out.lines() + .find(|line| line.split_whitespace().next() == Some(family)) + .unwrap_or_else(|| panic!("no {family} line in {out}")) + }; + assert!(line("branch").contains("remove"), "{}", line("branch")); + assert!( + line("project").contains("delete") && line("project").contains("remove-file"), + "{}", + line("project") + ); + assert!(line("store").contains("compact"), "{}", line("store")); + + for (words, usage, help) in CALLS { + let topic: Vec<&str> = std::iter::once("help") + .chain(words.iter().copied()) + .collect(); + let (code, out, err) = run_code(&home, &topic); + assert_eq!(code, Some(0), "{words:?}: {err}"); + assert!(err.is_empty(), "{words:?}: {err}"); + assert!(out.starts_with(usage), "{words:?}: {out}"); + assert!(out.contains(help), "{words:?}: {out}"); + // the same help after the call, through -h and --help + for flag in ["-h", "--help"] { + let with_flag: Vec<&str> = words.iter().copied().chain([flag]).collect(); + let (code, more, err) = run_code(&home, &with_flag); + assert_eq!(code, Some(0), "{with_flag:?}: {err}"); + assert_eq!(more, out, "{with_flag:?}"); + } + } + assert!(!home.join("store").exists(), "help made no home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn wrong_arity_and_options_are_usage_errors() { + let home = tempdir("gc-usage"); + let wrong: &[&[&str]] = &[ + &["branch", "remove"], + &["branch", "remove", "shop/api"], + &["branch", "remove", "shop/api", "work", "extra"], + &["branch", "remove", "shop/api", "work", "--force"], + &["project", "delete"], + &["project", "delete", "shop/api", "extra"], + &["project", "delete", "shop/api", "--force"], + &["project", "remove-file", "shop/api"], + &["project", "remove-file", "shop/api", "/a", "extra"], + &["project", "remove-file", "shop/api", "/a", "--force"], + &["store", "compact"], + &["store", "compact", "shop", "extra"], + &["store", "compact", "shop", "--force"], + ]; + for words in wrong { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(2), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("layerfs: "), "{words:?}: {err}"); + } + // no positional was called: a usage error makes no home + assert!(!home.join("store").exists(), "a usage error made a home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn well_formed_lines_reach_the_library() { + let home = tempdir("gc-reach"); + let well_formed: &[&[&str]] = &[ + &["branch", "remove", "shop/api", "work"], + &["project", "delete", "shop/api"], + &["project", "remove-file", "shop/api", "/a"], + &["store", "compact", "shop"], + ]; + for words in well_formed { + let (code, out, err) = run_code(&home, words); + // no Store in the home: the library refused after reading the line + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!(err.contains("\"code\":\"NoSuchStore\""), "{words:?}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_bare_project_is_refused_and_the_four_are_out_of_scope() { + let home = tempdir("gc-names"); + // the library, not the parser, refuses a project without its Store + for words in [ + &["branch", "remove", "api", "main"][..], + &["project", "delete", "api"], + &["project", "remove-file", "api", "/a"], + ] { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!( + err.contains("\"code\":\"InvalidInput\""), + "{words:?}: {err}" + ); + assert!(err.contains("with its Store"), "{words:?}: {err}"); + } + + // a folder scope guards one folder and one Branch: none of the four is a + // folder operation, so every one is refused, never widened + let scoped: &[(&[&str], &str)] = &[ + (&["branch", "remove", "shop/api", "main"], "branch.remove"), + (&["project", "delete", "shop/api"], "project.delete"), + ( + &["project", "remove-file", "shop/api", "/web/a"], + "project.remove_file", + ), + (&["store", "compact", "shop"], "store.compact"), + ]; + for (words, call) in scoped { + let mut command: Vec<&str> = words.to_vec(); + command.extend(["--scope", "/web", "--scope-project", "shop/api"]); + let (code, out, err) = run_code(&home, &command); + assert_eq!(code, Some(125), "{call}: {err}"); + assert!(out.is_empty(), "{call}: {out}"); + assert!(err.contains("out of scope: "), "{call}: {err}"); + assert!(err.contains(call), "{call}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/src/store.rs b/control/crates/layerfs-control/src/store.rs index d67e04a111..8b7dc2334f 100644 --- a/control/crates/layerfs-control/src/store.rs +++ b/control/crates/layerfs-control/src/store.rs @@ -18,6 +18,12 @@ use std::path::Path; pub use layerfs_store::close_idle; +/// `store.compact`: reclaim one sqlite Store's dead bytes, answering the +/// §11 report. +pub fn compact(home: &Path, events: &mut Events<'_>, name: &StoreName) -> Outcome { + layerfs_store::compact::compact(home, events, name.as_str()) +} + /// Who uses a Store or a project, read from the Sandbox records. pub const RELATED: Related = Related { store_idle: inventory::store_idle, diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index 997d24c27e..3e7d2cf8bd 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -28,11 +28,12 @@ use layerfs_home::outcome::{refused, Outcome}; use layerfs_machine::worker::{ read_frame, read_frame_within, write_frame, Hello, HELLO_WAIT, MAX_HELLO, }; -use layerfs_store::project::delete::delete_project; +use layerfs_store::project::delete::{admit_project, delete_project, release_project}; use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; use layerfs_store::project::frame::{ - DeleteFrame, ExportFrame, InitFrame, ListingFrame, RemoveFrame, StageFrame, + DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, ExportFrame, InitFrame, ListingFrame, + RemoveFrame, StageFrame, }; use layerfs_store::project::listing::list_projects; use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; @@ -280,11 +281,23 @@ fn run( } // retires one project in the Store under this worker root and // removes the worker own entry (2.0, T4.5b): `{ store, stack_id, - // deleted_ms?, project?, store_record? }` + // deleted_ms?, project?, op_id?, store_record? }` "project.delete" => { let root = home_of(Some(root))?; delete_project(&root, &DeleteFrame::decode(&args)?) } + // registers the caller's project-scope admission before its sidecar + // and marker (2.0, T4.6): `{ store, stack_id, op_id, store_record? }` + "project.delete.admit" => { + let root = home_of(Some(root))?; + admit_project(&root, &DeleteAdmitFrame::decode(&args)?) + } + // releases the caller's project-scope admission after the marker + // (2.0, T4.6): `{ store, op_id, store_record? }` + "project.delete.release" => { + let root = home_of(Some(root))?; + release_project(&root, &DeleteReleaseFrame::decode(&args)?) + } // reads the Store's own filtered project listing under this worker // root, the same `layerfs_project::store::projects` a local Store // uses (2.0, T4.5c): `{ store, store_record? }` diff --git a/control/crates/layerfs-control/tests/project_delete_worker.rs b/control/crates/layerfs-control/tests/project_delete_worker.rs new file mode 100644 index 0000000000..c1ab487c20 --- /dev/null +++ b/control/crates/layerfs-control/tests/project_delete_worker.rs @@ -0,0 +1,550 @@ +//! `project.delete` on a worker-backed Store (T4.6): the caller admits the +//! project scope through the worker before it writes its sidecar and sends +//! the marker. A live mount in the worker's Store refuses the call with +//! `MountsActive` (holders reported) and a live operation with `Busy`, with +//! no sidecar, no marker and no leaked admission row; the successful path +//! releases the admission after the marker. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{self, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store_service::history::lookup; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layerfs_home::home::layout::create(&directory).unwrap(); + directory +} + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + write_record( + &record_path, + &home::with_field(record, "machine", text(machine)), + ); +} + +fn write_record(path: &Path, value: &SdkValue) { + home::write_record(path, value).unwrap(); +} + +/// A worker's `OpenArgs` for Store `shop` under `root`, with the caller's +/// Store record (the same context every `project.delete*` frame carries). +fn worker_open(root: &Path, record: &StoreRecord) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(root, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + } +} + +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +fn project_names(open: &OpenArgs) -> Vec { + with_service(open, |service| { + let listed = service + .store + .published(layerfs_project::store::projects) + .unwrap(); + Ok(listed + .into_iter() + .map(|stack| stack.name.as_str().to_owned()) + .collect()) + }) + .unwrap() +} + +fn remove(home: &Path) -> Result { + let mut sink = |_| {}; + layerfs_control::project::delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + ) +} + +#[test] +fn the_worker_path_admits_before_the_sidecar_and_releases_after_the_marker() { + // a home with Store `shop` and project `app`, created locally. A + // sandbox filesystem cannot hold a directly opened Store; the fake-home + // cases in the `layerfs-store` tests cover the same home path, and this + // case runs where the Store's filesystem carries SQLite locks. + let home = tempdir("home"); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) + if refusal.code == "Unsupported" + && refusal.message.contains("does not carry SQLite locks") => + { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir("folder"); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: None, + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let caller_open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let (stack, branch) = with_service(&caller_open, |service| { + let stack = lookup::project(&service.store, "app").unwrap(); + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + let NamedSnapshot::Found(snapshot) = service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + else { + panic!("no main Branch"); + }; + Ok((stack.id, snapshot.branch.id)) + }) + .unwrap(); + + // the Store belongs to machine `box`; its file is under the worker's root + let root = tempdir("root"); + move_store_file(&home, &root, "shop"); + home::write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", text("box")), ("platform", text("lima"))]), + ) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(&address)), ("secret", text("k"))]), + ) + .unwrap(); + place_store(&home, "shop", "box"); + let served = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), served) + }); + + let worker = worker_open(&root, &record); + + // a live mount in the worker's Store: `MountsActive`, no sidecar, no marker + let mount = GcIntent::shared( + [9; 16], + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(&worker, |service| { + service.store.history().gc_register(&mount).unwrap(); + Ok(()) + }) + .unwrap(); + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + assert!( + intents(&worker, stack).is_empty(), + "no project-scope row yet" + ); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None, + "no sidecar was written" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert_eq!(project_names(&worker), vec!["app".to_owned()], "no marker"); + assert!( + intents(&worker, stack).is_empty(), + "the refused admission left no project-scope row" + ); + + // the mount ends; a live removal at project scope is `Busy` + with_service(&worker, |service| { + service.store.history().gc_release(&[9; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + let holder = GcIntent::operation( + [7; 16], + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("app"), None); + with_service(&worker, |service| { + service.store.history().gc_register(&holder).unwrap(); + Ok(()) + }) + .unwrap(); + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert_eq!( + intents(&worker, stack).len(), + 1, + "only the test's own holder" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + + // the holder ends: the delete succeeds through the worker and releases + // the admission it took, with the sidecar unlinked + with_service(&worker, |service| { + service.store.history().gc_release(&[7; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert!( + intents(&worker, stack).is_empty(), + "the admission was released" + ); + assert!(project_names(&worker).is_empty(), "the project is retired"); + let _ = std::fs::remove_dir_all(&home); + let _ = std::fs::remove_dir_all(&root); +} + +/// A fake worker on `listener`: answers its secret, answers each request with +/// `reply`, and shares back the requests it read. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Arc>> { + let seen: Arc>> = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + let _ = write_frame(&mut stream, &answer); + } + }); + seen +} + +/// The `op` of one request the fake worker read. +fn op_of(request: &SdkValue) -> &str { + request.get_text("op").unwrap_or("") +} + +/// A frame's `args`. +fn args_of(request: &SdkValue) -> &SdkValue { + request.get("args").expect("args") +} + +/// The `op_id` a request's args carry. +fn sent_op(request: &SdkValue) -> String { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") + .to_owned() +} + +fn admitted_for(stack: LayerStackId) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Admitted")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ]), + )]) +} + +fn removed_for(stack: LayerStackId, deleted_ms: i64) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Removed")), + ("project", text("app")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(deleted_ms)), + ]), + )]) +} + +fn released_for(request: &SdkValue) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Released")), + ("op_id", text(&sent_op(request))), + ]), + )]) +} + +fn refused(code: &str, detail: SdkValue) -> SdkValue { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", text(code)), + ("message", text("the fake worker refused")), + ("detail", detail), + ]), + )]) +} + +/// A caller's home naming Store `shop` on machine `box`, whose project `app` +/// records `stack`, with the fake worker's address. +fn caller_home(tag: &str, address: &str, stack: LayerStackId) -> PathBuf { + let home = tempdir(tag); + home::write_record( + &paths::store_record(&home, "shop"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("kind", text("sqlite")), + ("name", text("shop")), + ("machine", text("box")), + ]), + ) + .unwrap(); + home::write_record( + &paths::project_record(&home, "shop", "app"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", text("app")), + ("stack", text(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(address)), ("secret", text("k"))]), + ) + .unwrap(); + home +} + +#[test] +fn a_fake_workers_mounts_active_refusal_writes_no_sidecar_and_no_marker() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([3; 16]); + let home = caller_home("fake-mount", &address, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", text("mount")), + ("entity", text("main")), + ])]), + )]); + let seen = fake_worker(listener, move |request| { + assert_eq!(op_of(request), "project.delete.admit"); + refused("MountsActive", holders.clone()) + }); + + let refusal = remove(&home).expect_err("the fake worker's Store has a live mount"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + // the caller wrote no sidecar and sent no marker: only the admission frame + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1, "{requests:?}"); + assert_eq!(op_of(&requests[0]), "project.delete.admit"); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_marker_refusal_releases_the_admission_it_took() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([4; 16]); + let home = caller_home("fake-busy", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => refused("Busy", SdkValue::Null), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("a live operation holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + // the admission was taken, named in the sidecar, used by the marker and + // released after the marker's refusal + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + drop(requests); + let body: [u8; 16] = hex_decode(&op).try_into().unwrap(); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays for a retry") + .op, + Some(body) + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_success_removes_the_entry_and_releases_the_admission() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([5; 16]); + let home = caller_home("fake-ok", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => removed_for(stack, 42), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The bytes of a lowercase hex string. +fn hex_decode(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) + .collect() +} diff --git a/control/crates/layerfs-store/src/calls/branch.rs b/control/crates/layerfs-store/src/calls/branch.rs index 20dc55f4e9..a37bdee913 100644 --- a/control/crates/layerfs-store/src/calls/branch.rs +++ b/control/crates/layerfs-store/src/calls/branch.rs @@ -117,6 +117,15 @@ pub struct BranchPublish { pub commit: Option, } +/// `branch.remove(project, branch)`. +#[derive(Clone, Debug, Default)] +pub struct BranchRemove { + /// The project. + pub project: String, + /// The Branch. + pub branch: String, +} + /// `branch.diff(project, from_branch, to_branch, { paths_only, page_size, /// page_pos, cursor })`. #[derive(Clone, Debug, Default)] @@ -180,6 +189,23 @@ pub fn publish(home: &Path, events: &mut Events<'_>, request: &BranchPublish) -> on_project(home, events, &request.project, StoreOp::Publish(request)) } +/// `branch.remove`: one Branch hidden by its Store marker. The result names +/// the project as the caller gave it, `/`: the Store sees +/// only the project's name within it and echoes that alone. +pub fn remove(home: &Path, events: &mut Events<'_>, request: &BranchRemove) -> Outcome { + let value = on_project( + home, + events, + &request.project, + StoreOp::BranchRemove(request), + )?; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} + /// `branch.diff`: the changes between two Branches. pub fn diff(home: &Path, events: &mut Events<'_>, request: &BranchDiff) -> Outcome { on_project( diff --git a/control/crates/layerfs-store/src/calls/op.rs b/control/crates/layerfs-store/src/calls/op.rs index 26308d38cf..4e43f6cd81 100644 --- a/control/crates/layerfs-store/src/calls/op.rs +++ b/control/crates/layerfs-store/src/calls/op.rs @@ -4,6 +4,7 @@ //! a `store.call` frame to the Store's worker). use super::branch::{ BranchCommits, BranchDiff, BranchFork, BranchGetCommit, BranchMerge, BranchPublish, + BranchRemove, }; use super::commit::CommitDiff; use super::file::{ @@ -62,6 +63,9 @@ pub enum StoreOp<'a> { Statistics, /// Entry counts at one project pin. ProjectStats(&'a super::project::ProjectStats), + /// `project.remove_file`: hide one path from every node at or below its + /// marker frontier. + RemoveFile(&'a super::project::ProjectRemoveFile), /// `project.list_branches`: the Branch `name`, or one page. ListBranches { /// The one Branch it lists. @@ -88,6 +92,8 @@ pub enum StoreOp<'a> { Merge(&'a BranchMerge), /// `branch.publish`. Publish(&'a BranchPublish), + /// `branch.remove`. + BranchRemove(&'a BranchRemove), /// `branch.diff_branches`. DiffBranches(&'a BranchDiff), /// `branch.diff_commits`. @@ -236,6 +242,7 @@ impl StoreOp<'_> { match self { Self::Statistics => "store.stats", Self::ProjectStats(_) => "project.stats", + Self::RemoveFile(_) => "project.remove_file", Self::ListBranches { .. } => "project.list_branches", Self::ListCommits { .. } => "branch.list_commits", Self::BranchCommits(_) => "branch.commits", @@ -244,6 +251,7 @@ impl StoreOp<'_> { Self::Fork(_) => "branch.fork", Self::Merge(_) => "branch.merge", Self::Publish(_) => "branch.publish", + Self::BranchRemove(_) => "branch.remove", Self::DiffBranches(_) => "branch.diff_branches", Self::DiffCommits(_) => "branch.diff_commits", Self::Read(_) => "fs.read", @@ -269,6 +277,7 @@ impl StoreOp<'_> { match self { Self::Statistics => &mut args, Self::ProjectStats(r) => args.at(&r.branch, r.commit.as_deref()), + Self::RemoveFile(r) => args.text("path", &r.path), Self::ListBranches { name, page } => args.opt("name", *name).page(*page), Self::ListCommits { branch, page } => args.text("branch", branch).page(*page), Self::BranchCommits(r) => args @@ -297,6 +306,7 @@ impl StoreOp<'_> { Self::Publish(r) => args .text("branch", &r.branch) .opt("commit", r.commit.as_deref()), + Self::BranchRemove(r) => args.text("branch", &r.branch), Self::DiffBranches(r) => args .text("from", &r.from_branch) .text("to", &r.to_branch) diff --git a/control/crates/layerfs-store/src/calls/project.rs b/control/crates/layerfs-store/src/calls/project.rs index d61b25fb0f..baaeef1ee5 100644 --- a/control/crates/layerfs-store/src/calls/project.rs +++ b/control/crates/layerfs-store/src/calls/project.rs @@ -1,5 +1,6 @@ //! Saved-tree project statistics through the ordinary Store-call route. use super::op::{on_project, StoreOp}; +use layerfs_bridge::control::SdkValue; use layerfs_home::{events::Events, outcome::Outcome}; use std::path::Path; @@ -14,6 +15,15 @@ pub struct ProjectStats { pub commit: Option, } +/// `project.remove_file(project, path)`. +#[derive(Clone, Debug, Default)] +pub struct ProjectRemoveFile { + /// `/`. + pub project: String, + /// The absolute POSIX path. + pub path: String, +} + /// Count one immutable saved tree, excluding the root directory. pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Outcome { on_project( @@ -23,3 +33,15 @@ pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Ou StoreOp::ProjectStats(request), ) } + +/// `project.remove_file`: one path hidden from every node at or below +/// its marker frontier. The result names the project as the caller gave it, +/// `/`. +pub fn remove_file(home: &Path, events: &mut Events<'_>, request: &ProjectRemoveFile) -> Outcome { + let value = on_project(home, events, &request.project, StoreOp::RemoveFile(request))?; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs new file mode 100644 index 0000000000..dab93bac77 --- /dev/null +++ b/control/crates/layerfs-store/src/compact.rs @@ -0,0 +1,150 @@ +//! `store.compact`: one sqlite Store's §7 rebuild, run in this process. +//! +//! The call takes the Store's lease, so a move or remove cannot start, and +//! closes the Store this process holds open: a call still running on it is +//! `InUse`, before any effect. It then opens the Store's own session, runs +//! the §7 (SQLite) or §8 (PostgreSQL) compaction and answers the §11 report. +//! A SQLite Store on another machine is `Unsupported` here: its rebuild runs +//! in that machine's worker. +use crate::identity; +#[cfg(feature = "postgresql")] +use crate::identity::Kind; +use crate::lease; +use crate::open::close; +use crate::open_args::OpenArgs; +use crate::place::{self, Executor}; +#[cfg(feature = "postgresql")] +use crate::postgresql; +use crate::record::{self, StoreRecord, StoreRef}; +use layerfs_bridge::control::SdkRefusal; +use layerfs_home::events::Events; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused, Outcome}; +#[cfg(feature = "postgresql")] +use layerfs_persistence::PostgresHandles; +use layerfs_persistence::{ + CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_store_service::history::{admission, lookup}; +use std::path::Path; + +/// `store.compact(name)`: reclaim one sqlite Store's dead bytes, answering +/// the §11 report. +pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { + let home = home.to_path_buf(); + let (_lease, record) = lease::claim(&home, store)?; + let store_ref = record.store_ref()?.clone(); + if matches!(store_ref, StoreRef::Postgres { .. }) { + return compact_postgresql(&record, &home, store); + } + match place::plan(&home, &store_ref)? { + Executor::Local => {} + Executor::Worker(_) => { + return Err(refused( + "Unsupported", + format!("store.compact of Store {store} runs on its own machine, not this one"), + )) + } + } + let path = paths::store_file(&home, store); + // A cached open in this process must end first; a call running on it is + // `InUse`, before any effect. + close(&path)?; + let args = OpenArgs { + home: Some(home), + store: Some(store.to_owned()), + project: None, + store_path: Some(path.display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let identity = identity::identity(&args)?; + let handles = SqliteHandles::open_writable( + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable), + &identity.binding, + identity.cursor_key, + ) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let receipt = handles + .compact(&CompactRequest { + op_id: admission::op_id(), + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + ..CompactRequest::default() + }) + .map_err(lookup::history)?; + // One serializer for the shared §11 report: the SQLite receipt projects + // into `CompactReport`, and T9's PostgreSQL receipt will do the same. + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// `store.compact` of a PostgreSQL Store in a build without the `postgresql` +/// feature is `Unsupported`. +#[cfg(not(feature = "postgresql"))] +fn compact_postgresql(_record: &StoreRecord, _home: &Path, _store: &str) -> Outcome { + Err(refused( + "Unsupported", + "this LayerFS build has no PostgreSQL engine; build the binding with --features postgresql", + )) +} + +/// The §8 PostgreSQL path: dial the Store's recorded URL, open its schema as +/// this home's authority, and run Phase M/A/B/C. The Store is usable at +/// G + 1 from the swap on; a lost reply is judged by the next call (§9). +#[cfg(feature = "postgresql")] +fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome { + let url = record::url(record)?; + let id = identity::identity(&OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + store_record: Some(record.to_record().clone()), + ..OpenArgs::default() + })?; + let Kind::Postgresql { + schema, + synchronous_commit, + } = &id.kind + else { + return Err(refused( + "Failed", + format!("Store {store} is not a PostgreSQL Store"), + )); + }; + let selection = postgresql::open::selection(&url, schema, synchronous_commit); + let handles = PostgresHandles::open_writable(&selection, &id.binding, id.cursor_key) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let receipt = handles + .compact(&CompactRequest { + op_id: admission::op_id(), + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + ..CompactRequest::default() + }) + .map_err(lookup::history)?; + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// An open refusal before the run: `Busy` is definite, everything else a +/// failure with its own text. +fn open_refusal(error: layerfs_storage::port::PersistenceError) -> SdkRefusal { + use layerfs_storage::port::PersistenceError; + match error { + PersistenceError::Busy => refused("Busy", "the Store is held by another writer"), + PersistenceError::Refused { status } => refused( + "Busy", + format!("the Store refused the compaction: {status}"), + ), + other => failed(other), + } +} diff --git a/control/crates/layerfs-store/src/lib.rs b/control/crates/layerfs-store/src/lib.rs index 97fdd79303..449f106438 100644 --- a/control/crates/layerfs-store/src/lib.rs +++ b/control/crates/layerfs-store/src/lib.rs @@ -14,6 +14,7 @@ pub mod backup; pub mod call; #[doc(hidden)] pub mod calls; +pub mod compact; pub mod diagnostic; #[doc(hidden)] pub mod identity; @@ -26,6 +27,7 @@ pub mod open_args; #[doc(hidden)] pub mod place; pub mod postgresql; +pub mod report; #[doc(hidden)] pub mod project; #[doc(hidden)] diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs index dbf5dc9885..12fb89b68b 100644 --- a/control/crates/layerfs-store/src/project/delete.rs +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -17,28 +17,41 @@ //! that is there now. //! //! The worker op is `project.delete`; its frame is -//! `{store, stack_id, deleted_ms?, project?, store_record?}`. The two optional -//! context fields are the same ones `project.remove` sends: the worker opens -//! the Store under its own root with the caller's `store_record`, and the -//! project name lets its reply and its own home entry name the project. The -//! worker saves the legacy own-project identity, writes the marker, removes -//! its own entry and replies `{kind, project, stack_id, deleted_ms}`; the -//! caller removes its own entry and echoes `home_entry_removed`. +//! `{store, stack_id, deleted_ms?, project?, op_id?, store_record?}`. The two +//! optional context fields are the same ones `project.remove` sends: the +//! worker opens the Store under its own root with the caller's `store_record`, +//! and the project name lets its reply and its own home entry name the +//! project. The worker saves the legacy own-project identity, writes the +//! marker, removes its own entry and replies `{kind, project, stack_id, +//! deleted_ms}`; the caller removes its own entry and echoes +//! `home_entry_removed`. +//! +//! Before any of that, the home path admits the project scope (T4.6, +//! `project.delete.admit` on the worker route): the same `gc_register` the +//! Store-side removals rely on, taken before the identity save, the sidecar +//! and the marker. A live mount refuses `MountsActive` with its holders and a +//! live operation `Busy`, with no sidecar written and no marker. The marker +//! reuses the admission's operation identity, and the caller releases it on +//! every exit path (`project.delete.release`), so a refused or failed call +//! leaves no admission row behind. use crate::open_args::OpenArgs; use crate::place::{self, Executor, Worker}; -use crate::project::frame::{DeleteFrame, DeleteReply}; +use crate::project::frame::{DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, DeleteReply}; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; use crate::{identity, lease, service}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::StoreManifest; use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::HistoryError; use layerfs_history::LayerStackId; use layerfs_home::events::Events; use layerfs_home::home::{now_ms, paths}; use layerfs_home::names::{ProjectRef, StoreName}; use layerfs_home::outcome::{failed, refused, Outcome}; -use layerfs_store_service::history::deletion::{delete_stack, Deletion}; +use layerfs_store_service::history::admission as store_admission; +use layerfs_store_service::history::deletion::{delete_stack_with, Deletion}; +use layerfs_store_service::history::lookup; use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; @@ -98,6 +111,10 @@ pub struct Intent { pub attempt_ms: i64, /// Where the attempt got to. pub state: IntentState, + /// The project-scope admission this attempt took, when it took one. A + /// retry re-registers the same identity, so a call whose reply was lost + /// settles its own row instead of meeting it as another operation. + pub op: Option<[u8; 16]>, } static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0); @@ -135,6 +152,14 @@ impl Intent { .get_text("state") .and_then(IntentState::parse) .ok_or_else(malformed)?; + let op = match value.get_text("op_id") { + Some(text) => { + let bytes = unhex(text).ok_or_else(malformed)?; + let body: [u8; 16] = bytes.as_slice().try_into().map_err(|_| malformed())?; + Some(body) + } + None => None, + }; Ok(Some(Self { store, project, @@ -142,6 +167,7 @@ impl Intent { deleted_ms, attempt_ms, state, + op, })) } @@ -165,14 +191,21 @@ impl Intent { std::process::id(), unique )); - let value = SdkValue::map([ - ("store", SdkValue::text(&intent.store)), - ("project", SdkValue::text(&intent.project)), - ("stack_id", SdkValue::text(hex(&intent.stack.to_bytes()))), - ("deleted_ms", SdkValue::Int(intent.deleted_ms)), - ("attempt_ms", SdkValue::Int(intent.attempt_ms)), - ("state", SdkValue::text(intent.state.as_str())), - ]); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("store".to_owned(), SdkValue::text(&intent.store)), + ("project".to_owned(), SdkValue::text(&intent.project)), + ( + "stack_id".to_owned(), + SdkValue::text(hex(&intent.stack.to_bytes())), + ), + ("deleted_ms".to_owned(), SdkValue::Int(intent.deleted_ms)), + ("attempt_ms".to_owned(), SdkValue::Int(intent.attempt_ms)), + ("state".to_owned(), SdkValue::text(intent.state.as_str())), + ]; + if let Some(op) = intent.op { + fields.push(("op_id".to_owned(), SdkValue::text(hex(&op)))); + } + let value = SdkValue::Map(fields); let mut text = layerfs_home::json::print(&value); text.push('\n'); let mut file = OpenOptions::new() @@ -218,6 +251,24 @@ impl Intent { } } +/// The project-scope admission `project.delete` takes before its sidecar and +/// marker: the admission row's operation identity, reused by the marker so the +/// one row is excluded from its own scan and released once. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Admission { + /// The `gc_intent` operation identity. + pub op: [u8; 16], + /// The project scope it holds. + pub stack: LayerStackId, +} + +impl Admission { + /// One admission of `stack` under operation `op`. + pub const fn new(stack: LayerStackId, op: [u8; 16]) -> Self { + Self { op, stack } + } +} + /// The Store side of one `project.delete` attempt and the caller's entry /// removal, as one seam so the home path is testable without a Store. pub trait Effects { @@ -225,9 +276,21 @@ pub trait Effects { /// marker; idempotent on a retry. fn prepare(&mut self) -> Result<()>; - /// Marks (or reclassifies) `stack`. A retry passes the intent's - /// `deleted_ms`. - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result; + /// Takes the project-scope admission before the sidecar and the marker: + /// a live mount is `MountsActive` with its holders, a live operation + /// `Busy`, an uncertain owner `Unresolved`. A refusal registers nothing. + /// `reuse` is the identity the sidecar recorded for a retry; the same row + /// is re-registered under it, so a lost reply is settled, never met as a + /// second operation. + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result; + + /// Releases `admission` on every exit path: success, refusal and an + /// unknown Store outcome. A provider without admission releases nothing. + fn release(&mut self, admission: Admission) -> Result<()>; + + /// Marks (or reclassifies) `admission`'s Stack, reusing the admission's + /// operation identity. A retry passes the intent's `deleted_ms`. + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result; /// Removes the caller's home entry; `Ok(())` when nothing remains. fn remove_entry(&mut self, entry: &Path) -> Result<()>; @@ -267,12 +330,67 @@ fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool ]) } +/// The mutating part of `delete_with` after the project scope is admitted: +/// the identity save, the sidecar, the marker and the entry removal (spec §4). +fn mark_and_finish( + home: &Path, + store: &str, + project: &str, + stack: LayerStackId, + intent: Option, + mut deleted_ms: i64, + effects: &mut impl Effects, + admission: Admission, +) -> Outcome { + effects.prepare()?; + let mut current = match intent { + Some(intent) => intent, + None => Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms, + attempt_ms: now_ms(), + state: IntentState::Pending, + op: None, + }, + }; + // the sidecar names the admission, so a retry reuses it + current.op = Some(admission.op); + if current.state == IntentState::Pending { + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + } + + let deletion = effects.mark(admission, deleted_ms)?; + deleted_ms = deletion.deleted_ms(deleted_ms); + current.deleted_ms = deleted_ms; + current.state = IntentState::Marked; + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + + let home_entry_removed = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + if home_entry_removed { + current.state = IntentState::Done; + Intent::write(home, ¤t)?; + Intent::unlink(home, store, project)?; + } + Ok(removed(store, project, deleted_ms, home_entry_removed)) +} + /// Runs `project.delete` for a home entry whose Store side is `effects`. /// /// `stack` is the Stack id resolved from the record or the Store listing; a /// record without one whose Store does not list it is a stale entry and is /// removed without a marker. The same function serves the local and worker /// routes, and a fake in tests. +/// +/// The project scope is admitted before the identity save, the sidecar and +/// the marker: a live mount is `MountsActive` with its holders and a live +/// operation `Busy`, reported with no sidecar written and no marker. The +/// admission is released on every exit path. pub fn delete_with( home: &Path, store: &str, @@ -294,7 +412,7 @@ pub fn delete_with( } } - let (stack, mut deleted_ms) = match &intent { + let (stack, deleted_ms) = match &intent { Some(intent) => (intent.stack, intent.deleted_ms), None => { let Some(stack) = stack.or(record_stack) else { @@ -319,6 +437,8 @@ pub fn delete_with( if let Some(done) = &intent { if done.state == IntentState::Done { + // The retired-row entry-removal retry needs no admission: the + // marker is already there and only the home entry is removed. effects.prepare()?; let removed_entry = effects .remove_entry(&paths::project_dir(home, store, project)) @@ -331,41 +451,42 @@ pub fn delete_with( } } - // The identity save comes before the intent, and the intent before the - // marker (spec §4). - effects.prepare()?; - let mut current = match intent { - Some(intent) => intent, - None => Intent { - store: store.to_owned(), - project: project.to_owned(), - stack, - deleted_ms, - attempt_ms: now_ms(), - state: IntentState::Pending, - }, + // Admission before the identity save, the sidecar and the marker. A + // refusal returns here, so neither the sidecar nor the marker is written. + // A retry re-registers the identity its sidecar recorded; a refusal then + // releases that identity, never adopting it as a live operation. + let reuse = intent.as_ref().and_then(|intent| intent.op); + let admission = match effects.admit(stack, reuse) { + Ok(admission) => admission, + Err(refusal) => { + if let Some(op) = reuse { + let _ = effects.release(Admission::new(stack, op)); + } + return Err(refusal); + } }; - if current.state == IntentState::Pending { - current.attempt_ms = now_ms(); - Intent::write(home, ¤t)?; - } - - let deletion = effects.mark(stack, deleted_ms)?; - deleted_ms = deletion.deleted_ms(deleted_ms); - current.deleted_ms = deleted_ms; - current.state = IntentState::Marked; - current.attempt_ms = now_ms(); - Intent::write(home, ¤t)?; - - let home_entry_removed = effects - .remove_entry(&paths::project_dir(home, store, project)) - .is_ok(); - if home_entry_removed { - current.state = IntentState::Done; - Intent::write(home, ¤t)?; - Intent::unlink(home, store, project)?; + let attempt = mark_and_finish( + home, store, project, stack, intent, deleted_ms, effects, admission, + ); + let released = effects.release(admission); + match attempt { + Ok(outcome) => { + released?; + Ok(outcome) + } + // The original failure is the reported one; a failed release is named + // so a leaked admission row is not silent. + Err(error) => Err(match released { + Ok(()) => error, + Err(cause) => SdkRefusal { + message: format!( + "{}; releasing the project admission: {}", + error.message, cause.message + ), + ..error + }, + }), } - Ok(removed(store, project, deleted_ms, home_entry_removed)) } /// Removes a directory tree, treating an absent entry as removed. @@ -395,9 +516,36 @@ impl Effects for Local { identity::save_record(&self.home, &self.store, &id) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result { + let intent = match reuse { + Some(op) => store_admission::project_removal_at(stack, op), + None => store_admission::project_removal_from_id(stack), + }; + let op = intent.op_id; + service::with_service(&self.open, |service| { + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(Admission::new(stack, op)), + // A provider without admission cannot exclude a removal; the + // marker's own admission still runs, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(Admission::new(stack, op)), + Err(error) => Err(lookup::history(error)), + } + }) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + service::with_service(&self.open, |service| { + match service.store.history().gc_release(&admission.op) { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(()), + Err(error) => Err(lookup::history(error)), + } + }) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { + let intent = store_admission::project_removal_at(admission.stack, admission.op); service::with_service(&self.open, |service| { - delete_stack(&service.store, stack, deleted_ms) + delete_stack_with(&service.store, admission.stack, deleted_ms, &intent) }) } @@ -420,19 +568,61 @@ impl Effects for Remote { Ok(()) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { + fn admit(&mut self, stack: LayerStackId, reuse: Option<[u8; 16]>) -> Result { + let op = reuse.unwrap_or_else(store_admission::op_id); + let frame = DeleteAdmitFrame { + store: StoreName::parse(&self.store)?, + stack, + op, + store_record: Some(self.store_record.clone()), + }; + let answer = match self + .worker + .call("project.delete.admit", frame.encode(), &mut |_| {}) + { + Ok(answer) => answer, + Err(refusal) => { + // The request went out, so the worker may have registered the + // row before its answer was lost: settle the identity before + // reporting the unknown outcome. + if refusal.code == "Unknown" { + let _ = self.release(Admission::new(stack, op)); + } + return Err(refusal); + } + }; + if let Err(refusal) = admit_reply(&answer, stack) { + let _ = self.release(Admission::new(stack, op)); + return Err(refusal); + } + Ok(Admission::new(stack, op)) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + let frame = DeleteReleaseFrame { + store: StoreName::parse(&self.store)?, + op: admission.op, + store_record: Some(self.store_record.clone()), + }; + self.worker + .call("project.delete.release", frame.encode(), &mut |_| {})?; + Ok(()) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { let frame = DeleteFrame { store: StoreName::parse(&self.store)?, project: Some(self.project.clone()), - stack, + stack: admission.stack, deleted_ms: Some(deleted_ms), + op_id: Some(admission.op), store_record: Some(self.store_record.clone()), }; let answer = self .worker .call("project.delete", frame.encode(), &mut |_| {})?; let reply = DeleteReply::decode(&answer)?; - if reply.stack != stack { + if reply.stack != admission.stack { return Err(refused( "Failed", "the worker answered for another LayerStack", @@ -448,6 +638,24 @@ impl Effects for Remote { } } +/// Checks the worker's `project.delete.admit` answer. +fn admit_reply(answer: &SdkValue, stack: LayerStackId) -> Result<()> { + if answer.get_text("kind") != Some("Admitted") { + return Err(refused( + "Failed", + "the worker project.delete admission is not `Admitted`", + )); + } + let answered = answer + .get_text("stack_id") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()); + if answered != Some(stack) { + return Err(refused("Failed", "the worker admitted another LayerStack")); + } + Ok(()) +} + /// The `OpenArgs` a local Store call opens with. `record` is absent for a /// Store made by its own project Init, whose identity is the project entry. fn open_args( @@ -569,23 +777,95 @@ pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Out } } +/// The `OpenArgs` a worker's `project.delete` ops open the Store with: the +/// Store file under the worker's own root and the caller's Store record. +fn worker_open( + root: &Path, + store: &str, + project: Option, + store_record: SdkValue, +) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + project, + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + } +} + +/// The `{kind: "Admitted", stack_id}` answer. +fn admitted(stack: LayerStackId) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Admitted")), + ("stack_id", SdkValue::text(hex(&stack.to_bytes()))), + ]) +} + +/// The `{kind: "Released", op_id}` answer. +fn released(op: [u8; 16]) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Released")), + ("op_id", SdkValue::text(hex(&op))), + ]) +} + +/// The worker's `project.delete.admit`: registers the caller's project-scope +/// admission before the caller writes its sidecar and before the marker, so a +/// live mount refuses with `MountsActive` (its holders reported) and a live +/// operation with `Busy`, with no sidecar written. +pub fn admit_project(root: &Path, frame: &DeleteAdmitFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.admit needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + let intent = store_admission::project_removal_at(frame.stack, frame.op); + service::with_service(&open, |service| { + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(admitted(frame.stack)), + // A provider without admission cannot exclude a removal; the + // marker still admits, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(admitted(frame.stack)), + Err(error) => Err(lookup::history(error)), + } + }) +} + +/// The worker's `project.delete.release`: deletes the caller's admission row +/// after the marker, on every exit path. +pub fn release_project(root: &Path, frame: &DeleteReleaseFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.release needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + service::with_service(&open, |service| { + match service.store.history().gc_release(&frame.op) { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(released(frame.op)), + Err(error) => Err(lookup::history(error)), + } + }) +} + /// The worker's `project.delete` on a Store under `root`: save the legacy /// own-project identity, write the marker, remove the worker's own entry and -/// answer `{kind, project, stack_id, deleted_ms}`. +/// answer `{kind, project, stack_id, deleted_ms}`. The marker reuses the +/// caller's admission operation identity when its frame carries one, so the +/// caller's admitted row is excluded from its own scan; without one the +/// marker takes a fresh identity and its own admission, as before. pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { let store = frame.store.as_str(); let store_record = frame .store_record .clone() .ok_or_else(|| refused("InvalidInput", "project.delete needs the Store's record"))?; - let open = OpenArgs { - home: Some(root.to_path_buf()), - store: Some(store.to_owned()), - project: frame.project.clone(), - store_path: Some(paths::store_file(root, store).display().to_string()), - store_record: Some(store_record), - postgresql: None, - }; + let open = worker_open(root, store, frame.project.clone(), store_record); // The legacy own-project entry carries the Store's identity until the // marker; save it first, idempotently. let own = frame.project.as_deref() == Some(store) && identity::legacy(root, store)?; @@ -594,9 +874,11 @@ pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { layerfs_home::home::private_dir(&identity::store_dir(root, store)?)?; identity::save_record(root, store, &id)?; } + let op = frame.op_id.unwrap_or_else(store_admission::op_id); + let intent = store_admission::project_removal_at(frame.stack, op); let requested = frame.deleted_ms.unwrap_or_else(now_ms); let deletion = service::with_service(&open, |service| { - delete_stack(&service.store, frame.stack, requested) + delete_stack_with(&service.store, frame.stack, requested, &intent) })?; let deleted_ms = deletion.deleted_ms(requested); let project = remove_worker_entry(root, store, frame.stack, frame.project.as_deref())? diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 8de409d3a3..808fbd46e4 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,6 +1,7 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.list` (T4.5c), `project.stage` and -//! `project.export`, each encoded by the caller and +//! `project.remove`, `project.list` (T4.5c), `project.stage`, +//! `project.export`, `project.delete` (T4.5b) and its admission ops, each +//! encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. //! The worker sets the home to its own root; a frame names none. use layerfs_bridge::control::{SdkRefusal, SdkValue}; @@ -108,6 +109,17 @@ fn stack_id(text: &str) -> Result { }) } +/// The operation identity an `op_id` field holds: its 16 bytes in hex. +fn op_id(text: &str) -> Result<[u8; 16]> { + let bytes = unhex(text).ok_or_else(|| refused("InvalidInput", "`op_id` must be hex digits"))?; + bytes.as_slice().try_into().map_err(|_| { + refused( + "InvalidInput", + "`op_id` must be an operation identity (16 bytes)", + ) + }) +} + impl InitFrame { /// The `project.init` frame: `{ store, name, staged, timeoutMs, /// store_record }`, each sent when set. @@ -246,7 +258,9 @@ impl ExportFrame { /// worker, spec §4). The caller writes `deleted_ms` to its durable intent /// before the call and resends it on a retry; `project` and `store_record` /// are the caller context for the worker own entry and its Store open. -#[derive(Clone, Debug)] +/// `op_id` is the admission the caller already took, so the marker reuses it +/// instead of scanning past a second row. +#[derive(Clone, Debug, Eq, PartialEq)] pub struct DeleteFrame { /// The Store. pub store: StoreName, @@ -257,6 +271,37 @@ pub struct DeleteFrame { /// The `deleted_ms` the marker should hold; `None` lets the worker take /// the current time. pub deleted_ms: Option, + /// The admission operation identity the marker reuses, when the caller + /// took one. + pub op_id: Option<[u8; 16]>, + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.admit`: register the caller's project-scope admission +/// before it writes its sidecar and sends the marker (T4.6). The frame is +/// `{store, stack_id, op_id, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteAdmitFrame { + /// The Store. + pub store: StoreName, + /// The LayerStack the caller recorded. + pub stack: LayerStackId, + /// The admission operation identity. + pub op: [u8; 16], + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.release`: delete the caller's admission row after the +/// marker, on every exit path (T4.6). The frame is +/// `{store, op_id, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteReleaseFrame { + /// The Store. + pub store: StoreName, + /// The admission operation identity. + pub op: [u8; 16], /// The Store record. pub store_record: Option, } @@ -275,7 +320,7 @@ pub struct DeleteReply { impl DeleteFrame { /// The `project.delete` frame: `{ store, stack_id, deleted_ms?, project?, - /// store_record? }`, each optional field sent when set. + /// op_id?, store_record? }`, each optional field sent when set. pub fn encode(&self) -> SdkValue { let mut fields = vec![ ("store", SdkValue::text(self.store.as_str())), @@ -287,12 +332,14 @@ impl DeleteFrame { .as_deref() .map(|p| ("project", SdkValue::text(p))), ); + fields.extend(self.op_id.map(|op| ("op_id", SdkValue::text(hex(&op))))); fields.extend(self.store_record.clone().map(|r| ("store_record", r))); map(fields) } - /// A `project.delete` frame: no Store, a bad Store name, no Stack id, or - /// a Stack id that is not a LayerStack identity are refused. + /// A `project.delete` frame: no Store, a bad Store name, no Stack id, a + /// Stack id that is not a LayerStack identity, or an `op_id` that is not + /// an operation identity are refused. pub fn decode(frame: &SdkValue) -> Result { let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; @@ -301,6 +348,59 @@ impl DeleteFrame { project: frame.get_text("project").map(str::to_owned), stack, deleted_ms: frame.get_int("deleted_ms"), + op_id: frame.get_text("op_id").map(op_id).transpose()?, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteAdmitFrame { + /// The `project.delete.admit` frame: `{store, stack_id, op_id, + /// store_record?}`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.admit` frame: its Store, Stack and operation + /// identity are required and checked. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + stack, + op, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteReleaseFrame { + /// The `project.delete.release` frame: `{store, op_id, store_record?}`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.release` frame: its Store and operation identity + /// are required and checked. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + op, store_record: frame.get("store_record").cloned(), }) } diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs new file mode 100644 index 0000000000..a02dc621e6 --- /dev/null +++ b/control/crates/layerfs-store/src/report.rs @@ -0,0 +1,116 @@ +//! The wire rendering of the §11 compaction report and the `store.status` +//! compaction projection (T10). +//! +//! One serializer renders the shared [`CompactReport`], so the SQLite path and +//! T9's PostgreSQL path cannot disagree on a key: every key is snake_case (§10; +//! §11 names are logical), a zero field is named by its wire key, and +//! `residue.reason` appears only when something remains. +use layerfs_bridge::control::SdkValue; +use layerfs_persistence::{CompactReport, CompactStatus}; + +/// The §11 report as `store.compact` returns it: +/// `{kind: "Compacted", path, generation {before, after}, ...}`. +#[must_use] +pub fn compact_report(report: &CompactReport) -> SdkValue { + let int = |value: u64| SdkValue::Int(i64::try_from(value).unwrap_or(i64::MAX)); + let references = report + .kept_waste + .delta_base_references + .iter() + .map(|id| SdkValue::text(layerfs_bridge::sdk_values::hex(id))) + .collect(); + let mut residue = vec![ + ("bytes".to_owned(), int(report.residue.bytes)), + ( + "artifacts".to_owned(), + SdkValue::List( + report + .residue + .artifacts + .iter() + .map(|artifact| SdkValue::text(artifact)) + .collect(), + ), + ), + ]; + if let Some(reason) = &report.residue.reason { + residue.push(("reason".to_owned(), SdkValue::text(reason))); + } + let zero_reasons = report + .zero_reasons + .iter() + .map(|zero| { + SdkValue::map([ + ("field", SdkValue::text(zero.wire_field())), + ("reason", SdkValue::text(zero.reason)), + ]) + }) + .collect(); + SdkValue::Map( + [ + ("kind", SdkValue::text("Compacted")), + ("path", SdkValue::text(report.path.as_str())), + ( + "generation", + SdkValue::map([ + ("before", int(report.generation.before)), + ("after", int(report.generation.after)), + ]), + ), + ("dead_bytes_bound", int(report.dead_bytes_bound)), + ("dead_bytes_refined", int(report.dead_bytes_refined)), + ("logical_discarded", int(report.logical_discarded)), + ("backend_reusable", int(report.backend_reusable)), + ("physical_shrink", int(report.physical_shrink)), + ( + "kept_waste", + SdkValue::map([ + ( + "dead_in_retained_packs_bytes", + int(report.kept_waste.dead_in_retained_packs_bytes), + ), + ( + "delta_bases", + SdkValue::map([ + ("bytes", int(report.kept_waste.delta_bases_bytes)), + ("references", SdkValue::List(references)), + ]), + ), + ("inode_blocks_bytes", int(report.kept_waste.inode_blocks_bytes)), + ]), + ), + ("peak_temp_bytes", int(report.peak_temp_bytes)), + ("residue", SdkValue::Map(residue.into_iter().collect())), + ("estimate", SdkValue::Bool(report.estimate)), + ("zero_reasons", SdkValue::List(zero_reasons)), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The `store.status` projection of a compaction: `{id, name, phase, started, +/// uncertain, state, phases}`. The caller splits the op fields into its `op` +/// and leaves `state`/`phases` beside them (§11). +#[must_use] +pub fn compact_status(status: &CompactStatus) -> SdkValue { + SdkValue::map([ + ("id", SdkValue::text(&status.id)), + ("name", SdkValue::text(status.name)), + ("phase", SdkValue::text(&status.phase)), + ("started", SdkValue::Int(status.started)), + ("uncertain", SdkValue::Bool(status.uncertain)), + ("state", SdkValue::text(status.state)), + ( + "phases", + SdkValue::List( + status + .phases + .iter() + .map(|phase| SdkValue::text(*phase)) + .collect(), + ), + ), + ]) +} diff --git a/control/crates/layerfs-store/src/sqlite/transfer/step.rs b/control/crates/layerfs-store/src/sqlite/transfer/step.rs index 50dca83651..98dc7021d0 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/step.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/step.rs @@ -12,6 +12,11 @@ type Result = std::result::Result; pub enum StoreStep { /// Whether the file is there, its size and the free bytes. Stat, + /// The compaction operation recorded beside the Store, for `store.status`. + /// + /// It reads only `.compact-op`; never the Store database, an + /// artifact or a directory listing. + CompactOp, /// The idle Store's bytes, sha256 and heads. Snapshot, /// A consistent snapshot of the live Store, staged beside it. @@ -68,6 +73,7 @@ impl StoreStep { pub fn name(&self) -> &'static str { match self { Self::Stat => "stat", + Self::CompactOp => "compact-op", Self::Snapshot => "snapshot", Self::Stage => "stage", Self::Unstage => "unstage", @@ -161,6 +167,7 @@ impl StepFrame { layerfs_home::names::StoreName::parse(store)?; let step = match op { "stat" => StoreStep::Stat, + "compact-op" => StoreStep::CompactOp, "snapshot" => StoreStep::Snapshot, "stage" => StoreStep::Stage, "unstage" => StoreStep::Unstage, diff --git a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs index ef250836ef..fdca5f159a 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs @@ -25,6 +25,16 @@ fn with_suffix(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(name) } +/// Whether the process a compaction record names still runs on this machine. +/// The kernel start time and boot id, when recorded, keep a reused id from +/// passing as the owner. +fn owner_alive(op: &layerfs_persistence::CompactOpStatus) -> bool { + let Some(pid) = op.owner_pid else { + return false; + }; + layerfs_home::process::alive_as(pid, op.owner_pid_start, op.owner_pid_boot) +} + fn length(path: &Path) -> Result { match std::fs::metadata(path) { Ok(meta) => Ok(meta.len()), @@ -124,6 +134,12 @@ pub fn run_observed(root: &Path, frame: &StepFrame, progress: &mut dyn FnMut(u64 ), ("incoming_bytes", SdkValue::Int(length(&incoming)? as i64)), ])), + // the compaction record beside the Store only: `store.status` shows + // its op and phase without opening the Store database or an artifact + StoreStep::CompactOp => match layerfs_persistence::compact_op_status(&file) { + None => Ok(SdkValue::Null), + Some(op) => Ok(crate::report::compact_status(&op.status(owner_alive(&op)))), + }, // snapshot: the idle Store's bytes, sha256 and history heads. A Store // nobody holds has no write-ahead log left unless a process crashed; // if a crashed opener left one, checkpoint and close it first. diff --git a/control/crates/layerfs-store/src/status.rs b/control/crates/layerfs-store/src/status.rs index e62157e506..5656f73eb7 100644 --- a/control/crates/layerfs-store/src/status.rs +++ b/control/crates/layerfs-store/src/status.rs @@ -8,31 +8,65 @@ use layerfs_bridge::control::SdkValue; use layerfs_home::outcome::Outcome; use std::path::Path; -/// `store.status`: who holds it, the operation its record holds (`op`, as -/// H6 shows it), its size. It starts no machine: a Store whose machine is -/// not running is `down`, with the reason (O34). It changes nothing (#536 -/// §12 item 3): no lock, and a stale lease is ignored, not removed. +/// `store.status`: who holds it, the operation on it (`op`, as H6 shows it), +/// its size. It starts no machine: a Store whose machine is not running is +/// `down`, with the reason (O34). It changes nothing (#536 §12 item 3): no +/// lock, and a stale lease is ignored, not removed. +/// +/// While a compaction runs, or after a dead owner left one, `op` is that run's +/// projection and `state` is `busy`/`unresolved` (§11); `phases` names the +/// backend's ordered phases. The compaction record is read from +/// `.compact-op` alone: the Store database and its artifacts are never +/// opened for it. pub fn status(home: &Path, name: &str) -> Outcome { let record = record_of(home, name)?; let (holders, _) = lease::read_holders(home, name)?; let mut down = None; let mut more = Vec::new(); let store_ref = record.store_ref()?; + let mut compaction = SdkValue::Null; if let StoreRef::Postgres { .. } = store_ref { more.push(("server", crate::postgresql::connect::status(&record))); + // A PostgreSQL compaction's op lives in the Store's `gc_op`; T9 reads + // it here with its rebuild. This path reads no file. } else { more.push(("machine", SdkValue::text(store_ref.machine()?))); - // the file's size where it lives, starting nothing (P4); a machine - // that is down says why - let file = place::plan(home, store_ref) - .and_then(|mut executor| step_on(home, &mut executor, name, StoreStep::Stat)); + // the file's size where it lives, starting nothing (P4), and the + // compaction record beside it; a machine that is down says why + let mut placed = place::plan(home, store_ref); + let (file, op) = match placed.as_mut() { + Ok(executor) => ( + step_on(home, executor, name, StoreStep::Stat), + step_on(home, executor, name, StoreStep::CompactOp).unwrap_or_default(), + ), + Err(why) => (Err(why.clone()), SdkValue::Null), + }; + compaction = op; let file = file.unwrap_or_else(|why| { down = Some(why.message.clone()); SdkValue::map([("unavailable", SdkValue::Text(why.message))]) }); more.push(("file", file)); } - let state = record.state(down.is_some(), &holders).as_str(); + // A compaction's record is the operation the Store is running, even when + // some other process holds its lease; otherwise the record's own `op` + // (a push, pull or remove) is what `store.status` shows. + let op = if compaction.get("id").is_some() { + let field = |field: &str| compaction.get(field).cloned().unwrap_or_default(); + SdkValue::map([ + ("id", field("id")), + ("name", field("name")), + ("phase", field("phase")), + ("started", field("started")), + ("uncertain", field("uncertain")), + ]) + } else { + record.op().map(|op| op.to_sdk()).unwrap_or_default() + }; + let phases = compaction.get("phases").cloned().unwrap_or_default(); + let state = compaction + .get_text("state") + .unwrap_or_else(|| record.state(down.is_some(), &holders).as_str()); let mut fields = vec![ ("name", SdkValue::text(name)), ("kind", SdkValue::text(store_ref.kind().as_str())), @@ -46,7 +80,8 @@ pub fn status(home: &Path, name: &str) -> Outcome { .collect(), ), ), - ("op", record.op().map(|op| op.to_sdk()).unwrap_or_default()), + ("op", op), + ("phases", phases), ("down_reason", down.map(SdkValue::Text).unwrap_or_default()), ]; fields.extend(more); diff --git a/control/crates/layerfs-store/tests/compact_report_status.rs b/control/crates/layerfs-store/tests/compact_report_status.rs new file mode 100644 index 0000000000..88aa9e03da --- /dev/null +++ b/control/crates/layerfs-store/tests/compact_report_status.rs @@ -0,0 +1,212 @@ +//! T10: the §11 report wire shape and the `store.status` compaction op +//! projection. The report renders snake_case from the shared `CompactReport`; +//! status reads only the `.compact-op` sidecar and never opens the +//! Store database. +use layerfs_bridge::control::SdkValue; +use layerfs_home::home::{paths, private_dir, write_entity}; +use layerfs_home::names::StoreName; +use layerfs_persistence::{ + CompactKeptWaste, CompactPath, CompactReceipt, CompactReport, CompactResidue, CompactZeroReason, +}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t10-report-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + let SdkValue::List(items) = value else { + return Vec::new(); + }; + items + .iter() + .filter_map(|item| match item { + SdkValue::Text(text) => Some(text.clone()), + _ => None, + }) + .collect() +} + +/// A home holding one empty sqlite Store `s` on native. +fn store_home() -> PathBuf { + let home = tempdir("home"); + let record = SdkValue::map([ + ("name", SdkValue::text("s")), + ("kind", SdkValue::text("sqlite")), + ("machine", SdkValue::text("native")), + ]); + write_entity(&paths::store_record(&home, "s"), &record).unwrap(); + private_dir(&paths::store_dir(&home, "s")).unwrap(); + std::fs::write(paths::store_file(&home, "s"), b"").unwrap(); + home +} + +/// A complete `.compact-op` record, as the persistence layer writes it. +fn sidecar(home: &Path, phase: &str, owner: u32) -> PathBuf { + let mut path = paths::store_file(home, "s").into_os_string(); + path.push(".compact-op"); + let path = PathBuf::from(path); + let zeros = "00".repeat(32); + std::fs::write( + &path, + format!( + "version=1\nop_id=00112233445566778899aabbccddeeff\ngeneration=3\n\ +phase={phase}\nstarted_ms=1234\nowner_pid={owner}\nowner_pid_start=\nowner_pid_boot=\n\ +catalog_id=0102030405060708090a0b0c0d0e0f10\nnext_history_serial=5\nnext_pack_id=6\n\ +next_ordinal=7\nmark_serial=4\nmark_pack_id=6\nmark_digest={zeros}\nmark_totals=1,2,3,4\n\ +mark_stamp=9\nhistory_rows=10\nlive_heads_digest={zeros}\n" + ), + ) + .unwrap(); + path +} + +fn receipt() -> CompactReceipt { + CompactReceipt { + path: CompactPath::Rebuild, + generation_before: 3, + generation_after: 4, + dead_bytes_bound: 100, + dead_bytes_refined: 80, + logical_discarded: 40, + backend_reusable: 4096, + physical_shrink: 8192, + kept_waste: CompactKeptWaste { + dead_in_retained_packs_bytes: 5, + delta_bases_bytes: 6, + delta_base_references: vec![[7; 32]], + inode_blocks_bytes: 8, + }, + peak_temp_bytes: 900, + residue: CompactResidue { + bytes: 0, + artifacts: Vec::new(), + reason: None, + }, + estimate: true, + zero_reasons: vec![ + CompactZeroReason { + field: "deadBytesBound", + reason: "no-waste", + }, + CompactZeroReason { + field: "logicalDiscarded", + reason: "no-history", + }, + ], + } +} + +#[test] +fn the_report_wire_is_the_snake_case_contract_shape() { + let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt())); + assert_eq!(text(wire.get("kind").unwrap()), "Compacted"); + assert_eq!(text(wire.get("path").unwrap()), "rebuild"); + let generation = wire.get("generation").unwrap(); + assert_eq!(generation.get_int("before"), Some(3)); + assert_eq!(generation.get_int("after"), Some(4)); + assert_eq!(wire.get_int("dead_bytes_bound"), Some(100)); + assert_eq!(wire.get_int("dead_bytes_refined"), Some(80)); + assert_eq!(wire.get_int("logical_discarded"), Some(40)); + assert_eq!(wire.get_int("backend_reusable"), Some(4096)); + assert_eq!(wire.get_int("physical_shrink"), Some(8192)); + assert_eq!(wire.get_int("peak_temp_bytes"), Some(900)); + assert_eq!(wire.get("estimate"), Some(&SdkValue::Bool(true))); + let waste = wire.get("kept_waste").unwrap(); + assert_eq!(waste.get_int("dead_in_retained_packs_bytes"), Some(5)); + assert_eq!(waste.get_int("inode_blocks_bytes"), Some(8)); + let bases = waste.get("delta_bases").unwrap(); + assert_eq!(bases.get_int("bytes"), Some(6)); + assert_eq!( + texts(bases.get("references").unwrap()), + vec!["0707070707070707070707070707070707070707070707070707070707070707"] + ); + let residue = wire.get("residue").unwrap(); + assert_eq!(residue.get_int("bytes"), Some(0)); + assert!(texts(residue.get("artifacts").unwrap()).is_empty()); + assert!(residue.get("reason").is_none()); + // One reason per zero field, named by its snake_case wire key (§10). + let SdkValue::List(zero) = wire.get("zero_reasons").unwrap() else { + panic!("zero_reasons is a list"); + }; + let fields: Vec<&str> = zero + .iter() + .map(|zero| text(zero.get("field").unwrap())) + .collect(); + assert_eq!(fields, ["dead_bytes_bound", "logical_discarded"]); + assert_eq!( + text(zero[0].get("reason").unwrap()), + "no-waste", + "the closed vocabulary passes through" + ); + assert!(wire.get("zeroReason").is_none(), "no camelCase key"); +} + +#[test] +fn status_projects_the_sidecar_phase_and_state() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + + // No op: the existing idle shape, no phases. + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + + // A live owner is busy and certain; `build` is the sidecar's phase. + let op_path = sidecar(&home, "build", std::process::id()); + let busy = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + assert_eq!(text(op.get("name").unwrap()), "store.compact"); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get_int("started"), Some(1234)); + // The record remains unsettled while it runs; `state` is what tells a + // live owner from a leftover. + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + texts(busy.get("phases").unwrap()), + ["mark", "build", "verify", "install"] + ); + + // A gone owner leaves an uncertain op: unresolved, same phase, same id. + sidecar(&home, "build", 99_999_999); + let leftover = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + + // A settled run removes the record: the idle shape is back. + std::fs::remove_file(op_path).unwrap(); + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-store/tests/project_delete.rs b/control/crates/layerfs-store/tests/project_delete.rs index 43a30eab48..727bcb981b 100644 --- a/control/crates/layerfs-store/tests/project_delete.rs +++ b/control/crates/layerfs-store/tests/project_delete.rs @@ -9,7 +9,7 @@ use layerfs_history::LayerStackId; use layerfs_home::home::{paths, write_record}; use layerfs_home::outcome::{failed, refused}; use layerfs_store::project::delete::{ - delete_with, remove_worker_entry, Effects, Intent, IntentState, + delete_with, remove_worker_entry, Admission, Effects, Intent, IntentState, }; use layerfs_store_service::history::deletion::Deletion; use std::collections::VecDeque; @@ -54,11 +54,14 @@ fn intent(_home: &Path, store: &str, project: &str, stack: LayerStackId, id: i64 deleted_ms: id, attempt_ms: 1, state: IntentState::Pending, + op: None, } } -/// A fake Store side and entry remover. `mark` reads the intent as the -/// marker would see it, so a test can see the state that preceded it. +/// A fake home/worker: the Store side and entry remover, with the project +/// scope admission a live mount or operation would refuse. `mark` reads the +/// intent as the marker would see it, so a test can see the state that +/// preceded it. struct Fake { home: PathBuf, store: String, @@ -68,6 +71,14 @@ struct Fake { states: Vec, prepared: usize, fail_removal: bool, + /// What `admit` refuses with, before registering anything. + refuse: Option, + /// What `mark` refuses with, before writing anything. + mark_refusal: Option, + /// Admissions taken, in order. + admitted: Vec, + /// Admissions released, in order. + released: Vec, } impl Fake { @@ -81,6 +92,10 @@ impl Fake { states: Vec::new(), prepared: 0, fail_removal: false, + refuse: None, + mark_refusal: None, + admitted: Vec::new(), + released: Vec::new(), } } @@ -88,6 +103,22 @@ impl Fake { self.fail_removal = true; self } + + /// A Store with a live mount (or another live operation) at the project + /// scope: `admit` refuses with the same code the Store's admission does. + fn refusing(mut self, code: &str, detail: SdkValue) -> Self { + self.refuse = Some(SdkRefusal { + code: code.to_owned(), + message: format!("test admission refusal: {code}"), + detail, + }); + self + } + + /// A fake whose admission refuses. + fn refused(&self) -> bool { + self.refuse.is_some() + } } impl Effects for Fake { @@ -96,8 +127,29 @@ impl Effects for Fake { Ok(()) } - fn mark(&mut self, stack: LayerStackId, deleted_ms: i64) -> Result { - self.seen.push((stack, deleted_ms)); + fn admit( + &mut self, + stack: LayerStackId, + reuse: Option<[u8; 16]>, + ) -> Result { + if self.refused() { + return Err(self.refuse.clone().expect("refused above")); + } + let admission = Admission::new(stack, reuse.unwrap_or([0x5a; 16])); + self.admitted.push(admission); + Ok(admission) + } + + fn release(&mut self, admission: Admission) -> Result<(), SdkRefusal> { + self.released.push(admission); + Ok(()) + } + + fn mark(&mut self, admission: Admission, deleted_ms: i64) -> Result { + if let Some(refusal) = &self.mark_refusal { + return Err(refusal.clone()); + } + self.seen.push((admission.stack, deleted_ms)); let state = Intent::read(&self.home, &self.store, &self.project) .unwrap() .expect("the intent is durable before the marker") @@ -142,11 +194,104 @@ fn a_fresh_delete_marks_then_removes_the_entry_and_unlinks_the_intent() { // the identity save precedes the intent, and the intent precedes the mark assert_eq!(fake.prepared, 1); assert_eq!(fake.states, [IntentState::Pending]); + // the project scope was admitted before the marker and released after + assert_eq!(fake.admitted.len(), 1); + assert_eq!(fake.released, fake.admitted); assert!(!paths::project_dir(&home, store, project).exists()); assert_eq!(Intent::read(&home, store, project).unwrap(), None); let _ = std::fs::remove_dir_all(&home); } +#[test] +fn a_refused_admission_writes_no_sidecar_and_leaves_no_intent() { + let refused_home = home("refused"); + let (store, project, stack) = ("shop", "app", stack(20)); + project_record(&refused_home, store, project, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", t("mount")), + ("entity", t("main")), + ])]), + )]); + let mut fake = + Fake::new(&refused_home, store, project, Vec::new()).refusing("MountsActive", holders); + let refusal = delete_with(&refused_home, store, project, Some(stack), &mut fake).unwrap_err(); + // the live mount's refusal is the call's, with its holders + assert_eq!(refusal.code, "MountsActive"); + assert!(refusal.detail.get("holders").is_some()); + // no identity save, no sidecar, no marker + assert_eq!(fake.prepared, 0); + assert!(fake.seen.is_empty(), "{:?}", fake.seen); + assert_eq!(Intent::read(&refused_home, store, project).unwrap(), None); + // the admission refused, so there is no admission to release + assert!(fake.admitted.is_empty()); + assert!(fake.released.is_empty()); + // the home entry and the project's record stay + assert!(paths::project_dir(&refused_home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&refused_home); + + // a live operation at the project scope is `Busy`, with the same rule + let busy_home = home("busy"); + project_record(&busy_home, store, project, stack); + let conflict = SdkValue::map([ + ("kind", t("removal")), + ("entity", t("app")), + ("phase", SdkValue::Null), + ("op", t("11")), + ("uncertain", SdkValue::Bool(false)), + ]); + let mut fake = Fake::new(&busy_home, store, project, Vec::new()).refusing("Busy", conflict); + let refusal = delete_with(&busy_home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + assert_eq!(fake.prepared, 0); + assert!(fake.seen.is_empty()); + assert_eq!(Intent::read(&busy_home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&busy_home); +} + +#[test] +fn a_retry_reuses_the_sidecar_admission_identity() { + let home = home("reuse"); + let (store, project, stack) = ("shop", "app", stack(22)); + project_record(&home, store, project, stack); + let mut pending = intent(&home, store, project, stack, 111); + pending.op = Some([3; 16]); + Intent::write(&home, &pending).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 111 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + // the retry re-registers the recorded identity, then releases it + assert_eq!(fake.admitted, [Admission::new(stack, [3; 16])]); + assert_eq!(fake.released, fake.admitted); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_failed_marker_releases_the_admission_it_took() { + let home = home("marker-refused"); + let (store, project, stack) = ("shop", "app", stack(21)); + project_record(&home, store, project, stack); + let mut fake = Fake::new(&home, store, project, Vec::new()); + fake.mark_refusal = Some(refused("Busy", "the project is held")); + let refusal = delete_with(&home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + // the admission taken before the sidecar is released on the failure + assert_eq!(fake.admitted.len(), 1); + assert_eq!(fake.released, fake.admitted); + // the sidecar stays `pending` for the retry, and the entry stays + let left = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!(left.state, IntentState::Pending); + assert!(paths::project_dir(&home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&home); +} + #[test] fn a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry() { let home = home("retired"); @@ -350,9 +495,11 @@ fn the_intent_moves_pending_marked_done_and_unlinks() { value.state = IntentState::Marked; value.deleted_ms = 6; + value.op = Some([4; 16]); Intent::write(&home, &value).unwrap(); let read = Intent::read(&home, store, project).unwrap().unwrap(); assert_eq!((read.state, read.deleted_ms), (IntentState::Marked, 6)); + assert_eq!(read.op, Some([4; 16]), "the admission identity persists"); value.state = IntentState::Done; Intent::write(&home, &value).unwrap(); diff --git a/control/crates/layerfs-store/tests/project_delete_frames.rs b/control/crates/layerfs-store/tests/project_delete_frames.rs index 6d5b05e429..87b7178990 100644 --- a/control/crates/layerfs-store/tests/project_delete_frames.rs +++ b/control/crates/layerfs-store/tests/project_delete_frames.rs @@ -4,7 +4,9 @@ use layerfs_bridge::control::SdkValue; use layerfs_bridge::sdk_values::hex; use layerfs_history::LayerStackId; use layerfs_home::names::StoreName; -use layerfs_store::project::frame::{DeleteFrame, DeleteReply}; +use layerfs_store::project::frame::{ + DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, DeleteReply, +}; fn t(text: &str) -> SdkValue { SdkValue::text(text) @@ -21,12 +23,14 @@ fn stack(body: u8) -> LayerStackId { #[test] fn the_frame_names_the_store_stack_and_optional_context() { let stack = stack(1); + let op = [7u8; 16]; let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); let frame = DeleteFrame { store: StoreName::parse("shop").unwrap(), project: Some("app".to_owned()), stack, deleted_ms: Some(5), + op_id: Some(op), store_record: Some(record.clone()), }; let encoded = frame.encode(); @@ -37,6 +41,7 @@ fn the_frame_names_the_store_stack_and_optional_context() { ("stack_id", t(&hex(&stack.to_bytes()))), ("deleted_ms", SdkValue::Int(5)), ("project", t("app")), + ("op_id", t(&hex(&op))), ("store_record", record), ]) ); @@ -45,6 +50,7 @@ fn the_frame_names_the_store_stack_and_optional_context() { assert_eq!(decoded.project, frame.project); assert_eq!(decoded.stack, stack); assert_eq!(decoded.deleted_ms, Some(5)); + assert_eq!(decoded.op_id, Some(op)); assert_eq!(decoded.store_record, frame.store_record); } @@ -57,10 +63,77 @@ fn a_frame_with_only_the_store_and_stack_decodes() { ]); let decoded = DeleteFrame::decode(&minimal).unwrap(); assert_eq!((decoded.project, decoded.deleted_ms), (None, None)); + assert_eq!(decoded.op_id, None); assert_eq!(decoded.store_record, None); assert_eq!(decoded.stack, stack); } +#[test] +fn the_admission_and_release_frames_round_trip() { + let stack = stack(4); + let op = [9u8; 16]; + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let admit = DeleteAdmitFrame { + store: StoreName::parse("shop").unwrap(), + stack, + op, + store_record: Some(record.clone()), + }; + let encoded = admit.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t(&hex(&op))), + ("store_record", record.clone()), + ]) + ); + assert_eq!(DeleteAdmitFrame::decode(&encoded).unwrap(), admit); + + let release = DeleteReleaseFrame { + store: StoreName::parse("shop").unwrap(), + op, + store_record: None, + }; + let encoded = release.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("op_id", t(&hex(&op)))]) + ); + assert_eq!(DeleteReleaseFrame::decode(&encoded).unwrap(), release); +} + +#[test] +fn the_admission_frames_refuse_a_bad_operation_identity() { + let decode = |fields| DeleteAdmitFrame::decode(&map(fields)).unwrap_err().message; + let stack = stack(5); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("zz")), + ]), + "`op_id` must be hex digits" + ); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("ab")), + ]), + "`op_id` must be an operation identity (16 bytes)" + ); + assert_eq!( + delete_release_decode(&map(vec![("store", t("shop"))])), + "missing text argument `op_id`" + ); +} + +fn delete_release_decode(frame: &SdkValue) -> String { + DeleteReleaseFrame::decode(frame).unwrap_err().message +} + #[test] fn the_worker_refuses_a_bad_frame_in_order() { let decode = |fields| DeleteFrame::decode(&map(fields)).unwrap_err().message; diff --git a/control/crates/layerfs-store/tests/project_delete_store.rs b/control/crates/layerfs-store/tests/project_delete_store.rs new file mode 100644 index 0000000000..1d0e945a70 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete_store.rs @@ -0,0 +1,238 @@ +//! `project.delete`'s home path against a real local Store (T4.6): the +//! control layer admits the project scope through `gc_register`/`gc_release` +//! before its sidecar and marker, so a live mount refuses with +//! `MountsActive` (its holders), a live operation with `Busy`, and a refusal +//! leaves no sidecar, no marker and no leaked admission intent. +use layerfs_bridge::control::SdkValue; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{layout, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::project::delete::{delete, Intent}; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store_service::history::lookup; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layout::create(&directory).unwrap(); + directory +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// True when the sandbox filesystem cannot hold a directly opened Store, so +/// these real-Store cases cannot run here. The fake-home cases in +/// `project_delete` cover the same home path without a Store. +fn no_direct_store(refusal: &layerfs_bridge::control::SdkRefusal) -> bool { + refusal.code == "Unsupported" && refusal.message.contains("does not carry SQLite locks") +} + +/// A home whose Store `shop` holds the live project `app`, and the +/// `OpenArgs` (with the Store record) that reach it. `None` when this +/// filesystem cannot hold a directly opened Store. +fn shop(tag: &str) -> Option<(PathBuf, OpenArgs, LayerStackId)> { + let home = tempdir(tag); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) if no_direct_store(&refusal) => { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return None; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir(tag); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: None, + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = with_service(&open, |service| { + Ok(lookup::project(&service.store, "app").unwrap().id) + }) + .unwrap(); + Some((home, open, stack)) +} + +/// `project.delete shop/app` through the control-layer home path. +fn remove(home: &std::path::Path) -> Result { + let mut sink = |_| {}; + delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + ) +} + +/// The live intents at the project scope. +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +/// Registers a live mount of the project's `main` Branch, as a Workspace +/// bind does before it demands the root. +fn mount_main(open: &OpenArgs, stack: LayerStackId, op: [u8; 16]) { + // `lookup::branch` refuses `main` by design; the snapshot lookup names it + let branch = with_service(open, |service| { + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + match service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + { + NamedSnapshot::Found(snapshot) => Ok(snapshot.branch.id), + other => panic!("no main Branch: {other:?}"), + } + }) + .unwrap(); + let intent = GcIntent::shared( + op, + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(open, |service| { + service.store.history().gc_register(&intent).unwrap(); + Ok(()) + }) + .unwrap(); +} + +fn release(open: &OpenArgs, op: [u8; 16]) { + with_service(open, |service| { + service.store.history().gc_release(&op).unwrap(); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn a_live_mount_refuses_the_delete_before_the_sidecar_and_leaves_no_intent() { + let Some((home, open, stack)) = shop("mount") else { + return; + }; + mount_main(&open, stack, [9; 16]); + assert!(intents(&open, stack).is_empty(), "no project-scope row yet"); + + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + + // no sidecar pending, no marker and no leaked admission intent + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert!( + intents(&open, stack).is_empty(), + "the refused admission left no project-scope row" + ); + with_service(&open, |service| { + assert_eq!( + lookup::project(&service.store, "app") + .expect("still live") + .id, + stack + ); + Ok(()) + }) + .unwrap(); + + // the mount goes and the same home path succeeds, releasing its admission + release(&open, [9; 16]); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_text("project"), Some("shop/app")); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!( + intents(&open, stack).is_empty(), + "the admission was released" + ); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_live_operation_at_the_project_scope_is_busy_and_writes_no_sidecar() { + let Some((home, open, stack)) = shop("busy") else { + return; + }; + assert!(intents(&open, stack).is_empty()); + let holder = GcIntent::operation( + [7; 16], + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("app"), None); + with_service(&open, |service| { + service.store.history().gc_register(&holder).unwrap(); + Ok(()) + }) + .unwrap(); + + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert_eq!(intents(&open, stack).len(), 1, "only the test's own holder"); + assert_eq!(intents(&open, stack)[0].op_id, [7; 16]); + + // the operation ends: the delete proceeds and releases its own admission + release(&open, [7; 16]); + assert!(remove(&home).unwrap().get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!(intents(&open, stack).is_empty()); + let _ = std::fs::remove_dir_all(&home); +} diff --git a/core/crates/layerfs-history/src/contract/error.rs b/core/crates/layerfs-history/src/contract/error.rs index 940259da6e..32ab59b264 100644 --- a/core/crates/layerfs-history/src/contract/error.rs +++ b/core/crates/layerfs-history/src/contract/error.rs @@ -148,6 +148,8 @@ pub enum HistoryError { NameTaken(Box), /// A path is hidden by `remove_file`. Removed(Box), + /// A staged name or artifact already exists and no record names it. + Exists(&'static str), /// A required marker or fence row is missing or contradicts the request. Conflict(&'static str), /// The Store was replaced or moved under this handle. @@ -217,6 +219,7 @@ impl std::fmt::Display for HistoryError { Self::Unsupported(what) => write!(formatter, "unsupported: {what}"), Self::NameTaken(state) => write!(formatter, "name taken: {:?}", state.holder), Self::Removed(state) => write!(formatter, "removed: {} paths", state.paths.len()), + Self::Exists(what) => write!(formatter, "exists: {what}"), Self::Conflict(what) => write!(formatter, "conflict: {what}"), Self::StaleStore => formatter.write_str("stale Store"), Self::Busy => formatter.write_str("busy"), diff --git a/core/docs/issues/project-41/LayerFS-GC-spec.md b/core/docs/issues/project-41/LayerFS-GC-spec.md index 3cd2a9a942..1a8baa36cd 100644 --- a/core/docs/issues/project-41/LayerFS-GC-spec.md +++ b/core/docs/issues/project-41/LayerFS-GC-spec.md @@ -161,10 +161,13 @@ and returns `home_entry_removed: true`. The durable delete intent `{stack_id, deleted_ms, attempt_ms}` lives outside the removed entry (or the entry is **not** renamed; one sidecar file at `/store//delete/.json` with -`{store, project, stack_id, deleted_ms, attempt_ms, state}` is written +`{store, project, stack_id, deleted_ms, attempt_ms, state, op_id}` is written (temp + fsync + rename + dir fsync) after the identity save and before the marker, with `pending|marked|done` updated atomically and unlinked after the -entry removal; the retry +entry removal. `op_id` is additive (an older sidecar without it is still +valid): it records the project op the admit/mark attempt ran under, so the +retry can settle a lost admit/mark reply instead of leaving it uncertain +(§5.1, §10). The retry re-reads the Store by the recorded stack id and compares it with the home record, discarding a mismatch without touching a re-created project's entry; a retired row reuses `deleted_ms` and finishes the entry; a live row with no @@ -289,7 +292,8 @@ caller's record by stack id and allow-lists its retired stack. same position: the marker hides the path and a mount of the Branch that carries it whiteouts, so publish needs no whiteout of its own. A merge that cannot materialize the whiteout is refused `Removed` with - `detail {paths: [...]}`. + `detail {paths: [{path, frontier_serial}]}`, listing every unmaterializable + path as a full element, deduplicated and sorted by normalized path bytes. - A path explicitly re-created after the frontier is not a whiteout: the marker hides only content inherited from nodes at or below the frontier. In a merge of a head at or below the frontier with a head whose path was @@ -317,9 +321,11 @@ caller's record by stack id and allow-lists its retired stack. `NotFound` once compaction collects it, never different bytes. - The result is `{kind: "Removed", project, path, frontier_serial, removed_ms}` or `{kind: "UpToDate", project, path, frontier_serial, - removed_ms}` (the stored values). Every refusal carries one detail shape: - `{paths: [{path, frontier_serial}]}` (one element for read/stat; the - governing maximum frontier and the request path). `file.read`/`file.stat` + removed_ms}` (the stored values). A `Removed` refusal always carries the one + detail shape `{paths: [{path, frontier_serial}]}`: one element for + read/stat (the request path with its governing maximum frontier); a merge + lists every unmaterializable path, sorted by normalized path bytes. + `file.read`/`file.stat` refuse `Removed` with it. `file.list`/`file.glob` omit hidden entries and report one top-level `removed_count` over the queried subtree (a hidden target path is `Removed`; a marked-but-absent path is not counted). @@ -336,11 +342,11 @@ caller's record by stack id and allow-lists its retired stack. hidden at-or-below version (never skips past it), returning only visible versions. `file.blame` stops at the hidden version and attributes the remaining lines at the tip. `file.grep`/`file.blame` omit hidden - hits. `project.export` omits it; a cp-from source is `Removed`; `file.edit` - is `Removed`; `file.mv` from a hidden path is `NotFound` and to a hidden - path is a visible re-creation; `mkdir`/`cp`/`apply` follow the same - construction rule; `file.rm` of a hidden path is `NotFound`; a - post-frontier write is the visible re-creation. + hits. `project.export` omits it; `file.cp` from a hidden source and + `file.edit` are `Removed`; `file.mv`/`file.rm` from a hidden path return the + `NotFound`/`NoSuchPath` wire code, not `Removed`, and `file.mv` to a hidden + path is a visible re-creation; `mkdir`/`apply` follow the same + construction rule; a post-frontier write is the visible re-creation. - The call accepts an absolute POSIX path and converts it to the Store's normalized relative components (strip one leading `/`; `/a` and `a` are the same). It refuses `InvalidInput` for empty, `/`, `//a`, `a//b`, `a/`, `./a`, @@ -973,8 +979,9 @@ reported as `markBytes`; the Phase A table copy itself moves no pack bytes. | Code | Status | |---|---| | `MountsActive`, `Busy`, `Unresolved`, `NotFound`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NoSuchStore`, `Unsupported`, `ReservedName`, `Conflict`, `NameTaken`, `Capacity`, `Exists`, `InvalidInput` | existing (`Unresolved` now used by the four calls) | +| `Unknown` | service/control-plane outcome, not a `HistoryError` code (§4, §4.2): a lost reply or an undeterminable Store listing; detail `{entity: {kind, name}, phase, op}`, with `phase`/`op` `null` when no durable op is known | | `StaleStore` | new; one `HistoryError` variant with an explicit mapping arm, because the `_ => "Failed"` wildcard (`engine/crates/layerfs-store-service/src/history/lookup.rs:54,64`) would hide it | -| `Removed` | new; reading a path removed by `remove_file`; an explicit `HistoryError::Removed` arm in `lookup.rs` before the `_ => "Failed"` wildcard; detail `{paths: [{path, frontier_serial}]}` | +| `Removed` | new; reading a path removed by `remove_file`; an explicit `HistoryError::Removed` arm in `lookup.rs` before the `_ => "Failed"` wildcard; detail always `{paths: [{path, frontier_serial}]}` — a merge lists every unmaterializable path, sorted by path bytes; `file.mv`/`file.rm` from a hidden path are the `NotFound`/`NoSuchPath` wire code, while `file.cp`/`file.edit` return `Removed` | - `project.init` whose name is held returns `NameTaken` (today it returns @@ -1059,8 +1066,10 @@ inodeBlocksBytes}`; `residue` is `{bytes, artifacts: [string], reason?}`; top-level `estimate: bool`; a capped field is `{value, cap}`; `zeroReason` is per zero field with the closed vocabulary `empty-store`, `none-path`, `no-history`, `no-waste`, `postgresql-in-place`; `markBytes` is a #52 metric, -not a result field. The SDK/CLI render every key -snake_case. On the `none` path `deadBytesRefined` equals `deadBytesBound` (no +not a result field. The names in this section (e.g. `deadBytesBound`, +`keptWaste.deadInRetainedPacksBytes`) are logical Rust names only; the wire, +SDK and CLI keys are snake_case (`dead_bytes_bound`, +`kept_waste.dead_in_retained_packs_bytes`), per §10. On the `none` path `deadBytesRefined` equals `deadBytesBound` (no payload sweep runs below the threshold), and the report says so. `physicalShrink` reads the old size before the rename; `backendReusable` reads the SQLite free-page count × page size before it; `logicalDiscarded` is the summed diff --git a/core/docs/issues/project-41/gc-t10-report-status.md b/core/docs/issues/project-41/gc-t10-report-status.md new file mode 100644 index 0000000000..48df05d38d --- /dev/null +++ b/core/docs/issues/project-41/gc-t10-report-status.md @@ -0,0 +1,96 @@ +# T10: the §11 report and `store.status` + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §10/§11. The spec wins; `§N` cites it. No contract is added here. + +## Placement + +- `engine/crates/layerfs-persistence/src/report.rs` — the backend-neutral + `CompactReport` (with `CompactGeneration`, `CompactKeptWaste`, + `CompactResidue`, `CompactZeroReason`), the `CompactReportSource` trait that + projects a backend receipt into it, and `CompactStatus`, the rendered + `store.status` projection of a run. The sidecar reader itself is T12.2's + `compact_op_status`/`CompactOpStatus`; T10 extends that projection with the + run's `started_ms` and owner (`pid`, kernel start, boot). +- `control/crates/layerfs-store/src/report.rs` — the one serializer: + `compact_report` and `compact_status`. Every key is snake_case (§10; §11 + names are logical) and `CompactZeroReason::wire_field` names each zero field + by its rendered key. +- `CompactReceipt` is unchanged: `From<&CompactReceipt>`/`From` + and the trait project it. The control renderer no longer builds a map itself. +- `layerfs_store::sqlite::transfer::step::StoreStep::CompactOp` reads only + `.compact-op`; it travels the same `store.file` frame to a worker, so + a remote SQLite Store answers it without opening the database, scanning the + directory or touching an artifact. + +## Report keys + +| §11 name | rendered key | +|---|---| +| `path` | `path` (`none`/`rebuild`) | +| `generation` | `generation {before, after}` | +| `deadBytesBound` / `deadBytesRefined` | `dead_bytes_bound` / `dead_bytes_refined` | +| `logicalDiscarded` | `logical_discarded` | +| `backendReusable` | `backend_reusable` | +| `physicalShrink` | `physical_shrink` | +| `keptWaste` | `kept_waste {dead_in_retained_packs_bytes, delta_bases {bytes, references}, inode_blocks_bytes}` | +| `peakTempBytes` | `peak_temp_bytes` | +| `residue` | `residue {bytes, artifacts, reason?}` | +| `estimate` | `estimate` | +| `zeroReason` per zero field | `zero_reasons [{field, reason}]` | + +`kind` is `Compacted`. `residue.bytes` is what is **left**, not what was +removed; `clear_artifacts` now adds a size only when a removal left a file, so +a successful settle reports an empty residue of 0 bytes (T10 fixes the T8 +accounting). + +## `store.status` + +While a compaction runs, or after a dead owner left one, the call adds: + +- `op` — `{id, name, phase, started, uncertain}` from `.compact-op` + (SQLite) / the Store's `gc_op` (PostgreSQL, T9); +- `state` — `busy` while the owner lives, `unresolved` once it is gone + (otherwise the existing `idle`/`in_use`/`busy`/`down` state). `uncertain` + is the engine projection's own fact and is true for **every** record that + remains, live or leftover; `state` is what separates the two. Equating + `uncertain == true` with `unresolved` is wrong. +- `phases` — the backend's ordered phases: `mark`, `build`, `verify`, + `install` on SQLite (T9 adds `mark`, `build`, `swap`, `cleanup`). + +No compaction record means `op: null` and `phases: null`, and a transfer +`op` still renders exactly as before. The sidecar now records the owner +`pid`, kernel start time and boot id; the status step compares them with +`layerfs_home::process::alive_as`, as leases do, so a reused pid cannot pass +as a live owner. The run writes `mark`, then `build`, `verify` and `install` +as it enters each phase, so the phase is visible while it runs (AC-51). + +## PostgreSQL hook (T9) + +`CompactReportSource` is the hook: SQLite's `CompactReceipt` implements it, +and T9 implements it for its PostgreSQL receipt. The report struct names no +backend-only field, so PostgreSQL reports its zeros through the same keys +(`backend_reusable = 0`, `dead_bytes_refined == dead_bytes_bound`) without a +wire change. `store.status` reads the Store-side `gc_op` at the same point in +`status.rs`; nothing is read from a file there. + +## Tests + +- `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: + `the_report_projects_every_section_11_field`, + `the_empty_store_report_explains_its_zeros`, + `a_leftover_op_reports_its_phase_and_state` (a real `AfterBuild` failpoint: + phase `build`; `status(true)` is `busy`, `status(false)` is `unresolved`, + both uncertain; ordered phases, then the next run settles the artifact and + the record). +- `control/crates/layerfs-store/tests/compact_report_status.rs`: the report + wire is the snake_case contract shape (including `zero_reasons` and the + `delta_bases.references` hex), and `store.status` projects an idle Store, + a live owner's record and a leftover one (`op`, `state`, `phases`). + +## Gaps + +- T9's PostgreSQL rebuild and its `gc_op` status read are not implemented; + the trait and the status comment are the only PostgreSQL pieces. +- The `{value, cap}` capped-field shape is not produced: the SQLite receipt + has no cap today. diff --git a/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md b/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md new file mode 100644 index 0000000000..c96960b7a7 --- /dev/null +++ b/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md @@ -0,0 +1,88 @@ +# T12.1: SQLite compaction fixtures and the golden receipt + +> Acceptance fixture for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §11 (report), §12 (acceptance) and §6.4 (live-heads digest). It adds no +> contract: the spec and the matrix win; `§N` and `AC-nn` cite them. + +## What the fixture is + +`engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs` builds one +Store through the public APIs only (`Fixture::put`/`record`/`inode`, the +`HistoryCatalog` methods, `SqliteHandles::compact`). It is the first fixture +that exercises the §7 rebuild with every §11 report component non-trivial: + +| Fixture part | What it pins | +|---|---| +| Two Stacks (one live, one `mark_stack_deleted`) | retained history and a project collected whole | +| A published Branch retired but still named by its retained Layers | `branch_retired` marker survives; name stays taken | +| An unmerged Branch retired and named by nothing | retired Branch row and marker collected | +| Commits with known `statistics.logical` (120/240/360/70/40) | `logicalDiscarded` is the exact history `logical` delta (160) | +| 12 unreferenced whole files, one per pack | wholly dead packs and `deadBytesBound` | +| A depth-2 PREFIX chain `base2 <- base1 <- target` with only `target` a retained root | `keptWaste.deltaBases` references both bases; pack ids survive (AC-17) | +| One retained pack with a reached group and an unreached group | `keptWaste.deadInRetainedPacksBytes`; a 2.0 rebuild keeps the group | +| A pooled inode leaf inside every tree | `keptWaste.inodeBlocksBytes` (all `inode_value_block` rows are kept) | +| Two `file_digest` rows, one root retained and one collected | AC-18 | +| A retired Branch with no head and a fixed-id digest Store | §6.4 golden vector (AC-56, SQLite half) | + +The transcript covers three outcomes: + +- `emptyNone`: an empty Store is `none` with `zeroReason: empty-store`, no + generation change, no artifact. +- `noDeadNone`: a Store whose every pack holds a live object is `none` with + `zeroReason: no-waste`; `deadBytesRefined == deadBytesBound` and the digest + and generation are unchanged. +- `rebuild`: `G -> G+1`; live objects (including the delta target through its + chain) read, dead objects and packs disappear, the retained pack keeps its + unreached group, collected history rows and their markers disappear, a + retained `branch_retired` marker stays, the full doctor passes and the + `live_heads_digest` is unchanged. + +## Defect found by the fixture + +The rebuild report's `residue.bytes` counted the bytes of every artifact the +settle *removed*, not only the ones that remain, so a fully successful rebuild +reported a nonzero residue with `artifacts: []` and no `reason` — contrary to +§11 ("a nonzero residue is explained") and the T8 note ("residue reports +anything that could not be removed"). `clear_artifacts` now adds a file's size +only when its removal failed; the fixture asserts `residue.bytes == 0` and the +golden pins it. `sqlite_compact` still passes. + +## Golden transcript + +`engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json` +is the byte-for-byte golden (`Json::render`, every object sorted by key). It +contains the three receipts, the mark tuple (digest, object count, packs), the +history markers and the fixture identities. A change to any value fails the +test; update it deliberately: + +``` +UPDATE_GOLDEN=1 cargo test -p layerfs-persistence --test sqlite_compact_fixtures +cargo test -p layerfs-persistence --test sqlite_compact_fixtures +``` + +The second run recompiles the embedded golden and checks it. + +## Matrix rows covered (SQLite half) + +| Case | Evidence | +|---|---| +| AC-04 | deterministic fixture: no live object reaches a collected row; closure compared before/after | +| AC-05 | full doctor (includes `diagnostic/scan.rs`) on the compacted Store | +| AC-17 | depth-2 chain copied; `pack_id` preserved on the bases and the target | +| AC-18 | collected digest row gone, retained one kept | +| AC-25 | retained `branch_retired` survives; collected `branch_retired`/`layer_stack_retired` go | +| AC-38 | the successful artifact set is removed and no stray name remains | +| AC-40 | SQLite `live_heads_digest` is identical before and after; a retired head moves it | +| AC-47 | `none` leaves no `.compact`; a successful settle leaves no artifact | +| AC-48 | empty Store (0/0) is `none` with `zeroReason`; the fixture is a `rebuild` arm | +| AC-52 | every §11 field on both paths, with units and sampling point, pinned by the golden | +| AC-56 | fixed-id digest vector with out-of-order ids, a retired Branch and stack, and an absent head (SQLite half) | +| AC-58 | `none` leaves no `.compact`/`.compact-visited`; `rebuild` settles to no artifact | + +Not covered here, by design (T12.2/T12.3): crash failpoints and `kill -9` +(AC-01, AC-03, AC-33, AC-46, AC-47 crash arm, AC-58 mid-run arm, AC-59), the +9%/10%/30% boundary arms and property tests (AC-04 at E3, AC-48 boundary, +AC-57), the PostgreSQL halves (AC-02, AC-28..AC-44, AC-60), admission races +(AC-07, AC-08, AC-49), and the report/`store.status` op projection (AC-51). +The PostgreSQL digest agreement (AC-40) and the shared literal vector (AC-56) +are proven on SQLite only here. diff --git a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md new file mode 100644 index 0000000000..cc8461befe --- /dev/null +++ b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md @@ -0,0 +1,138 @@ +# T8: SQLite `store.compact` rebuild + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §7 (SQLite rebuild), §9 (failure and leftovers) and §11 (report). It adds no +> contract: the spec wins and `§N` cites it. + +## Placement + +`SqliteHandles::compact` is the host-side primitive T10 wraps as +`store.compact`. It consumes the handles like `SqliteHandles::seal` +(`engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs`): a rebuild +replaces the Store file and the caller reopens it afterwards. The public +result is `CompactReceipt`, one value per §11 field, plus the small +`CompactPath`, `CompactKeptWaste`, `CompactResidue` and `CompactZeroReason` +types. The `GcIntent` admission row is registered through the T6 +`HistoryCatalog::gc_register` API; a mark is the T7.2 `HistoryProvider::mark`. + +## Run (SQLite) + +1. **Lane.** `Session::hold_lane()` takes the Store's FIFO turn and the + cross-process `writer_admission` lock, and keeps both. Every later + transaction of this thread is nested work of that lane, so no admitted + writer can interleave between the mark, the seal and the rename. +2. **Checkpoint.** `Session::checkpoint_holding()` runs + `PRAGMA wal_checkpoint(TRUNCATE)` under the held lane; a nonzero `busy` is + `Busy`, never retried. +3. **Admission.** The compacting call writes `.compact-op` (the SQLite + sidecar equivalent of `gc_op`) and registers one Store-scoped + `gc_intent(kind='compaction')` row with the tuple: generation, mark serial, + mark pack id, digest, totals, stamp, phase, owner tokens. +4. **Mark.** `mark()` runs in its write transaction under the same lane. The + §6.1 history closure is written to `kept_node`; the §6.2 closure follows + logical references and delta bases to its fixpoint, so the packs it keeps + are already the refined live set. `deadBytesBound` and + `deadBytesRefined` are therefore equal on both §3 paths. +5. **Path.** `rebuild iff SUM(pack.length) > 0 AND dead * 10 >= SUM(pack.length)`. + `none` clears its intent and op record and returns; it writes no Store + table and creates no artifact, so no scratch exists to delete. +6. **Seal.** `Session::seal_holding(writer)` checkpoints, checked-closes, + checks `-wal`/`-shm`/`-journal`, and captures `(size, mtime, change + counter)` while the lane is still held (§7 step 1). The sealed tuple and + the §6.4 `live_heads_digest` are then re-read through an `immutable=1` + handle and compared with the mark; a difference is `Conflict`. +7. **Build.** `.compact-visited` (`journal_mode=OFF`) holds the visited + object and live-pack sets. `VACUUM INTO .compact` copies the sealed + file compactly; inside one transaction the retained rows are kept and the + complement deleted (dead packs/pack units, unreferenced locations, + signatures and digests, history/lineage/catalog rows outside `kept_node`, + markers of collected rows, merge scratch, `gc_intent`/`gc_dead`/`gc_op`, + acquisition working rows), `binding_hint_block` is cleared, + `catalog_incarnation` becomes `G + 1`, and `store_totals` is recomputed + from the retained rows with its `init_*` columns untouched. `VACUUM`, a + recreated `sqlite_sequence`, and a WAL switch with a zero-`busy` TRUNCATE + checkpoint finish the artifact. Package immutability triggers are dropped + for the delete and recreated from their stored SQL. +8. **Verify.** An `immutable=1` reader runs `schema::check`, + `foreign_key_check`, row counts against the mark (`object_location`, + `pack`, `kept_node`), both live-heads digests, the allocator tuple, and the + recomputed totals against their row sums. +9. **Install.** Sidecars and `-mj*` on both files are rechecked, the sealed + `(size, mtime, change counter)` and the full tuple are re-read last, the + artifact gets `chmod 0600` and `sync_all`, then `rename` and a directory + fsync. A rename/fsync failure is installed-but-uncertain: the op record is + kept at phase `install` and `UnknownOutcome` is returned. +10. **Settle.** The artifact set is removed, then the op record. `residue` + reports anything that could not be removed. + +## Failure and leftovers (§9) + +The next `store.compact` reads `.compact-op` under the lane. Because +the lane was free, the recorded owner is gone: it releases that op's admission +intent (`gc_release`), removes the named artifact set, and removes the op +record, judging by the generation it read (`G` = not installed, `G + 1` or +later = installed/advanced). An artifact set that no op record names is +refused `Exists` and is never deleted. A definite pre-install failure deletes +the artifact set and the op record; the old Store is unchanged apart from the +checkpoint. Failpoints (`AfterSeal`, `AfterBuild`, `AfterWalSwitch`, +`AfterVerify`, `BeforeInstall`, `AfterInstall`) exercise the crash state +without killing the process. + +## Report (§11) + +`CompactReceipt` carries `path`, `generation {before, after}`, +`deadBytesBound`, `deadBytesRefined`, `logicalDiscarded`, +`backendReusable` (free pages × page size), `physicalShrink` (old size − new), +`keptWaste`, `peakTempBytes`, `residue`, `estimate` (`true`: §13) and one +`zeroReason` per zero field from the closed vocabulary. `logicalDiscarded` is +the collected history rows' `logical` bytes; `parallel` note: `estimate` is +always true until #52 measures the arms. + +## Tests + +`engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: + +- empty Store → `none`, `zeroReason: empty-store`, unchanged generation; +- no wholly dead pack → `none`, `deadBytesRefined == deadBytesBound`, no + artifact, unchanged generation; +- a wholly dead pack → `rebuild`, `G + 1`, dead object gone, retained object + readable, no artifact/sidecar, full doctor clean; +- retained Commit/Layer history and its objects survive a rebuild; the full + doctor passes on the installed Store; +- a stray `.compact` with no op record is `Exists` and not deleted; +- `AfterBuild` leaves the staged Store, visited scratch and op record; the + reopened Store is still at `G`, and the next call settles and rebuilds; +- `AfterInstall` leaves the installed Store at `G + 1` and the op record; the + next call judges it and settles; +- generation increases once per installed rebuild. + +Runs: `cargo test -p layerfs-persistence --offline` (all binaries pass), +`cargo check --workspace --offline` passes. `kernel_guard` in +`layerfs-store-service` fails in this container because `/tmp` is tmpfs and +the guard refuses direct Store access; it is unrelated to this change. + +## Gaps (for T10/T12) + +- ~~The `(dev, ino)` `StaleStore` fence (§10) is not implemented in the SQLite + session.~~ Implemented in T8.1: `Session` records `(dev, ino)` at open and + refuses the next explicit transaction, read-only snapshot or immutable read + with `StaleStore`; the install fence compares the same pair with + `SealedFacts` before the rename. +- ~~`Capacity` mapping is implemented but no capacity/failpoint test asserts + it.~~ T8.1 adds `CompactFailpoint::CapacityAtBuild` (ENOSPC after the staged + file exists) and unit tests for the `ENOSPC`/`EDQUOT`/`EFBIG` mapping. +- ~~The 9%/10%/30% boundary and the `GroupRows`/`GroupRowsIndexed` and + acquisition layouts are exercised only on the Monolithic layout.~~ T8.1 + exercises 9% (`none`), 10% (rebuild at the boundary) and 30% (rebuild), both + grouped layouts on the `none` and rebuild paths, and the acquisition layout + on open/`none`/rebuild with its working rows cleared. The `postgresql` + feature is not built here. +- A definite pre-install failure now releases its `gc_intent` while the Store + is still open (T8.1): the admission row cannot be cleared after the seal, + and a live intent that outlived its op record refused every later GC call + `BusyOp`. The op record is kept only when the release itself fails, as the + next run's recovery breadcrumb (§9). +- Peak temp is the new file plus the visited scratch; journal/WAL sizes + after the checkpoint are zero and are not added. +- The mark runs before the seal under the same lane (the spec's step 1.5 runs + it after the seal); the sealed tuple re-read makes the window explicit. diff --git a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md new file mode 100644 index 0000000000..1d9265c448 --- /dev/null +++ b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md @@ -0,0 +1,115 @@ +# T9: PostgreSQL `store.compact` rebuild + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §8 (PostgreSQL rebuild), §9 (failure and leftovers) and §11 (report). It adds +> no contract: the spec wins and `§N` cites it. + +## Placement + +`PostgresHandles::compact` is the Store-side primitive +(`engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs`); the +`store.compact` control call dials the Store's recorded URL, opens its schema, +and runs it (`control/crates/layerfs-store/src/compact.rs`). Unlike SQLite the +session is kept; the caller reopens when it needs the new generation pinned +(the diagnostic guard rejects a stale incarnation by design, §10). + +The mark reuses T7.2: `HistoryProvider::mark_gc` reads the §8 tuple as the +mark's first statement and then runs the shared §6.1/§6.2 closure in one write +transaction. The new SQLite-portable statement `gc_tuple_read` is registered +with the other history statements, so both engines share it. + +## Run (PostgreSQL) + +1. **Admission.** `settle_leftover` judges a recorded `gc_op` first (§9): `G` + with a live `idle` owner is terminated with the two-argument + `pg_terminate_backend(pid, backend_start)` and adopted; a live working owner + is `Busy`; `G + 1` resumes Phase C. Then one Store-scoped `gc_intent` + (`compaction`, exclusive) is registered: its session advisory key is held + through Phase C, and a failure releases it. +2. **Phase M — mark.** The first statement reads `(catalog_incarnation, + next_history_serial, next_pack_id, next_ordinal, store_totals.*, + MAX(stamp), history rows)`; the T7.2 closure writes `kept_node` and returns + the object/pack set and live-heads digest. The set is loaded into session + temporary tables (`gc_visited`, `gc_live_pack`). + `deadBytesBound` = stored bytes minus the retained packs' bytes. + `gc_op` (phase `mark`, deterministic scratch name `lfs_gc_`) and the + persisted `gc_dead` membership (dead Commit/Layer/Branch/Stack ids, + `content_signature` slots, `file_digest` roots) are written next; the + `dead_logical` sum of dead history rows is recorded. A `none` decision + truncates `gc_dead`, clears `gc_op`, releases the intent and returns. +3. **Phase A — build** (session advisory key, no table lock). `CREATE SCHEMA` + `lfs_gc_`; `CREATE TABLE ... (LIKE schema.pack INCLUDING STORAGE + INCLUDING COMPRESSION INCLUDING CONSTRAINTS)` plus the explicit primary key + (the copy does not carry it), the same for `object_location`; owner, + tablespace and reloptions are copied from `pg_class`. Retained rows are + copied in pack-id batches, each its own transaction and idempotent by + deleting its own range first; `object_location` uses an `EXISTS` semi-join. + The prefix index, foreign key and `ANALYZE` follow; counts are checked + against the mark. +4. **Phase B — swap** (one short transaction). Every Store base table is + locked `ACCESS EXCLUSIVE NOWAIT`; `55P03` rolls back, drops the scratch + schema in a new transaction and is `Busy`. The mark tuple and live-heads + digest are re-read under the lock; any change is `Conflict` with no writes. + Owner/ACL are applied, the two tables swapped, the `pack_immutable` trigger + recreated, counts and the four totals checked, `dead_canonical/stored/ + metadata` persisted, `catalog_incarnation = G + 1`, the effective + `synchronous_commit` (Store and server, never lowered) set, the standby + predicate re-checked against `synchronous_standby_names` and `pg_stat_replication`, + the scratch schema dropped, and `COMMIT`. +5. **Phase C — cleanup** (batched, after the swap). Only `gc_dead` members are + deleted, child first; `binding_hint_block` is cleared; the acquisition + working tables are emptied. `store_totals` is reduced once by the persisted + `dead_*` totals in the settle transaction and checked against the retained + row sums; `gc_dead`/`gc_op` are cleared and the intent released. A row + created after the mark is not a member, so it survives. + +Report (§11): `path`, `generation {before, after}`, `deadBytesBound` = +`deadBytesRefined`, `logicalDiscarded`, `backendReusable = 0` +(`postgresql-in-place`), `physicalShrink` from `pg_total_relation_size`, +`keptWaste`, `peakTempBytes` (scratch schema), empty `residue`, `estimate` and +the zero reasons; PostgreSQL reports 0 for rebuilt-table reusable space. + +## Tests + +`engine/crates/layerfs-persistence/tests/postgresql_compact.rs` (real +PostgreSQL; `LAYERFS_TEST_POSTGRES_URL`): + +| Test | Covers | +|---|---| +| `an_empty_store_takes_the_none_path` | empty Store → `none`, `empty-store`, no artifact, cleared `gc_op`/`gc_dead`/intent | +| `a_live_store_takes_the_none_path` | live Store → `none`, bound = refined, no shrink | +| `a_wholly_dead_pack_is_rebuilt_away_and_totals_settle` | rebuild, dead pack collected, retained object kept, `G → G+1`, totals invariant, scratch dropped | +| `retained_roots_survive_a_rebuild_that_collects_a_deleted_project` | retained roots survive; a deleted project's history, markers and objects are collected; `logicalDiscarded > 0` | +| `a_lost_swap_reply_is_resumed_by_the_next_compact` | failpoint after the swap: the op stays at phase `swap`, `G+1`; the next compact finishes Phase C and reports the installed outcome | +| `a_stale_owner_backend_is_terminated_before_the_next_round` | a failpoint before Phase A leaves an idle owner; the next session terminates it with `pg_terminate_backend(pid, backend_start)` and adopts the op | +| `the_same_session_resumes_an_uncertain_round` | a failpoint before Phase A leaves the caller's own idle session as the recorded owner; the retry adopts its own op instead of terminating itself | +| `an_exclusive_table_lock_makes_the_swap_busy` | a conflicting `ACCESS EXCLUSIVE` lock at the swap is `Busy` (55P03), and the scratch schema is dropped | + +Unit tests in `compact.rs` pin the standby predicate (`rank < remote_write` or +no names → no wait; names → a streaming sync/quorum standby is required) and +the `synchronous_commit` strength order. + +Result at T9: `cargo test -p layerfs-persistence --features postgresql --test +postgresql_compact` — 8 passed on PostgreSQL 17.12 (Debian, server on +`127.0.0.1:55432`); `--lib` 4 passed; the SQLite suite +(`--test sqlite_compact`) passed (17 when T9 branched, 20 on the rebased main with the T12 fixtures); `sqlite_compact_crash` passed 13. + +## Gaps (not contract changes) + +- **AC-41/AC-42/AC-43/AC-44/AC-60 remain `open`** in the acceptance matrix: + the tests are E1/E2 evidence, not the full #52 matrix. Not covered here: + a reserved-pack publish racing Phase A/C, a mid-cleanup crash failpoint, a + real synchronous standby, and a truly lost `COMMIT` reply from a killed + backend. The swap-phase failpoint proves the resume path, not the server's + reply loss. +- **ACL replay** is best-effort: the owner and a default table ACL are copied; + non-default column/function ACLs are outside this note. +- **Busy holder detail** is the code only (`{entity, phase, op}` and PG + `pid/client_addr/backend_start` are not populated yet); the runbook's lock + query is the manual path. +- The Phase A batch size is a fixed 512 packs, not a 1 GB byte budget; the + scratch copy is `INSERT ... SELECT`, so no pack bytes cross the client. +- `physicalShrink`/`peakTempBytes` are sampled, and a resumed cleanup reports + them as 0 (the op does not persist them). +- `store.status` reading the Store-side `gc_op` is T10/T11; this note only + writes the row/phases it needs. diff --git a/docs/architecture/api/calls/README.md b/docs/architecture/api/calls/README.md index dc953392b0..d17fa5cc97 100644 --- a/docs/architecture/api/calls/README.md +++ b/docs/architecture/api/calls/README.md @@ -118,8 +118,14 @@ with one line in `/trace/calls.jsonl`, except the `logs` calls and with | `tag` | a top-level `log_tag` argument (taken out before the check; no entry declares it), else `LAYERFS_LOGS_TAG`; at most 128 bytes | Values are cut to 200 characters. Content, JSON, and the `Text` arguments -`command`, `message`, `old`, `new`, `password` and `url` are logged only as -`` (`REDACTED`). The line is written after the outcome and never +`command`, `message`, `old`, `new`, `password`, `url` and every path argument +(`path`, `left_path`, `right_path`, `from_path`, `to_path`, `path_prefix`) +are logged only as `` (`REDACTED`), so a `project.remove_file` or +`file.*` path never enters the log (#41 §4.3). A path counts its normalized +form, one leading `/` stripped (`/a` and `a` are both `<1 bytes>`); a glob or +grep `pattern` is not a path and counts its own bytes. A refusal contributes +only its code: its message and `detail` (which may echo the path, as `Removed` +does) are never logged. The line is written after the outcome and never changes it. Reading and clearing it are the [`logs`](../sdk/logs.md) calls. ## Invariants diff --git a/docs/architecture/control/home/README.md b/docs/architecture/control/home/README.md index 9ed4321e45..8c7ed8b17e 100644 --- a/docs/architecture/control/home/README.md +++ b/docs/architecture/control/home/README.md @@ -155,8 +155,12 @@ already holds as it is. | `tag` | per-call `log_tag`, else `LAYERFS_LOGS_TAG`; at most 128 bytes, no control characters, else dropped | Values are cut to 200 characters. Content and Json arguments and the Text -arguments `command`, `message`, `old`, `new`, `password` and `url` are -logged as ``. +arguments `command`, `message`, `old`, `new`, `password`, `url` and every +path argument (`path`, `left_path`, `right_path`, `from_path`, `to_path`, +`path_prefix`) are logged as `` of the normalized form, one leading +`/` stripped (`/a` and `a` are both `<1 bytes>`); a glob or grep `pattern` +counts its own bytes. A refusal contributes only its code: its message and +`detail` are never logged. No line is written for a running or killed call, worker requests, a call's internal parts, commands outside the call table (`help`, diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql new file mode 100644 index 0000000000..67853fc6f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/gc_epoch_read.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql new file mode 100644 index 0000000000..67853fc6f2 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_epoch_read.sql @@ -0,0 +1 @@ +SELECT epoch FROM project_epoch WHERE layer_stack_id = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql new file mode 100644 index 0000000000..5c235efcef --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql @@ -0,0 +1,6 @@ +SELECT h.catalog_incarnation,h.next_history_serial,p.next_pack_id,p.next_ordinal, + t.logical,t.canonical,t.stored,t.metadata, + (SELECT COALESCE(MAX(stamp),0) FROM content_signature), + (SELECT COUNT(*) FROM "commit")+(SELECT COUNT(*) FROM layer) + FROM history_meta h, store_policy p, store_totals t + WHERE h.id=1 AND p.id=1 AND t.id=1 diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs new file mode 100644 index 0000000000..a359f6cd6f --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs @@ -0,0 +1,1413 @@ +//! §8 PostgreSQL Store compaction: Phase M mark, Phase A build, Phase B swap +//! and Phase C cleanup. +//! +//! One call takes the T6 Store-scoped exclusive intent (a session advisory +//! key held through cleanup), records the §8 mark tuple as its first +//! statement, runs the T7.2 mark, and decides from the wholly dead pack +//! bytes. A `rebuild` copies only retained `pack`/`object_location` rows into +//! a scratch schema, swaps the two tables in one short transaction at +//! `G -> G + 1`, and deletes only the ids the mark persisted in `gc_dead`. +//! The op row in `gc_op` names the scratch schema before it is created, so a +//! crash (or a lost COMMIT reply) is judged by the next `store.compact`: +//! generation `G` removes the artifact, `G + 1` resumes cleanup. + +use super::session::{ + connection::Session, + transaction::{Mode, Transaction}, +}; +use crate::backend::records::Param; +use crate::backend::sqlite::compact::{ + CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, + CompactResidue, CompactZeroReason, +}; +use crate::history::GcTuple; +use crate::{MarkResult, PostgresHandles}; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::HistoryCatalog; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// The §3 rebuild threshold: rebuild when dead * 10 >= stored. +const THRESHOLD: u64 = 10; +/// `lfs_gc_`: the deterministic scratch schema name. +const SCRATCH_PREFIX: &str = "lfs_gc_"; + +/// The three wholly dead domains the swap records before COMMIT. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct DeadTotals { + canonical: i64, + stored: i64, + metadata: i64, + logical: i64, +} + +/// One compaction op row, as the next call judges it. +#[derive(Clone, Debug)] +struct OpRow { + generation: i64, + artifact: Option, + dead: DeadTotals, + totals_applied: bool, + pid: Option, + backend_start: Option, +} + +/// What Phase A leaves for Phase B. +struct Built { + scratch: String, + scratch_bytes: u64, + dead: DeadTotals, +} + +/// What the swap records for the report and Phase C. +struct Swapped { + physical_before: u64, + physical_after: u64, + peak_temp: u64, + dead: DeadTotals, +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn scratch_name(op_id: &[u8; 16]) -> String { + format!("{SCRATCH_PREFIX}{}", hex(op_id)) +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// One write transaction on the Store session; scratch/DDL work never waits +/// for the publication lane (the session advisory key is the exclusion). +fn stage( + session: &Arc, + body: impl FnOnce(&Transaction<'_>) -> HistoryResult, +) -> HistoryResult { + session.run_as(Mode::Stage, body) +} + +fn scalar(tx: &Transaction<'_>, sql: &str, params: Vec) -> HistoryResult { + let rows = tx.query(sql, params)?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction scalar")); + } + rows[0].get(0).map_err(Into::into) +} + +fn scalar_session(session: &Arc, sql: &str, params: Vec) -> HistoryResult { + stage(session, |tx| scalar(tx, sql, params)) +} + +fn read_generation(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!("SELECT catalog_incarnation FROM {schema}.history_meta WHERE id=1"), + Vec::new(), + ) +} + +fn read_op(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT generation,artifact,dead_canonical,dead_stored,dead_metadata,dead_logical,totals_applied,pid,backend_start FROM {schema}.gc_op WHERE id=1" + ), + Vec::new(), + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + Ok(Some(OpRow { + generation: row.get(0)?, + artifact: row.get(1)?, + dead: DeadTotals { + canonical: row.get(2)?, + stored: row.get(3)?, + metadata: row.get(4)?, + logical: row.get(5)?, + }, + totals_applied: row.get::(6)? != 0, + pid: row.get(7)?, + backend_start: row.get(8)?, + })) + }) +} + +fn set_phase(session: &Arc, schema: &str, phase: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.query( + &format!("UPDATE {schema}.gc_op SET phase=$1 WHERE id=1"), + vec![Param::Text(phase.to_owned())], + )?; + Ok(()) + }) +} + +fn stop( + request: &CompactRequest, + point: CompactFailpoint, + session: &Arc, + schema: &str, + phase: &str, +) -> HistoryResult<()> { + if request.failpoint != Some(point) { + return Ok(()); + } + set_phase(session, schema, phase)?; + Err(HistoryError::UnknownOutcome) +} + +/// The set of live object ids and live pack ids, as session temporary tables +/// the Phase A copy and the dead-set query read. +fn load_visited(session: &Arc, schema: &str, mark: &MarkResult) -> HistoryResult<()> { + let objects: Vec> = mark + .objects() + .objects() + .iter() + .map(|id| id.as_bytes().to_vec()) + .collect(); + let packs: Vec = mark.objects().packs().to_vec(); + stage(session, |tx| { + tx.batch( + "DROP TABLE IF EXISTS pg_temp.gc_visited;\ + DROP TABLE IF EXISTS pg_temp.gc_live_pack;\ + CREATE TEMP TABLE pg_temp.gc_visited(object_id bytea PRIMARY KEY) ON COMMIT PRESERVE ROWS;\ + CREATE TEMP TABLE pg_temp.gc_live_pack(pack_id bigint PRIMARY KEY) ON COMMIT PRESERVE ROWS;", + )?; + let bytes = objects.iter().map(Vec::len).sum::() as u64; + tx.typed( + "INSERT INTO pg_temp.gc_visited(object_id) SELECT unnest($1::bytea[])", + &[&objects], + bytes, + )?; + tx.typed( + "INSERT INTO pg_temp.gc_live_pack(pack_id) SELECT unnest($1::bigint[])", + &[&packs], + 8 * packs.len() as u64, + )?; + let _ = schema; + Ok(()) + }) +} + +/// Phase M: write `gc_op` (phase mark) and the persisted `gc_dead` set in +/// batches. The set is a membership, not a predicate, so a row created after +/// the mark is never collected. +fn persist_mark( + session: &Arc, + schema: &str, + mark: &MarkResult, + tuple: &GcTuple, + scratch: &str, + request: &CompactRequest, +) -> HistoryResult { + let digest = mark.live_heads_digest().to_vec(); + stage(session, |tx| { + tx.query(&format!("DELETE FROM {schema}.gc_dead"), Vec::new())?; + tx.batch(&format!( + "INSERT INTO {schema}.gc_dead(kind,id) \ + SELECT 'commit',c.commit_id FROM {schema}.\"commit\" c \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='commit' AND k.node_id=c.commit_id) \ + UNION ALL \ + SELECT 'layer',l.layer_id FROM {schema}.layer l \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='layer' AND k.node_id=l.layer_id) \ + UNION ALL \ + SELECT 'branch',b.branch_id FROM {schema}.branch b \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='branch' AND k.node_id=b.branch_id) \ + UNION ALL \ + SELECT 'stack',s.layer_stack_id FROM {schema}.layer_stack s \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='stack' AND k.node_id=s.layer_stack_id) \ + UNION ALL \ + SELECT 'signature',int8send(cs.slot) FROM {schema}.content_signature cs \ + WHERE NOT EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=cs.object_id) \ + UNION ALL \ + SELECT 'digest',fd.digest FROM {schema}.file_digest fd \ + WHERE NOT EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=fd.root)" + ))?; + let logical = scalar( + tx, + &format!( + "SELECT COALESCE((SELECT SUM(c.logical)::bigint FROM {schema}.\"commit\" c \ + WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='commit' AND d.id=c.commit_id)),0) \ + + COALESCE((SELECT SUM(l.logical)::bigint FROM {schema}.layer l \ + WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='layer' AND d.id=l.layer_id)),0)" + ), + Vec::new(), + )?; + let started = now_ms(); + tx.query( + &format!( + "INSERT INTO {schema}.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,mark_totals,mark_stamp,started_ms,dead_canonical,dead_stored,dead_metadata,dead_logical,totals_applied,artifact,pid,backend_start,incarnation) \ + VALUES (1,$1,'mark',$2,$3,$4,$5,int8send($6)||int8send($7)||int8send($8)||int8send($9),$10,$11,0,0,0,$12,FALSE,$2,pg_backend_pid(),(SELECT backend_start::text FROM pg_stat_activity WHERE pid=pg_backend_pid()),$1) \ + ON CONFLICT (id) DO UPDATE SET generation=EXCLUDED.generation,phase=EXCLUDED.phase,scratch_schema=EXCLUDED.scratch_schema,mark_serial=EXCLUDED.mark_serial,mark_pack_id=EXCLUDED.mark_pack_id,mark_digest=EXCLUDED.mark_digest,mark_totals=EXCLUDED.mark_totals,mark_stamp=EXCLUDED.mark_stamp,started_ms=EXCLUDED.started_ms,dead_canonical=0,dead_stored=0,dead_metadata=0,dead_logical=EXCLUDED.dead_logical,totals_applied=FALSE,artifact=EXCLUDED.artifact,pid=EXCLUDED.pid,backend_start=EXCLUDED.backend_start,incarnation=EXCLUDED.incarnation" + ), + vec![ + Param::I64(tuple.generation), + Param::Text(scratch.to_owned()), + Param::I64(tuple.next_history_serial), + Param::I64(tuple.next_pack_id), + Param::Bytes(digest), + Param::I64(tuple.logical), + Param::I64(tuple.canonical), + Param::I64(tuple.stored), + Param::I64(tuple.metadata), + Param::I64(tuple.stamp), + Param::I64(started), + Param::I64(logical), + ], + )?; + let _ = request; + Ok(DeadTotals { + logical, + ..DeadTotals::default() + }) + }) +} + +fn clear_round(session: &Arc, schema: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_dead; DELETE FROM {schema}.gc_op;" + ))?; + Ok(()) + }) +} + +/// The stored bytes of every pack the mark retained, in the current Store. +fn live_stored(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + ) +} + +fn total_stored(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack"), + Vec::new(), + ) +} + +fn retained_totals(session: &Arc, schema: &str) -> HistoryResult<(i64, i64, i64)> { + stage(session, |tx| { + let canonical = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(l.canonical_length)::bigint,0) FROM {schema}.object_location l \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=l.object_id)" + ), + Vec::new(), + )?; + let stored = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + )?; + let metadata = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE p.domain=0 AND EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + )?; + Ok((canonical, stored, metadata)) + }) +} + +fn relation_bytes(session: &Arc, schema: &str, names: &[&str]) -> HistoryResult { + let list = names + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(","); + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT COALESCE(SUM(pg_total_relation_size(c.oid))::bigint,0) FROM pg_class c \ + JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname IN ({list})" + ), + vec![Param::Text(schema.to_owned())], + )?; + let value: i64 = rows[0].get(0)?; + Ok(u64::try_from(value).unwrap_or(0)) + }) +} + +fn table_storage( + tx: &Transaction<'_>, + schema: &str, + table: &str, +) -> HistoryResult<(String, Vec)> { + { + let rows = tx.query( + "SELECT COALESCE((SELECT t.spcname FROM pg_tablespace t WHERE t.oid=c.reltablespace),''), \ + COALESCE(array_to_string(c.reloptions,','),'') \ + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname=$2", + vec![Param::Text(schema.to_owned()), Param::Text(table.to_owned())], + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction source table")); + } + let tablespace: String = rows[0].get(0)?; + let reloptions: String = rows[0].get(1)?; + let options = reloptions + .split(',') + .filter(|option| !option.is_empty()) + .map(str::to_owned) + .collect(); + Ok((tablespace, options)) + } +} + +fn create_scratch(session: &Arc, schema: &str, scratch: &str) -> HistoryResult<()> { + let exists = scalar_session( + session, + "SELECT count(*) FROM pg_namespace WHERE nspname=$1", + vec![Param::Text(scratch.to_owned())], + )?; + if exists != 0 { + return Err(HistoryError::Exists("compaction scratch schema")); + } + stage(session, |tx| { + tx.batch(&format!("CREATE SCHEMA {scratch}"))?; + for table in ["pack", "object_location"] { + let (tablespace, options) = table_storage(tx, schema, table)?; + let mut sql = format!( + "CREATE TABLE {scratch}.{table} (LIKE {schema}.{table} INCLUDING STORAGE INCLUDING COMPRESSION INCLUDING CONSTRAINTS)" + ); + if !options.is_empty() { + sql.push_str(&format!(" WITH ({})", options.join(", "))); + } + if !tablespace.is_empty() { + sql.push_str(&format!(" TABLESPACE {}", quote_ident(&tablespace))); + } + tx.batch(&sql)?; + } + tx.batch(&format!( + "ALTER TABLE {scratch}.pack ADD PRIMARY KEY (pack_id)" + ))?; + Ok(()) + }) +} + +/// Phase A: copy retained rows in pack-id batches, each its own transaction; +/// a retried batch first deletes its own range, so the copy is idempotent. +fn build( + session: &Arc, + schema: &str, + scratch: &str, + mark: &MarkResult, + tuple: &GcTuple, +) -> HistoryResult { + create_scratch(session, schema, scratch)?; + let packs: Vec = mark.objects().packs().to_vec(); + let chunk = 512usize; + for batch in packs.chunks(chunk) { + let lo = batch[0]; + let hi = batch[batch.len() - 1]; + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {scratch}.pack WHERE pack_id BETWEEN {lo} AND {hi};\ + DELETE FROM {scratch}.object_location WHERE pack_id BETWEEN {lo} AND {hi};" + ))?; + tx.query( + &format!( + "INSERT INTO {scratch}.pack SELECT p.* FROM {schema}.pack p \ + WHERE p.pack_id BETWEEN $1 AND $2 \ + AND EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + vec![Param::I64(lo), Param::I64(hi)], + )?; + tx.query( + &format!( + "INSERT INTO {scratch}.object_location SELECT l.* FROM {schema}.object_location l \ + WHERE l.pack_id BETWEEN $1 AND $2 \ + AND EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=l.object_id)" + ), + vec![Param::I64(lo), Param::I64(hi)], + )?; + Ok(()) + })?; + } + stage(session, |tx| { + tx.batch(&format!( + "CREATE INDEX object_location_prefix ON {scratch}.object_location \ + ((('x' || encode(substring(object_id FROM 1 FOR 8), 'hex'))::bit(64)::bigint));\ + ALTER TABLE {scratch}.object_location \ + ADD FOREIGN KEY (pack_id) REFERENCES {scratch}.pack(pack_id);\ + ANALYZE {scratch}.pack; ANALYZE {scratch}.object_location;" + ))?; + Ok(()) + })?; + let (objects, distinct, pack_rows) = stage(session, |tx| { + let objects = scalar( + tx, + &format!("SELECT count(*) FROM {scratch}.object_location"), + Vec::new(), + )?; + let distinct = scalar( + tx, + &format!("SELECT count(DISTINCT object_id) FROM {scratch}.object_location"), + Vec::new(), + )?; + let pack_rows = scalar( + tx, + &format!("SELECT count(*) FROM {scratch}.pack"), + Vec::new(), + )?; + Ok((objects, distinct, pack_rows)) + })?; + if distinct != mark.objects().len() as i64 || objects != distinct { + return Err(HistoryError::Integrity("compaction object count")); + } + if pack_rows != mark.objects().packs().len() as i64 { + return Err(HistoryError::Integrity("compaction pack count")); + } + let (canonical, stored, metadata) = retained_totals(session, schema)?; + let dead = DeadTotals { + canonical: tuple.canonical - canonical, + stored: tuple.stored - stored, + metadata: tuple.metadata - metadata, + logical: 0, + }; + let scratch_bytes = relation_bytes(session, scratch, &["pack", "object_location"])?; + Ok(Built { + scratch: scratch.to_owned(), + scratch_bytes, + dead, + }) +} + +/// The §6.4 live-heads digest, computed inside one transaction (the mark's +/// SQL is the same vector). +fn digest(tx: &Transaction<'_>) -> HistoryResult<[u8; 32]> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let stacks = tx.query( + "SELECT s.layer_stack_id, s.head_layer_id FROM layer_stack s \ + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) \ + ORDER BY s.layer_stack_id", + Vec::new(), + )?; + for row in &stacks { + let id: Vec = row.get(0)?; + let head: Vec = row.get(1)?; + if id.len() != 17 || head.len() != 33 { + return Err(HistoryError::Integrity("live-heads stack width")); + } + hasher.update(&id); + hasher.update(&head); + } + let branches = tx.query( + "SELECT b.branch_id, b.head_commit_id, b.base_layer_id FROM branch b \ + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) \ + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) \ + ORDER BY b.branch_id", + Vec::new(), + )?; + for row in &branches { + let id: Vec = row.get(0)?; + let head: Option> = row.get(1)?; + let base: Vec = row.get(2)?; + if id.len() != 17 || base.len() != 33 { + return Err(HistoryError::Integrity("live-heads branch width")); + } + hasher.update(&id); + match head { + Some(head) if head.len() == 33 => hasher.update(&head), + Some(_) => return Err(HistoryError::Integrity("live-heads commit width")), + None => hasher.update(&[0u8; 33]), + }; + hasher.update(&base); + } + Ok(*hasher.finalize().as_bytes()) +} + +/// The strength order of §8 step 10: off < local < remote_write < on < +/// remote_apply. +fn sync_rank(setting: &str) -> u8 { + match setting { + "local" => 1, + "remote_write" => 2, + "on" => 3, + "remote_apply" => 4, + _ => 0, + } +} + +fn sync_name(rank: u8) -> &'static str { + match rank { + 1 => "local", + 2 => "remote_write", + 3 => "on", + 4 => "remote_apply", + _ => "off", + } +} + +/// §8 step 10: the swap waits iff the effective level is at least +/// `remote_write` and `synchronous_standby_names` is non-empty; then a named +/// standby must be streaming with `sync_state` sync or quorum. +fn standby_satisfied(rank: u8, names: &str, streaming: i64) -> bool { + if rank < 2 || names.trim().is_empty() { + return true; + } + streaming > 0 +} + +/// The effective `synchronous_commit` and whether the standby predicate +/// applies, read from the server and the Store's frozen setting. +fn effective_sync_rank(session: &Arc) -> HistoryResult { + stage(session, |tx| { + let rows = tx.query("SELECT current_setting('synchronous_commit')", Vec::new())?; + let server: String = rows[0].get(0)?; + let stored = session.settings.synchronous_commit.setting(); + Ok(sync_rank(&server).max(sync_rank(stored))) + }) +} + +/// §8 step 10: the standby predicate checked before Phase M. +fn standby_precheck(session: &Arc, rank: u8) -> HistoryResult<()> { + stage(session, |tx| { + let rows = tx.query( + "SELECT current_setting('synchronous_standby_names')", + Vec::new(), + )?; + let names: String = rows[0].get(0)?; + standby_check(tx, rank, &names) + }) +} + +fn standby_check(tx: &Transaction<'_>, rank: u8, names: &str) -> HistoryResult<()> { + if rank < 2 || names.trim().is_empty() { + return Ok(()); + } + let streaming = scalar( + tx, + "SELECT count(*) FROM pg_stat_replication WHERE state='streaming' AND sync_state IN ('sync','quorum')", + Vec::new(), + )?; + if !standby_satisfied(rank, names, streaming) { + return Err(HistoryError::Busy); + } + Ok(()) +} + +/// The recorded owner backend's state when it is still alive: `idle` means it +/// is not executing the op, so §9's dead-owner rule applies; any other state +/// is a live owner and stays `Busy`. +fn current_backend_pid(session: &Arc) -> HistoryResult { + scalar_session(session, "SELECT pg_backend_pid()", Vec::new()) +} + +fn owner_state(session: &Arc, op: &OpRow) -> HistoryResult> { + let (Some(pid), Some(start)) = (op.pid, op.backend_start.as_deref()) else { + return Ok(None); + }; + stage(session, |tx| { + let rows = tx.query( + "SELECT state::text FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2", + vec![Param::I64(pid), Param::Text(start.to_owned())], + )?; + rows.first() + .map(|row| row.get(0)) + .transpose() + .map_err(Into::into) + }) +} + +/// §12 PG privileges and reply: the two-argument `pg_terminate_backend(pid, +/// backend_start)` proof, scoped to this database and never the caller. Used +/// only for a recorded owner the server reports `idle` (it cannot finish the +/// op); a live owner is `Busy`, never killed. +fn terminate_owner(session: &Arc, op: &OpRow) -> HistoryResult { + let (Some(pid), Some(start)) = (op.pid, op.backend_start.as_deref()) else { + return Ok(false); + }; + stage(session, |tx| { + let rows = tx.query( + "SELECT pg_terminate_backend(a.pid, 30000) FROM pg_stat_activity a \ + WHERE a.pid=$1 AND a.backend_start::text=$2 AND a.datname=current_database() \ + AND a.pid<>pg_backend_pid()", + vec![Param::I64(pid), Param::Text(start.to_owned())], + )?; + let terminated: Option = rows.first().map(|row| row.get(0)).transpose()?; + Ok(terminated.unwrap_or(0) != 0) + }) +} + +fn drop_scratch(session: &Arc, scratch: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!("DROP SCHEMA IF EXISTS {scratch} CASCADE"))?; + Ok(()) + }) +} + +fn compaction_intent(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT op_id FROM {schema}.gc_intent WHERE kind='compaction' ORDER BY updated_ms LIMIT 1" + ), + Vec::new(), + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let bytes: Vec = row.get(0)?; + let op: [u8; 16] = bytes + .as_slice() + .try_into() + .map_err(|_| HistoryError::Integrity("compaction intent op"))?; + Ok(Some(op)) + }) +} + +/// §9: judge a recorded leftover. `G` removes the artifact and clears the +/// op; `G + 1` finishes Phase C and answers the installed outcome. `None` +/// means there was nothing to resume. +fn settle_leftover( + handles: &PostgresHandles, + session: &Arc, + schema: &str, +) -> HistoryResult> { + let Some(op) = read_op(session, schema)? else { + return Ok(None); + }; + let current = read_generation(session, schema)?; + if current == op.generation { + // The caller's own session is the recorded owner when it retries its + // own uncertain round: adopt it without terminating itself. + if op.pid != Some(current_backend_pid(session)?) { + if let Some(state) = owner_state(session, &op)? { + if state != "idle" { + return Err(HistoryError::Busy); + } + // The backend is alive but not executing the op: terminate it + // with the (pid, backend_start) pair (§12), then adopt it as a + // dead owner. + if !terminate_owner(session, &op)? { + return Err(HistoryError::Busy); + } + } + } + if let Some(artifact) = &op.artifact { + drop_scratch(session, artifact)?; + } + clear_round(session, schema)?; + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_intent WHERE kind='compaction'" + ))?; + Ok(()) + })?; + return Ok(None); + } + if current != op.generation + 1 { + return Err(HistoryError::Integrity("compaction op generation")); + } + let op_id = compaction_intent(session, schema)?; + cleanup(handles, session, schema, op.dead, op.totals_applied, op_id)?; + Ok(Some(report( + CompactPath::Rebuild, + u64::try_from(op.generation).unwrap_or(0), + u64::try_from(current).unwrap_or(0), + 0, + 0, + u64::try_from(op.dead.logical).unwrap_or(0), + 0, + 0, + CompactKeptWaste::default(), + 0, + CompactResidue::default(), + 0, + 0, + ))) +} + +/// Phase B: lock every base table, re-check the mark tuple, swap the two +/// payload tables, bump the generation and drop the scratch schema in one +/// short transaction. `55P03` is `Busy`; any tuple change is `Conflict` with +/// no writes. +fn swap( + _handles: &PostgresHandles, + session: &Arc, + schema: &str, + tuple: &GcTuple, + mark: &MarkResult, + built: &Built, +) -> HistoryResult { + let physical_before = relation_bytes(session, schema, &["pack", "object_location"])?; + let rank = effective_sync_rank(session)?; + let tables = tables_of(session, schema)?; + let locked = tables + .iter() + .map(|table| format!("{schema}.{}", quote_ident(table))) + .collect::>() + .join(", "); + let scratch = built.scratch.clone(); + let result = stage(session, |tx| -> HistoryResult { + tx.batch(&format!( + "SET LOCAL synchronous_commit TO {}", + sync_name(rank) + ))?; + tx.batch(&format!( + "LOCK TABLE {locked} IN ACCESS EXCLUSIVE MODE NOWAIT" + ))?; + // The tuple the mark recorded, re-read under the lock. Any publication + // after the mark moved one of these, so nothing is written. + let now = read_tuple(tx, schema)?; + if now != *tuple { + return Err(HistoryError::Conflict("Store moved after the mark")); + } + if digest(tx)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict("live heads moved after the mark")); + } + // Copy the owner (and a default ACL, when one exists) before the swap. + copy_owner(tx, schema, &scratch, "pack")?; + copy_owner(tx, schema, &scratch, "object_location")?; + tx.batch(&format!( + "DROP TABLE {schema}.object_location;\ + DROP TABLE {schema}.pack;\ + ALTER TABLE {scratch}.object_location SET SCHEMA {schema};\ + ALTER TABLE {scratch}.pack SET SCHEMA {schema};\ + CREATE TRIGGER pack_immutable BEFORE UPDATE OR DELETE ON {schema}.pack \ + FOR EACH ROW EXECUTE FUNCTION {schema}.pack_immutable();" + ))?; + verify_swap(tx, schema, mark, tuple, &built.dead)?; + let applied = scalar( + tx, + &format!("SELECT totals_applied FROM {schema}.gc_op WHERE id=1"), + Vec::new(), + )?; + if applied != 0 { + return Err(HistoryError::Integrity("compaction totals already applied")); + } + tx.query( + &format!( + "UPDATE {schema}.gc_op SET phase='swap',dead_canonical=$1,dead_stored=$2,dead_metadata=$3 WHERE id=1" + ), + vec![ + Param::I64(built.dead.canonical), + Param::I64(built.dead.stored), + Param::I64(built.dead.metadata), + ], + )?; + tx.query( + &format!( + "UPDATE {schema}.history_meta SET catalog_incarnation={} WHERE id=1", + tuple.generation + 1 + ), + Vec::new(), + )?; + tx.batch(&format!("DROP SCHEMA {scratch} CASCADE"))?; + let names: String = tx + .query( + "SELECT current_setting('synchronous_standby_names')", + Vec::new(), + )? + .first() + .ok_or(HistoryError::Integrity("standby names"))? + .get(0)?; + standby_check(tx, rank, &names)?; + Ok(Swapped { + physical_before, + physical_after: 0, + peak_temp: built.scratch_bytes, + dead: built.dead, + }) + }); + let result = match result { + Ok(swapped) => swapped, + Err(HistoryError::Busy) => { + // §8 step 5: roll back first, then drop the scratch schema in a + // new transaction, and report Busy with the holder detail. + let _ = drop_scratch(session, &scratch); + return Err(HistoryError::Busy); + } + Err(other) => return Err(other), + }; + let physical_after = relation_bytes(session, schema, &["pack", "object_location"])?; + Ok(Swapped { + physical_after, + ..result + }) +} + +fn tables_of(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + "SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relkind='r' ORDER BY c.relname", + vec![Param::Text(schema.to_owned())], + )?; + rows.iter() + .map(|row| row.get::(0).map_err(HistoryError::from)) + .collect() + }) +} + +fn read_tuple(tx: &Transaction<'_>, schema: &str) -> HistoryResult { + let rows = tx.query( + &format!( + "SELECT h.catalog_incarnation,h.next_history_serial,p.next_pack_id,p.next_ordinal,\ + t.logical,t.canonical,t.stored,t.metadata,\ + (SELECT COALESCE(MAX(stamp),0) FROM {schema}.content_signature),\ + (SELECT COUNT(*) FROM {schema}.\"commit\")+(SELECT COUNT(*) FROM {schema}.layer) \ + FROM {schema}.history_meta h, {schema}.store_policy p, {schema}.store_totals t \ + WHERE h.id=1 AND p.id=1 AND t.id=1" + ), + Vec::new(), + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("Store tuple")); + } + let row = &rows[0]; + Ok(GcTuple { + generation: row.get(0)?, + next_history_serial: row.get(1)?, + next_pack_id: row.get(2)?, + next_ordinal: row.get(3)?, + logical: row.get(4)?, + canonical: row.get(5)?, + stored: row.get(6)?, + metadata: row.get(7)?, + stamp: row.get(8)?, + history_rows: row.get(9)?, + }) +} + +fn copy_owner(tx: &Transaction<'_>, schema: &str, scratch: &str, table: &str) -> HistoryResult<()> { + let rows = tx.query( + "SELECT pg_get_userbyid(c.relowner)::text, COALESCE(array_to_string(c.relacl,E'\\n'),'') \ + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname=$2", + vec![ + Param::Text(schema.to_owned()), + Param::Text(table.to_owned()), + ], + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction owner")); + } + let owner: String = rows[0].get(0)?; + tx.batch(&format!( + "ALTER TABLE {scratch}.{} OWNER TO {}", + quote_ident(table), + quote_ident(&owner) + ))?; + let acl: String = rows[0].get(1)?; + for line in acl.lines().filter(|line| !line.is_empty()) { + if let Some(grant) = grant_of(line, schema, table) { + tx.batch(&grant)?; + } + } + Ok(()) +} + +/// One `relacl` entry as a `GRANT` on `schema.table`, or `None` when the +/// entry is the owner's own implicit grant (`owner=arwdDxt/owner`). +fn grant_of(line: &str, schema: &str, table: &str) -> Option { + let (grantee, rest) = line.split_once('=')?; + let (privileges, grantor) = rest.split_once('/')?; + if grantee.is_empty() { + return None; + } + // The owner's implicit self-grant is re-created by ALTER TABLE OWNER. + if grantee == grantor { + return None; + } + let names: Vec<&str> = privileges + .chars() + .filter_map(|letter| { + Some(match letter { + 'r' => "SELECT", + 'w' => "UPDATE", + 'a' => "INSERT", + 'd' => "DELETE", + 'D' => "TRUNCATE", + 'x' => "REFERENCES", + 't' => "TRIGGER", + _ => return None, + }) + }) + .collect(); + if names.is_empty() { + return None; + } + let target = if grantee.is_empty() { + "PUBLIC".to_owned() + } else { + quote_ident(grantee) + }; + Some(format!( + "GRANT {} ON TABLE {schema}.{} TO {target}", + names.join(","), + quote_ident(table) + )) +} + +fn verify_swap( + tx: &Transaction<'_>, + schema: &str, + mark: &MarkResult, + tuple: &GcTuple, + dead: &DeadTotals, +) -> HistoryResult<()> { + let objects = scalar( + tx, + &format!("SELECT count(*) FROM {schema}.object_location"), + Vec::new(), + )?; + let distinct = scalar( + tx, + &format!("SELECT count(DISTINCT object_id) FROM {schema}.object_location"), + Vec::new(), + )?; + let packs = scalar( + tx, + &format!("SELECT count(*) FROM {schema}.pack"), + Vec::new(), + )?; + if objects != mark.objects().len() as i64 + || distinct != objects + || packs != mark.objects().packs().len() as i64 + { + return Err(HistoryError::Integrity("compaction swapped count")); + } + let canonical = scalar( + tx, + &format!("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {schema}.object_location"), + Vec::new(), + )?; + let stored = scalar( + tx, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack"), + Vec::new(), + )?; + let metadata = scalar( + tx, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack WHERE domain=0"), + Vec::new(), + )?; + if canonical != tuple.canonical - dead.canonical + || stored != tuple.stored - dead.stored + || metadata != tuple.metadata - dead.metadata + { + return Err(HistoryError::Integrity("compaction swapped totals")); + } + let logical = scalar( + tx, + &format!("SELECT logical FROM {schema}.store_totals WHERE id=1"), + Vec::new(), + )?; + let sum = scalar( + tx, + &format!( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.layer)" + ), + Vec::new(), + )?; + if logical != sum { + return Err(HistoryError::Integrity( + "Store totals disagree with the history rows", + )); + } + if digest(tx)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict("live heads moved after the mark")); + } + Ok(()) +} + +/// Phase C: delete only the persisted `gc_dead` members, settle the totals +/// once, and clear the op. The Store is usable at `G + 1` throughout. +fn cleanup( + handles: &PostgresHandles, + session: &Arc, + schema: &str, + dead: DeadTotals, + totals_applied: bool, + op_id: Option<[u8; 16]>, +) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.\"commit\" c WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='commit' AND d.id=c.commit_id);\ + DELETE FROM {schema}.layer l WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='layer' AND d.id=l.layer_id);\ + DELETE FROM {schema}.branch b WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='branch' AND d.id=b.branch_id);\ + DELETE FROM {schema}.layer_stack s WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='stack' AND d.id=s.layer_stack_id);\ + DELETE FROM {schema}.layer_lineage ll WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer l WHERE l.layer_id=ll.layer_id);\ + DELETE FROM {schema}.commit_lineage cl WHERE NOT EXISTS (SELECT 1 FROM {schema}.\"commit\" c WHERE c.commit_id=cl.commit_id);\ + DELETE FROM {schema}.absorbed a WHERE NOT EXISTS (SELECT 1 FROM {schema}.commit_lineage cl WHERE cl.run=a.run);\ + DELETE FROM {schema}.reach r WHERE r.key NOT IN (SELECT layer_stack_id FROM {schema}.layer_stack UNION SELECT run FROM {schema}.commit_lineage);\ + DELETE FROM {schema}.branch_retired br WHERE NOT EXISTS (SELECT 1 FROM {schema}.branch b WHERE b.branch_id=br.branch_id);\ + DELETE FROM {schema}.layer_stack_retired lr WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=lr.layer_stack_id);\ + DELETE FROM {schema}.path_removed pr WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=pr.layer_stack_id);\ + DELETE FROM {schema}.project_epoch pe WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=pe.layer_stack_id);\ + DELETE FROM {schema}.content_signature cs WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='signature' AND d.id=int8send(cs.slot));\ + DELETE FROM {schema}.file_digest fd WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='digest' AND d.id=fd.digest);\ + DELETE FROM {schema}.binding_hint_block;\ + DELETE FROM {schema}.init_operation;\ + DELETE FROM {schema}.init_entry;\ + DELETE FROM {schema}.init_host_file;" + ))?; + if !totals_applied { + tx.query( + &format!( + "UPDATE {schema}.store_totals SET \ + canonical=canonical-$1,stored=stored-$2,metadata=metadata-$3,logical=logical-$4 WHERE id=1" + ), + vec![ + Param::I64(dead.canonical), + Param::I64(dead.stored), + Param::I64(dead.metadata), + Param::I64(dead.logical), + ], + )?; + tx.query( + &format!( + "UPDATE {schema}.gc_op SET totals_applied=TRUE,phase='cleanup' WHERE id=1" + ), + Vec::new(), + )?; + } + let totals = tx.query( + &format!( + "SELECT logical,canonical,stored,metadata FROM {schema}.store_totals WHERE id=1" + ), + Vec::new(), + )?; + let logical: i64 = totals[0].get(0)?; + let canonical: i64 = totals[0].get(1)?; + let stored: i64 = totals[0].get(2)?; + let metadata: i64 = totals[0].get(3)?; + let sums = tx.query( + &format!( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.layer),\ + (SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {schema}.object_location),\ + (SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack),\ + (SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack WHERE domain=0)" + ), + Vec::new(), + )?; + let sum_logical: i64 = sums[0].get(0)?; + let sum_canonical: i64 = sums[0].get(1)?; + let sum_stored: i64 = sums[0].get(2)?; + let sum_metadata: i64 = sums[0].get(3)?; + if logical != sum_logical + || canonical != sum_canonical + || stored != sum_stored + || metadata != sum_metadata + { + return Err(HistoryError::Integrity("compaction settled totals")); + } + tx.batch(&format!( + "DELETE FROM {schema}.gc_dead; DELETE FROM {schema}.gc_op;" + ))?; + Ok(()) + })?; + if let Some(op) = op_id { + let _ = handles.history.gc_release(&op); + } else { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_intent WHERE kind='compaction'" + ))?; + Ok(()) + })?; + } + Ok(()) +} + +/// One round after admission: mark, decide, build, swap, cleanup. +fn round( + handles: &PostgresHandles, + session: &Arc, + schema: &str, + request: &CompactRequest, +) -> HistoryResult { + let (mark, tuple) = handles.history.mark_gc()?; + load_visited(session, schema, &mark)?; + let stored = u64::try_from(total_stored(session, schema)?).unwrap_or(0); + let live = u64::try_from(live_stored(session, schema)?).unwrap_or(0); + let dead_bytes = stored.saturating_sub(live); + let scratch = scratch_name(&request.op_id); + let mark_dead = persist_mark(session, schema, &mark, &tuple, &scratch, request)?; + let generation = u64::try_from(tuple.generation).unwrap_or(0); + let kept = CompactKeptWaste { + dead_in_retained_packs_bytes: mark.objects().waste().dead_in_retained_packs_bytes(), + delta_bases_bytes: mark.objects().waste().delta_bases_bytes(), + delta_base_references: mark + .objects() + .waste() + .delta_base_references() + .iter() + .map(|id| *id.as_bytes()) + .collect(), + inode_blocks_bytes: mark.objects().waste().inode_blocks_bytes(), + }; + let rebuild = stored > 0 && dead_bytes.saturating_mul(THRESHOLD) >= stored; + if !rebuild { + clear_round(session, schema)?; + let _ = handles.history.gc_release(&request.op_id); + return Ok(report( + CompactPath::None, + generation, + generation, + dead_bytes, + dead_bytes, + 0, + 0, + 0, + kept, + 0, + CompactResidue::default(), + stored, + mark.history().len() as u64, + )); + } + stop( + request, + CompactFailpoint::PostgresAfterMark, + session, + schema, + "mark", + )?; + let mut built = build(session, schema, &scratch, &mark, &tuple)?; + built.dead.logical = mark_dead.logical; + stop( + request, + CompactFailpoint::PostgresAfterBuild, + session, + schema, + "build", + )?; + let swapped = swap(handles, session, schema, &tuple, &mark, &built)?; + stop( + request, + CompactFailpoint::PostgresAfterSwap, + session, + schema, + "swap", + )?; + let logical_discarded = u64::try_from(swapped.dead.logical).unwrap_or(0); + cleanup( + handles, + session, + schema, + swapped.dead, + false, + Some(request.op_id), + )?; + stop( + request, + CompactFailpoint::PostgresAfterCleanup, + session, + schema, + "cleanup", + )?; + let shrink = swapped + .physical_before + .saturating_sub(swapped.physical_after); + Ok(report( + CompactPath::Rebuild, + generation, + generation + 1, + dead_bytes, + dead_bytes, + logical_discarded, + 0, + shrink, + kept, + swapped.peak_temp, + CompactResidue::default(), + stored, + mark.history().len() as u64, + )) +} + +#[allow(clippy::too_many_arguments)] +fn report( + path: CompactPath, + generation_before: u64, + generation_after: u64, + dead_bytes: u64, + dead_refined: u64, + logical_discarded: u64, + backend_reusable: u64, + physical_shrink: u64, + kept_waste: CompactKeptWaste, + peak_temp_bytes: u64, + residue: CompactResidue, + total_stored: u64, + history_rows: u64, +) -> CompactReceipt { + let mut zero_reasons = Vec::new(); + if dead_bytes == 0 { + let reason = if total_stored == 0 { + "empty-store" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "deadBytesBound", + reason, + }); + zero_reasons.push(CompactZeroReason { + field: "deadBytesRefined", + reason, + }); + } + if logical_discarded == 0 { + let reason = if history_rows == 0 { + "no-history" + } else if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "logicalDiscarded", + reason, + }); + } + if physical_shrink == 0 { + zero_reasons.push(CompactZeroReason { + field: "physicalShrink", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + zero_reasons.push(CompactZeroReason { + field: "backendReusable", + reason: "postgresql-in-place", + }); + if kept_waste.is_empty() { + zero_reasons.push(CompactZeroReason { + field: "keptWaste", + reason: "no-waste", + }); + } + if peak_temp_bytes == 0 { + zero_reasons.push(CompactZeroReason { + field: "peakTempBytes", + reason: "none-path", + }); + } + CompactReceipt { + path, + generation_before, + generation_after, + dead_bytes_bound: dead_bytes, + dead_bytes_refined: dead_refined, + logical_discarded, + backend_reusable, + physical_shrink, + kept_waste, + peak_temp_bytes, + residue, + estimate: true, + zero_reasons, + } +} + +impl PostgresHandles { + /// Runs one §8 PostgreSQL Store compaction on this Store's session. + /// + /// The call takes the Store's exclusive intent (a session advisory key) + /// through cleanup, marks at `G`, and on the rebuild path copies the two + /// payload tables, swaps them in one short transaction at `G + 1`, and + /// deletes only the persisted dead set. The caller keeps its session; the + /// Store is usable at `G + 1` from the swap on. + pub fn compact(&self, request: &CompactRequest) -> HistoryResult { + let session = self.storage.session.clone(); + let schema = session.settings.schema.clone(); + let _ = read_generation(&session, &schema)?; + if let Some(receipt) = settle_leftover(self, &session, &schema)? { + return Ok(receipt); + } + let intent = GcIntent::operation( + request.op_id, + GcIntentKind::Compaction, + GcScopeKind::Store, + self.history.catalog_id.as_slice().to_vec(), + ) + .owned( + request.owner_runtime.as_deref(), + request.owner_boot.as_deref(), + ) + .with_phase(Some("mark")); + self.history.gc_register(&intent)?; + let result = (|| { + let rank = effective_sync_rank(&session)?; + standby_precheck(&session, rank)?; + round(self, &session, &schema, request) + })(); + if result.is_err() { + let _ = self.history.gc_release(&request.op_id); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::{standby_satisfied, sync_name, sync_rank}; + + #[test] + fn the_standby_predicate_waits_only_for_a_named_standby() { + // Below remote_write, or with no names, no standby is required. + assert!(standby_satisfied(0, "", 0)); + assert!(standby_satisfied(1, "", 0)); + assert!(standby_satisfied(2, "", 0)); + assert!(standby_satisfied(4, " ", 0)); + // At or above remote_write with names, a streaming sync/quorum + // standby must exist. + assert!(!standby_satisfied(2, "FIRST 1 (s1)", 0)); + assert!(!standby_satisfied(4, "ANY 1 (s1)", 0)); + assert!(standby_satisfied(2, "FIRST 1 (s1)", 1)); + assert!(standby_satisfied(3, "ANY 2 (s1,s2)", 1)); + } + + #[test] + fn synchronous_commit_strength_is_the_spec_order() { + assert_eq!(sync_rank("off"), 0); + assert_eq!(sync_rank("local"), 1); + assert_eq!(sync_rank("remote_write"), 2); + assert_eq!(sync_rank("on"), 3); + assert_eq!(sync_rank("remote_apply"), 4); + assert_eq!(sync_name(4), "remote_apply"); + assert_eq!(sync_name(0), "off"); + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs index bdc010b348..bc66630699 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs @@ -12,6 +12,9 @@ pub(crate) fn statement(key: &str) -> Result<&'static str, BackendError> { "statistics_totals" => Ok(include_str!( "../../../../sql/postgresql/queries/history/statistics_totals.sql" )), + "gc_epoch_read" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/gc_epoch_read.sql" + )), "gc_epoch_share" => Ok(include_str!( "../../../../sql/postgresql/queries/history/gc_epoch_share.sql" )), diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs index 98439899fc..74225ff555 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs @@ -2,6 +2,7 @@ //! lane, objects, metadata and history statements, and Init's acquisition. pub(crate) mod access; pub(crate) mod acquisition; +pub(crate) mod compact; pub(crate) mod history; pub(crate) mod lane; mod mechanics; diff --git a/engine/crates/layerfs-persistence/src/backend/records.rs b/engine/crates/layerfs-persistence/src/backend/records.rs index 9efc76cd55..4cfb62853d 100644 --- a/engine/crates/layerfs-persistence/src/backend/records.rs +++ b/engine/crates/layerfs-persistence/src/backend/records.rs @@ -118,6 +118,8 @@ pub(crate) enum BackendError { #[cfg_attr(not(target_os = "linux"), allow(dead_code))] Cancelled, ReadOnly, + /// §10: the Store path no longer names the file this handle opened. + Stale, Capacity, Integrity, /// A SQLite Store of schema version 1 (before #91): refused at open with @@ -158,6 +160,8 @@ impl From for layerfs_history::HistoryError { Self::OwnershipUnavailable } BackendError::ReadOnly => Self::ContinuityUnavailable, + // §10: the file was replaced or moved under this handle. + BackendError::Stale => Self::StaleStore, BackendError::Capacity => Self::Capacity("persistence"), BackendError::Integrity => Self::Integrity("persistence"), BackendError::SchemaVersion1 => Self::Integrity(SCHEMA_VERSION_1), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs new file mode 100644 index 0000000000..61c3b99aef --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs @@ -0,0 +1,1967 @@ +//! §7 SQLite Store compaction: mark, rebuild, immutable verification and the +//! atomic install. +//! +//! One call takes the Store's writer lane before it registers its admission +//! intent, runs the T7.2 mark in the same exclusive window, seals the file +//! (checkpoint, checked close, sidecar check and `(size, mtime, change +//! counter)` capture), and then either stops on the §3 `none` path or builds +//! `.compact`, verifies it through an `immutable=1` URI, and renames it +//! into place at `G → G+1`. The run's phase and mark tuple are kept beside the +//! Store in `.compact-op`, the SQLite equivalent of the PostgreSQL +//! `gc_op` row (§11); the next `store.compact` judges a leftover by the Store +//! generation (§9) and removes the artifact set it names. +//! +//! The T7.2 mark already resolves every logical reference and delta base, so +//! the bytes of packs that hold no reached object are the refined dead bytes: +//! §7's step-4 payload sweep cannot discover another live object on SQLite. +//! `deadBytesRefined` therefore equals `deadBytesBound`, as §11 requires for +//! the `none` path. + +use super::lifecycle::seal::SealedFacts; +use super::schema; +use super::session::connection::{Session, SqlWork}; +use super::session::rows; +use crate::backend::records::BackendError; +use crate::{KeptWaste, MarkResult, SqliteHandles}; +use layerfs_content::ObjectId; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::HistoryCatalog; +use layerfs_storage::port::PersistenceError; +use rusqlite::{params, Connection, OpenFlags}; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// The sidecar record suffix (§11: SQLite keeps its phase outside the sealed +/// database). +const OP_SUFFIX: &str = ".compact-op"; +/// The staged rebuild Store beside the live one. +const COMPACT_SUFFIX: &str = ".compact"; +/// The §6.3 visited scratch database. +const VISITED_SUFFIX: &str = ".compact-visited"; +/// The full SQLite artifact set of §9, without the variable `-mj*` journals. +const ARTIFACT_SUFFIXES: [&str; 7] = [ + COMPACT_SUFFIX, + ".compact-journal", + ".compact-wal", + ".compact-shm", + VISITED_SUFFIX, + ".compact-visited-journal", + ".compact-vacuum", +]; + +/// The mark tuple a compaction records before its first checkpoint (§8 Phase M +/// on PostgreSQL; the same first-statement tuple in the SQLite sidecar here). +#[derive(Clone, Debug, Eq, PartialEq)] +struct MarkTuple { + catalog_id: Vec, + generation: i64, + next_history_serial: i64, + next_pack_id: i64, + next_ordinal: i64, + logical: i64, + canonical: i64, + stored: i64, + metadata: i64, + stamp: i64, + history_rows: i64, +} + +/// The first-statement tuple, in the column order every reader decodes. +const TUPLE_SQL: &str = "SELECT h.catalog_id,h.catalog_incarnation,h.next_history_serial,\ +p.next_pack_id,p.next_ordinal,t.logical,t.canonical,t.stored,t.metadata,\ +(SELECT COALESCE(MAX(stamp),0) FROM content_signature),\ +(SELECT COUNT(*) FROM \"commit\")+(SELECT COUNT(*) FROM layer) \ +FROM history_meta h, store_policy p, store_totals t \ +WHERE h.id=1 AND p.id=1 AND t.id=1"; + +/// The §6.4 live-heads digest input, in identity order. +const LIVE_STACKS: &str = "SELECT s.layer_stack_id,s.head_layer_id FROM layer_stack s \ +WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) \ +ORDER BY s.layer_stack_id"; +/// The marker-free Branches, in identity order. +const LIVE_BRANCHES: &str = "SELECT b.branch_id,b.head_commit_id,b.base_layer_id FROM branch b \ +WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) \ +AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) \ +ORDER BY b.branch_id"; + +/// Which §3 path a compaction took. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CompactPath { + /// Below the 10% wholly-dead threshold: mark only, nothing written. + None, + /// A rebuild installed at `G → G+1`. + Rebuild, +} + +impl CompactPath { + /// The report's `path` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Rebuild => "rebuild", + } + } +} + +/// A test-only stop point of the run, so a crash between phases can be +/// exercised. The run leaves the state a crash would leave and reports an +/// unknown outcome; the next `store.compact` must recover it (§9). +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CompactFailpoint { + /// Before the seal, while the open session, the writer lane and the + /// admission intent are still held. + BeforeSeal, + /// After the seal returns and before the sealed file is read back. + BeforeSealRead, + /// After the seal and the sealed-state comparison, before any artifact. + AfterSeal, + /// Inside the build, before the visited scratch or the copy is written. + BeforeCopy, + /// Inside the build, after `VACUUM INTO` produced `.compact` and + /// before the copy is pruned: the state a crash during the copy leaves. + DuringCopy, + /// After the staged `.compact` is built and compacted. + AfterBuild, + /// The staged Store's `VACUUM INTO` meets a bounded filesystem (ENOSPC). + CapacityAtBuild, + /// After the staged file is switched to WAL and checkpointed. + AfterWalSwitch, + /// Inside the verification, after the schema and row-count checks. + DuringVerify, + /// After the immutable verification passes. + AfterVerify, + /// Immediately before the rename would run. + BeforeInstall, + /// After the rename and before the directory fsync (rename/journal). + DuringRename, + /// After the rename and directory fsync. + AfterInstall, + /// Inside the settle, after the artifact set is removed and before the op + /// record. + DuringSettle, + /// PostgreSQL: after Phase M wrote the op and the dead set, before Phase A. + PostgresAfterMark, + /// PostgreSQL: after Phase A built and verified the scratch tables. + PostgresAfterBuild, + /// PostgreSQL: after Phase B committed the swap, before Phase C. + PostgresAfterSwap, + /// PostgreSQL: after Phase C settled the op. + PostgresAfterCleanup, +} + +impl CompactFailpoint { + /// The phase the sidecar records for this stop point. + const fn phase(self) -> &'static str { + match self { + Self::BeforeSeal | Self::BeforeSealRead | Self::AfterSeal => "mark", + Self::BeforeCopy + | Self::DuringCopy + | Self::AfterBuild + | Self::CapacityAtBuild + | Self::AfterWalSwitch => "build", + Self::DuringVerify | Self::AfterVerify | Self::BeforeInstall => "verify", + Self::DuringRename | Self::AfterInstall | Self::DuringSettle => "install", + Self::PostgresAfterMark => "mark", + Self::PostgresAfterBuild => "build", + Self::PostgresAfterSwap => "swap", + Self::PostgresAfterCleanup => "cleanup", + } + } +} + +/// Inputs one `store.compact` attempt uses. +#[derive(Clone, Debug, Default)] +pub struct CompactRequest { + /// 16-byte operation identity, written as the admission intent's `op_id`. + pub op_id: [u8; 16], + /// Owning runtime token, for the op and intent records. + pub owner_runtime: Option, + /// Owning boot token, for the op and intent records. + pub owner_boot: Option, + /// The owning process id, so `store.status` can tell a live run from a + /// leftover (§11). The kernel start time and boot id, when known, keep a + /// reused id from passing as the owner. + pub owner_pid: Option, + /// The owner's kernel start time (the lease's second field). + pub owner_pid_start: Option, + /// The owner's kernel boot id (the lease's third field). + pub owner_pid_boot: Option, + /// A test-only stop point; production callers leave it `None`. + pub failpoint: Option, + /// A test-only hook called after verification and immediately before the + /// §10 install fence, so a test can replace the Store under the sealed + /// handle. Production callers leave it `None`. + #[doc(hidden)] + pub before_install_hook: Option, +} + +/// The `keptWaste` §11 part of a compaction report. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CompactKeptWaste { + /// Bytes of unreferenced records inside packs the mark retained. + pub dead_in_retained_packs_bytes: u64, + /// Bytes of objects reached only as a delta base. + pub delta_bases_bytes: u64, + /// The objects counted by [`Self::delta_bases_bytes`], in ascending order. + pub delta_base_references: Vec<[u8; 32]>, + /// Bytes of `inode_value_block` rows, which 2.0 keeps in full. + pub inode_blocks_bytes: u64, +} + +impl CompactKeptWaste { + /// One mark's kept-waste accounting. + fn from_mark(waste: &KeptWaste) -> Self { + Self { + dead_in_retained_packs_bytes: waste.dead_in_retained_packs_bytes(), + delta_bases_bytes: waste.delta_bases_bytes(), + delta_base_references: waste + .delta_base_references() + .iter() + .map(|id| *id.as_bytes()) + .collect(), + inode_blocks_bytes: waste.inode_blocks_bytes(), + } + } + + /// Whether every part is zero. + #[must_use] + pub fn is_empty(&self) -> bool { + self.dead_in_retained_packs_bytes == 0 + && self.delta_bases_bytes == 0 + && self.inode_blocks_bytes == 0 + } +} + +/// The `residue` §11 part of a compaction report. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CompactResidue { + /// Bytes left behind by the artifact set after settle. + pub bytes: u64, + /// The artifact file names that remain. + pub artifacts: Vec, + /// Why anything remains, when it does. + pub reason: Option, +} + +/// One zero report field and its §11 `zeroReason`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompactZeroReason { + /// The report field the reason explains. + pub field: &'static str, + /// One word of the §11 closed vocabulary. + pub reason: &'static str, +} + +impl CompactZeroReason { + /// The §10 wire key this reason names: the §11 logical name rendered + /// snake_case, the same mapping the report serializer uses. + #[must_use] + pub fn wire_field(self) -> &'static str { + match self.field { + "deadBytesBound" => "dead_bytes_bound", + "deadBytesRefined" => "dead_bytes_refined", + "logicalDiscarded" => "logical_discarded", + "backendReusable" => "backend_reusable", + "physicalShrink" => "physical_shrink", + "keptWaste" => "kept_waste", + "peakTempBytes" => "peak_temp_bytes", + other => other, + } + } +} + +/// The §11 report `store.compact` returns. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactReceipt { + /// `none` or `rebuild`. + pub path: CompactPath, + /// The generation the mark ran at. + pub generation_before: u64, + /// The installed generation: equal on `none`. + pub generation_after: u64, + /// Bytes in wholly dead packs after the metadata walk; the path's input. + pub dead_bytes_bound: u64, + /// The same bytes after the payload sweep. + pub dead_bytes_refined: u64, + /// Collected history rows' logical bytes. + pub logical_discarded: u64, + /// SQLite free pages of the new (rebuild) or live (`none`) Store. + pub backend_reusable: u64, + /// Old file size minus new. + pub physical_shrink: u64, + /// The bytes the mark keeps that no retained root needs. + pub kept_waste: CompactKeptWaste, + /// Peak temporary bytes the run needed. + pub peak_temp_bytes: u64, + /// The artifact set left after settle; empty on success. + pub residue: CompactResidue, + /// §13: the report is an estimate, not a measurement. + pub estimate: bool, + /// One `zeroReason` per zero report field. + pub zero_reasons: Vec, +} + +/// One recorded compaction run beside the Store. +#[derive(Clone, Debug)] +struct RunOp { + op_id: [u8; 16], + generation: i64, + phase: String, + started_ms: i64, + owner_pid: Option, + owner_pid_start: Option, + owner_pid_boot: Option, + tuple: MarkTuple, + live_heads_digest: [u8; 32], +} + +impl RunOp { + fn encode(&self) -> String { + format!( + "version=1\nop_id={}\ngeneration={}\nphase={}\nstarted_ms={}\n\ +owner_pid={}\nowner_pid_start={}\nowner_pid_boot={}\n\ +catalog_id={}\nnext_history_serial={}\nnext_pack_id={}\nnext_ordinal={}\n\ +mark_serial={}\nmark_pack_id={}\nmark_digest={}\nmark_totals={},{},{},{}\nmark_stamp={}\nhistory_rows={}\n\ +live_heads_digest={}\n", + hex(&self.op_id), + self.generation, + self.phase, + self.started_ms, + self.owner_pid.map_or_else(String::new, |pid| pid.to_string()), + self.owner_pid_start.map_or_else(String::new, |start| start.to_string()), + self.owner_pid_boot.map_or_else(String::new, |boot| format!("{boot:032x}")), + hex(&self.tuple.catalog_id), + self.tuple.next_history_serial, + self.tuple.next_pack_id, + self.tuple.next_ordinal, + self.tuple.next_history_serial.saturating_sub(1), + self.tuple.next_pack_id, + hex(&self.live_heads_digest), + self.tuple.logical, + self.tuple.canonical, + self.tuple.stored, + self.tuple.metadata, + self.tuple.stamp, + self.tuple.history_rows, + hex(&self.live_heads_digest), + ) + } + + fn write(&self, store: &Path) -> HistoryResult<()> { + let path = op_path(store); + let mut temporary = path.as_os_str().to_os_string(); + temporary.push(".tmp"); + let temporary = PathBuf::from(temporary); + std::fs::write(&temporary, self.encode()).map_err(io_error)?; + if let Ok(file) = std::fs::File::open(&temporary) { + let _ = file.sync_all(); + } + std::fs::rename(&temporary, &path).map_err(io_error)?; + if let Some(parent) = path.parent() { + let _ = std::fs::File::open(parent).and_then(|directory| directory.sync_all()); + } + Ok(()) + } + + fn read(store: &Path) -> Option { + let text = std::fs::read_to_string(op_path(store)).ok()?; + let mut values = BTreeMap::new(); + for line in text.lines() { + let (key, value) = line.split_once('=')?; + values.insert(key, value); + } + if values.get("version") != Some(&"1") { + return None; + } + let op_id = unhex::<16>(values.get("op_id")?)?; + let generation: i64 = values.get("generation")?.parse().ok()?; + let phase = (*values.get("phase")?).to_owned(); + let started_ms: i64 = values.get("started_ms")?.parse().ok()?; + let owner_pid = values + .get("owner_pid") + .filter(|text| !text.is_empty()) + .and_then(|text| text.parse().ok()); + let owner_pid_start = values + .get("owner_pid_start") + .filter(|text| !text.is_empty()) + .and_then(|text| text.parse().ok()); + let owner_pid_boot = values + .get("owner_pid_boot") + .filter(|text| !text.is_empty()) + .and_then(|text| u128::from_str_radix(text, 16).ok()); + let catalog_id = unhex_vec(values.get("catalog_id")?)?; + let tuple = MarkTuple { + catalog_id, + generation, + next_history_serial: values.get("next_history_serial")?.parse().ok()?, + next_pack_id: values.get("next_pack_id")?.parse().ok()?, + next_ordinal: values.get("next_ordinal")?.parse().ok()?, + logical: values.get("mark_totals")?.split(',').next()?.parse().ok()?, + canonical: values.get("mark_totals")?.split(',').nth(1)?.parse().ok()?, + stored: values.get("mark_totals")?.split(',').nth(2)?.parse().ok()?, + metadata: values.get("mark_totals")?.split(',').nth(3)?.parse().ok()?, + stamp: values.get("mark_stamp")?.parse().ok()?, + history_rows: values.get("history_rows")?.parse().ok()?, + }; + let live_heads_digest = unhex::<32>(values.get("live_heads_digest")?)?; + Some(Self { + op_id, + generation, + phase, + started_ms, + owner_pid, + owner_pid_start, + owner_pid_boot, + tuple, + live_heads_digest, + }) + } +} + +/// The operation a Store's `.compact-op` records, as `store.status` +/// (T10.1) projects it: the operation identity, the generation the run started +/// at, its phase, its owner and whether its outcome is known. A settled run +/// removes the record, so [`compact_op_status`] returns `None`. +#[doc(hidden)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactOpStatus { + /// The 16-byte operation identity. + pub op_id: [u8; 16], + /// The generation the run started at. + pub generation: i64, + /// The recorded phase: `admission`, `mark`, `build`, `verify` or `install`. + pub phase: String, + /// When the run started, in Unix milliseconds. + pub started_ms: i64, + /// The owning process id, when the run recorded one. + pub owner_pid: Option, + /// The owner's kernel start time, when recorded. + pub owner_pid_start: Option, + /// The owner's kernel boot id, when recorded. + pub owner_pid_boot: Option, + /// True while the record remains: the run did not settle, so its outcome + /// is unknown. `store.status` reports `busy` while the owner lives and + /// `unresolved` once it is gone. + pub uncertain: bool, +} + +impl CompactOpStatus { + /// The `store.status` projection of this operation; `alive` says whether + /// the owning process is still running. A live owner is `busy`, a gone one + /// left an uncertain `unresolved` outcome. `uncertain` stays the record's + /// own fact: it is true for every unsettled run, live or leftover. + #[must_use] + pub fn status(&self, alive: bool) -> crate::report::CompactStatus { + // A read in the admission window still projects a §11 phase: the run + // is about to mark, and `admission` is not one of the four. + let phase = if self.phase == "admission" { + "mark".to_owned() + } else { + self.phase.clone() + }; + crate::report::CompactStatus { + id: format!("op-{}", hex(&self.op_id)), + name: "store.compact", + phase, + started: self.started_ms, + uncertain: self.uncertain, + state: if alive { "busy" } else { "unresolved" }, + phases: &crate::report::SQLITE_COMPACT_PHASES, + } + } +} + +/// Reads the operation `.compact-op` records, or `None` when the Store +/// has no compaction record. `store.status` (T10.1) uses this to show the +/// operation, its phase and its uncertain outcome; a settled run removes the +/// record (§9). +#[doc(hidden)] +#[must_use] +pub fn compact_op_status(store: &Path) -> Option { + RunOp::read(store).map(|op| CompactOpStatus { + op_id: op.op_id, + generation: op.generation, + phase: op.phase, + started_ms: op.started_ms, + owner_pid: op.owner_pid, + owner_pid_start: op.owner_pid_start, + owner_pid_boot: op.owner_pid_boot, + uncertain: true, + }) +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn unhex_vec(text: &str) -> Option> { + if text.len() % 2 != 0 { + return None; + } + (0..text.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&text[index..index + 2], 16).ok()) + .collect() +} + +fn unhex(text: &str) -> Option<[u8; N]> { + let bytes = unhex_vec(text)?; + bytes.try_into().ok() +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, COMPACT_SUFFIX) +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, VISITED_SUFFIX) +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, OP_SUFFIX) +} + +/// The §9 SQLite artifact set: the fixed suffixes plus every multi-database +/// super-journal the staged file may have left (`.compact-mj*`). +fn artifact_paths(store: &Path) -> Vec { + let mut paths: Vec = ARTIFACT_SUFFIXES + .iter() + .map(|suffix| with_suffix(store, suffix)) + .collect(); + let name = store + .file_name() + .map_or_else(|| store.as_os_str().to_os_string(), |name| name.to_os_string()); + let mut prefix = name; + prefix.push(".compact-mj"); + let prefix = prefix.to_string_lossy().into_owned(); + if let Some(parent) = store.parent() { + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&prefix) { + paths.push(entry.path()); + } + } + } + } + paths +} + +fn io_error(error: std::io::Error) -> HistoryError { + match error.raw_os_error() { + Some(28) | Some(122) | Some(27) | Some(69) => HistoryError::Capacity("disk space"), + _ => HistoryError::Integrity("Store compaction I/O"), + } +} + +fn db_error(error: rusqlite::Error) -> HistoryError { + rows::error(error).into() +} + +fn seal_error(error: PersistenceError) -> HistoryError { + match error { + PersistenceError::Busy => HistoryError::Busy, + PersistenceError::Refused { status } if status == "Capacity" => { + HistoryError::Capacity("Store compaction") + } + PersistenceError::Refused { status } if status == "Store sidecar remains after seal" => { + HistoryError::Conflict("Store sidecar remains after seal") + } + PersistenceError::Refused { .. } | PersistenceError::Malformed => { + HistoryError::Integrity("Store seal") + } + PersistenceError::Uncertain => HistoryError::UnknownOutcome, + PersistenceError::Missing | PersistenceError::BackendUnavailable => { + HistoryError::Unsupported("SQLite Store compaction") + } + } +} + +/// The read-only URI of one database file that must not be changed while it is +/// read. Path bytes are percent-encoded so `?`, `#` and `%` do not misdirect +/// the URI (§7 step 6). +#[cfg(unix)] +fn immutable_uri(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + let mut uri = String::from("file:"); + for byte in path.as_os_str().as_bytes() { + match byte { + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b'/' => { + uri.push(char::from(*byte)); + } + _ => { + use std::fmt::Write as _; + let _ = write!(uri, "%{byte:02X}"); + } + } + } + uri.push_str("?immutable=1"); + uri +} + +fn open_immutable(path: &Path) -> HistoryResult { + #[cfg(unix)] + { + let connection = Connection::open_with_flags( + immutable_uri(path), + OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + Ok(connection) + } + #[cfg(not(unix))] + { + let _ = path; + Err(HistoryError::Unsupported("SQLite Store compaction")) + } +} + +fn live_tuple(session: &Arc) -> HistoryResult { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query(TUPLE_SQL, vec![])?; + let row = rows.first().ok_or(BackendError::Integrity)?; + Ok(MarkTuple { + catalog_id: row.get(0)?, + generation: row.get(1)?, + next_history_serial: row.get(2)?, + next_pack_id: row.get(3)?, + next_ordinal: row.get(4)?, + logical: row.get(5)?, + canonical: row.get(6)?, + stored: row.get(7)?, + metadata: row.get(8)?, + stamp: row.get(9)?, + history_rows: row.get(10)?, + }) + }) + .map_err(Into::into) +} + +fn file_tuple(connection: &Connection) -> HistoryResult { + connection + .query_row(TUPLE_SQL, params![], |row| { + Ok(MarkTuple { + catalog_id: row.get(0)?, + generation: row.get(1)?, + next_history_serial: row.get(2)?, + next_pack_id: row.get(3)?, + next_ordinal: row.get(4)?, + logical: row.get(5)?, + canonical: row.get(6)?, + stored: row.get(7)?, + metadata: row.get(8)?, + stamp: row.get(9)?, + history_rows: row.get(10)?, + }) + }) + .map_err(db_error) +} + +fn pack_lengths(session: &Arc) -> HistoryResult> { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query("SELECT pack_id,length FROM pack", vec![])?; + let mut lengths = BTreeMap::new(); + for row in &rows { + let id: i64 = row.get(0)?; + let length: i64 = row.get(1)?; + lengths.insert( + id, + u64::try_from(length).map_err(|_| BackendError::Integrity)?, + ); + } + Ok(lengths) + }) + .map_err(Into::into) +} + +fn object_packs(session: &Arc) -> HistoryResult> { + session + .run::<_, BackendError>(false, |tx| { + let rows = tx.query("SELECT object_id,pack_id FROM object_location", vec![])?; + let mut packs = BTreeMap::new(); + for row in &rows { + let bytes: Vec = row.get(0)?; + let pack: i64 = row.get(1)?; + let id = ObjectId::from_bytes(&bytes).map_err(|_| BackendError::Integrity)?; + packs.insert(id, pack); + } + Ok(packs) + }) + .map_err(Into::into) +} + +fn live_page_metrics(session: &Arc) -> HistoryResult<(u64, u64)> { + session + .run::<_, BackendError>(false, |tx| { + let free = tx.query("PRAGMA freelist_count", vec![])?; + let page = tx.query("PRAGMA page_size", vec![])?; + let free = free.first().ok_or(BackendError::Integrity)?.get::(0)?; + let page = page.first().ok_or(BackendError::Integrity)?.get::(0)?; + Ok(( + u64::try_from(free).map_err(|_| BackendError::Integrity)?, + u64::try_from(page).map_err(|_| BackendError::Integrity)?, + )) + }) + .map_err(Into::into) +} + +fn file_page_metrics(connection: &Connection) -> HistoryResult<(u64, u64)> { + let free: i64 = connection + .query_row("PRAGMA freelist_count", params![], |row| row.get(0)) + .map_err(db_error)?; + let page: i64 = connection + .query_row("PRAGMA page_size", params![], |row| row.get(0)) + .map_err(db_error)?; + Ok(( + u64::try_from(free).map_err(|_| HistoryError::Integrity("freelist count"))?, + u64::try_from(page).map_err(|_| HistoryError::Integrity("page size"))?, + )) +} + +fn file_heads_digest(connection: &Connection) -> HistoryResult<[u8; 32]> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let mut stacks = connection.prepare(LIVE_STACKS).map_err(db_error)?; + let mut rows = stacks.query(params![]).map_err(db_error)?; + while let Some(row) = rows.next().map_err(db_error)? { + let id: Vec = row.get(0).map_err(db_error)?; + let head: Vec = row.get(1).map_err(db_error)?; + if id.len() != 17 || head.len() != 33 { + return Err(HistoryError::Integrity("live-heads stack width")); + } + hasher.update(&id); + hasher.update(&head); + } + drop(rows); + drop(stacks); + let mut branches = connection.prepare(LIVE_BRANCHES).map_err(db_error)?; + let mut rows = branches.query(params![]).map_err(db_error)?; + while let Some(row) = rows.next().map_err(db_error)? { + let id: Vec = row.get(0).map_err(db_error)?; + let head: Option> = row.get(1).map_err(db_error)?; + let base: Vec = row.get(2).map_err(db_error)?; + if id.len() != 17 || base.len() != 33 { + return Err(HistoryError::Integrity("live-heads branch width")); + } + hasher.update(&id); + match head { + Some(head) if head.len() == 33 => hasher.update(&head), + Some(_) => return Err(HistoryError::Integrity("live-heads commit width")), + None => hasher.update(&[0u8; 33]), + }; + hasher.update(&base); + } + Ok(*hasher.finalize().as_bytes()) +} + +/// Writes the §6.3 visited set and the retained pack set to the scratch +/// database the §7 copy joins against. The scratch runs `journal_mode=OFF`, so +/// it leaves no journal of its own. +fn write_visited( + scratch: &Path, + mark: &MarkResult, + packs: &BTreeMap, +) -> HistoryResult { + if std::fs::symlink_metadata(scratch).is_ok() { + return Err(HistoryError::Exists("staged compaction artifact")); + } + let connection = Connection::open_with_flags( + scratch, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + connection + .execute_batch( + "PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;\ +CREATE TABLE visited (\ + object_id BLOB PRIMARY KEY CHECK(length(object_id)=32),\ + pack_id INTEGER NOT NULL\ +) STRICT, WITHOUT ROWID;\ +CREATE TABLE live_pack (pack_id INTEGER PRIMARY KEY) STRICT, WITHOUT ROWID;", + ) + .map_err(db_error)?; + connection.execute_batch("BEGIN IMMEDIATE").map_err(db_error)?; + { + let mut visited = connection + .prepare("INSERT OR IGNORE INTO visited(object_id,pack_id) VALUES(?1,?2)") + .map_err(db_error)?; + let mut live = connection + .prepare("INSERT OR IGNORE INTO live_pack(pack_id) VALUES(?1)") + .map_err(db_error)?; + for pack in mark.objects().packs() { + live.execute(params![pack]).map_err(db_error)?; + } + for id in mark.objects().objects() { + let pack = packs + .get(id) + .ok_or(HistoryError::Integrity("mark object locator"))?; + visited + .execute(params![id.as_bytes().as_slice(), pack]) + .map_err(db_error)?; + } + } + connection.execute_batch("COMMIT").map_err(db_error)?; + connection.close().map_err(|(_, error)| db_error(error))?; + for suffix in ["-journal", "-wal", "-shm"] { + let mut sidecar = scratch.as_os_str().to_os_string(); + sidecar.push(suffix); + if std::fs::symlink_metadata(sidecar).is_ok() { + return Err(HistoryError::Integrity("visited scratch sidecar")); + } + } + Ok(std::fs::metadata(scratch).map_err(io_error)?.len()) +} + +/// What the staged rebuild produced, for the report and the install fence. +struct Built { + target: PathBuf, + target_bytes: u64, + visited_bytes: u64, + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, + free_bytes: u64, +} + +impl Built { + fn peak_temp_bytes(&self) -> u64 { + self.target_bytes.saturating_add(self.visited_bytes) + } +} + +/// Escaped SQL text literal for an `ATTACH` path (the statement cannot bind). +fn quote(text: &str) -> String { + text.replace('\'', "''") +} + +fn table_present(connection: &Connection, name: &str) -> HistoryResult { + let found: i64 = connection + .query_row( + "SELECT count(*) FROM sqlite_schema WHERE type='table' AND name=?1", + params![name], + |row| row.get(0), + ) + .map_err(db_error)?; + Ok(found == 1) +} + +/// The §7 steps 2–5: build the staged Store, copy only retained rows, compact +/// it, and switch it to WAL. +fn build( + facts: &SealedFacts, + mark: &MarkResult, + tuple: &MarkTuple, + packs: &BTreeMap, + layout: crate::SqlitePackLayout, + synchronous: i64, + capacity: bool, + request: &CompactRequest, + op: &RunOp, +) -> HistoryResult { + let store = &facts.path; + let target = compact_path(store); + let visited = visited_path(store); + for staged in [&target, &visited] { + if std::fs::symlink_metadata(staged).is_ok() { + return Err(HistoryError::Exists("staged compaction artifact")); + } + } + stop_at(request, CompactFailpoint::BeforeCopy, op, store)?; + let visited_bytes = write_visited(&visited, mark, packs)?; + if capacity { + // §7 step 3 on a bounded filesystem: `VACUUM INTO` has already created + // the staged file when the write meets ENOSPC. The abort path removes + // it and the live Store is unchanged. + std::fs::write(&target, b"staged").map_err(io_error)?; + return Err(io_error(std::io::Error::from_raw_os_error(28))); + } + + let source = open_immutable(store)?; + let target_text = target + .to_str() + .ok_or(HistoryError::InvalidInput("compaction path"))?; + source + .execute("VACUUM INTO ?1", params![target_text]) + .map_err(db_error)?; + source + .close() + .map_err(|(_, error)| db_error(error))?; + // The copy produced the staged file but has not pruned or verified it. + stop_at(request, CompactFailpoint::DuringCopy, op, store)?; + + let connection = Connection::open_with_flags( + &target, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + connection + .execute_batch(&format!( + "PRAGMA journal_mode=DELETE; PRAGMA synchronous={synchronous};" + )) + .map_err(db_error)?; + // The copied published-pack rows may be deleted in the artifact; the live + // immutability triggers are restored before the file is verified. + // `sqlite_sequence` is the acquisition operation counter: the copy keeps + // every row the source had, and the sequence never falls (§4.1, §7). + let sequences: Vec<(String, i64)> = if table_present(&connection, "sqlite_sequence")? { + let mut statement = connection + .prepare("SELECT name,seq FROM sqlite_sequence") + .map_err(db_error)?; + let mut rows = statement.query(params![]).map_err(db_error)?; + let mut found = Vec::new(); + while let Some(row) = rows.next().map_err(db_error)? { + found.push(( + row.get::<_, String>(0).map_err(db_error)?, + row.get::<_, i64>(1).map_err(db_error)?, + )); + } + found + } else { + Vec::new() + }; + let triggers: Vec = { + let mut statement = connection + .prepare( + "SELECT sql FROM sqlite_schema \ +WHERE type='trigger' AND name LIKE '%immutable%' AND sql IS NOT NULL", + ) + .map_err(db_error)?; + let mut rows = statement.query(params![]).map_err(db_error)?; + let mut found = Vec::new(); + while let Some(row) = rows.next().map_err(db_error)? { + found.push(row.get::<_, String>(0).map_err(db_error)?); + } + found + }; + connection + .execute_batch( + "DROP TRIGGER IF EXISTS pack_immutable_update;\ +DROP TRIGGER IF EXISTS pack_immutable_delete;\ +DROP TRIGGER IF EXISTS pack_unit_immutable_update;\ +DROP TRIGGER IF EXISTS pack_unit_immutable_delete;\ +PRAGMA foreign_keys=OFF;\ +PRAGMA defer_foreign_keys=ON;", + ) + .map_err(db_error)?; + connection + .execute_batch(&format!( + "ATTACH DATABASE '{}' AS scratch;", + quote(visited.to_str().ok_or(HistoryError::InvalidInput("compaction path"))?) + )) + .map_err(db_error)?; + + let units = layout.uses_units(); + let acquisition_tables = table_present(&connection, "init_operation")?; + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(db_error)?; + let staged = (|| -> HistoryResult<()> { + // §7 step 4: the totals trigger invariant holds before the copy. + let before: (i64, i64) = connection + .query_row( + "SELECT (SELECT logical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer)", + params![], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(db_error)?; + if before.0 != before.1 { + return Err(HistoryError::Integrity( + "Store totals disagree with the history rows", + )); + } + let mut deletes = String::from( + "DELETE FROM object_location WHERE object_id NOT IN \ +(SELECT object_id FROM scratch.visited);\ +DELETE FROM content_signature WHERE object_id NOT IN \ +(SELECT object_id FROM scratch.visited);\ +DELETE FROM file_digest WHERE root NOT IN (SELECT object_id FROM scratch.visited);\ +DELETE FROM binding_hint_block;\ +DELETE FROM \"commit\" WHERE commit_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='commit');\ +DELETE FROM layer WHERE layer_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='layer');\ +DELETE FROM branch WHERE branch_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='branch');\ +DELETE FROM layer_stack WHERE layer_stack_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='stack');\ +DELETE FROM layer_lineage WHERE layer_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='layer');\ +DELETE FROM commit_lineage WHERE commit_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='commit');\ +DELETE FROM absorbed WHERE run NOT IN (SELECT run FROM commit_lineage);\ +DELETE FROM reach WHERE key NOT IN \ +(SELECT layer_stack_id FROM layer_stack UNION SELECT run FROM commit_lineage);\ +DELETE FROM branch_retired WHERE branch_id NOT IN \ +(SELECT node_id FROM kept_node WHERE kind='branch');\ +DELETE FROM layer_stack_retired WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM path_removed WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM project_epoch WHERE layer_stack_id NOT IN \ +(SELECT layer_stack_id FROM layer_stack);\ +DELETE FROM merge_record;\ +DELETE FROM merge_operation;\ +DELETE FROM gc_intent;\ +DELETE FROM gc_dead;\ +DELETE FROM gc_op;", + ); + if units { + deletes.push_str( + "DELETE FROM pack_unit WHERE pack_id NOT IN \ +(SELECT pack_id FROM scratch.live_pack);", + ); + } + deletes.push_str( + "DELETE FROM pack WHERE pack_id NOT IN (SELECT pack_id FROM scratch.live_pack);", + ); + if acquisition_tables { + for table in ["init_operation", "init_entry", "init_host_file"] { + deletes.push_str(&format!("DELETE FROM {table};")); + } + } + for trigger in &triggers { + deletes.push_str(trigger.trim_end_matches(';')); + deletes.push_str(";\n"); + } + deletes.push_str(&format!( + "UPDATE history_meta SET catalog_incarnation={} WHERE id=1;\ +UPDATE store_totals SET \ +logical=(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer),\ +canonical=(SELECT COALESCE(SUM(canonical_length),0) FROM object_location),\ +stored=(SELECT COALESCE(SUM(length),0) FROM pack),\ +metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", + tuple.generation + 1 + )); + connection.execute_batch(&deletes).map_err(db_error)?; + Ok(()) + })(); + match staged { + Ok(()) => { + connection.execute_batch("COMMIT").map_err(db_error)?; + } + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + let _ = connection.execute_batch("DETACH DATABASE scratch"); + let _ = connection.close(); + return Err(error); + } + } + connection + .execute_batch("DETACH DATABASE scratch") + .map_err(db_error)?; + // Reclaim the deleted pages: a wholly live Store, no freelist. + connection.execute_batch("VACUUM").map_err(db_error)?; + if !sequences.is_empty() && table_present(&connection, "sqlite_sequence")? { + connection + .execute_batch("DELETE FROM sqlite_sequence") + .map_err(db_error)?; + let mut statement = connection + .prepare("INSERT INTO sqlite_sequence(name,seq) VALUES(?1,?2)") + .map_err(db_error)?; + for (name, seq) in &sequences { + statement.execute(params![name, seq]).map_err(db_error)?; + } + } + let totals: (i64, i64, i64, i64) = connection + .query_row( + "SELECT logical,canonical,stored,metadata FROM store_totals WHERE id=1", + params![], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .map_err(db_error)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + + // §7 step 5: switch to WAL, checkpoint, and leave no sidecar. + let connection = Connection::open_with_flags( + &target, + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(db_error)?; + connection + .busy_timeout(std::time::Duration::ZERO) + .map_err(db_error)?; + let mode: String = connection + .query_row("PRAGMA journal_mode=wal", params![], |row| row.get(0)) + .map_err(db_error)?; + if mode != "wal" { + return Err(HistoryError::Integrity("staged Store is not in WAL mode")); + } + connection + .execute_batch(&format!("PRAGMA synchronous={synchronous}")) + .map_err(db_error)?; + let busy: i64 = connection + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", params![], |row| { + row.get(0) + }) + .map_err(db_error)?; + if busy != 0 { + return Err(HistoryError::Busy); + } + connection + .close() + .map_err(|(_, error)| db_error(error))?; + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = target.as_os_str().to_os_string(); + sidecar.push(suffix); + if std::fs::symlink_metadata(sidecar).is_ok() { + return Err(HistoryError::Integrity("staged Store sidecar")); + } + } + let target_bytes = std::fs::metadata(&target).map_err(io_error)?.len(); + let free_bytes = file_free_bytes(&target)?; + Ok(Built { + target, + target_bytes, + visited_bytes, + logical: u64::try_from(totals.0).unwrap_or(0), + canonical: u64::try_from(totals.1).unwrap_or(0), + stored: u64::try_from(totals.2).unwrap_or(0), + metadata: u64::try_from(totals.3).unwrap_or(0), + free_bytes, + }) +} + +/// The §7 step 6 verification through an `immutable=1` reader: schema identity, +/// foreign keys, row counts against the mark, both live-heads digests, the +/// allocator tuple and the recomputed totals. `quick_check` and the full doctor +/// stay separate commands. +fn verify( + store: &Path, + built: &Built, + mark: &MarkResult, + tuple: &MarkTuple, + layout: crate::SqlitePackLayout, + acquisition: crate::SqliteAcquisitionSchema, + request: &CompactRequest, + op: &RunOp, +) -> HistoryResult<()> { + let connection = open_immutable(&built.target)?; + schema::check( + &connection, + &RefCell::new(SqlWork::default()), + (layout, acquisition), + ) + .map_err(HistoryError::from)?; + let count = |sql: &str| -> HistoryResult { + let value: i64 = connection + .query_row(sql, params![], |row| row.get(0)) + .map_err(db_error)?; + u64::try_from(value).map_err(|_| HistoryError::Integrity("verification count")) + }; + if count("SELECT count(*) FROM pragma_foreign_key_check")? != 0 { + return Err(HistoryError::Integrity("staged Store foreign key")); + } + if count("SELECT count(*) FROM object_location")? != mark.objects().len() as u64 { + return Err(HistoryError::Integrity("staged Store object count")); + } + if count("SELECT count(*) FROM pack")? != mark.objects().packs().len() as u64 { + return Err(HistoryError::Integrity("staged Store pack count")); + } + if count("SELECT count(*) FROM kept_node")? != mark.history().len() as u64 { + return Err(HistoryError::Integrity("staged Store history count")); + } + stop_at(request, CompactFailpoint::DuringVerify, op, store)?; + if file_heads_digest(&connection)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict( + "staged Store live heads differ from the mark", + )); + } + let checked = file_tuple(&connection)?; + if checked.catalog_id != tuple.catalog_id + || checked.generation != tuple.generation + 1 + || checked.next_history_serial != tuple.next_history_serial + || checked.next_pack_id != tuple.next_pack_id + || checked.next_ordinal != tuple.next_ordinal + || checked.stamp > tuple.stamp + { + return Err(HistoryError::Integrity("staged Store allocator tuple")); + } + let totals = connection + .query_row( + "SELECT \ +(SELECT logical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(logical),0) FROM \"commit\")+\ +(SELECT COALESCE(SUM(logical),0) FROM layer),\ +(SELECT canonical FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(canonical_length),0) FROM object_location),\ +(SELECT stored FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(length),0) FROM pack),\ +(SELECT metadata FROM store_totals WHERE id=1),\ +(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0)", + params![], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, i64>(7)?, + )) + }, + ) + .map_err(db_error)?; + if totals.0 != totals.1 + || totals.2 != totals.3 + || totals.4 != totals.5 + || totals.6 != totals.7 + { + return Err(HistoryError::Integrity("staged Store totals")); + } + let built_logical = u64::try_from(checked.logical).unwrap_or(u64::MAX); + let built_canonical = u64::try_from(checked.canonical).unwrap_or(u64::MAX); + let built_stored = u64::try_from(checked.stored).unwrap_or(u64::MAX); + let built_metadata = u64::try_from(checked.metadata).unwrap_or(u64::MAX); + if built_logical != built.logical + || built_canonical != built.canonical + || built_stored != built.stored + || built_metadata != built.metadata + { + return Err(HistoryError::Integrity("staged Store build totals")); + } + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(()) +} + +fn file_free_bytes(target: &Path) -> HistoryResult { + let measure = open_immutable(target)?; + let (free, page) = file_page_metrics(&measure)?; + measure + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(free.saturating_mul(page)) +} + +fn source_state(store: &Path) -> HistoryResult<(u64, SystemTime, u32)> { + let metadata = std::fs::metadata(store).map_err(io_error)?; + let modified = metadata.modified().map_err(io_error)?; + let mut file = std::fs::File::open(store).map_err(io_error)?; + use std::io::{Read as _, Seek as _, SeekFrom}; + let mut header = [0u8; 4]; + file.seek(SeekFrom::Start(24)).map_err(io_error)?; + file.read_exact(&mut header).map_err(io_error)?; + Ok((metadata.len(), modified, u32::from_be_bytes(header))) +} + +/// True when one live or staged file carries a `-wal`, `-shm`, `-journal` or +/// multi-database super-journal (§7 step 7). +fn sidecar_present(path: &Path) -> Option { + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = path.as_os_str().to_os_string(); + sidecar.push(suffix); + let sidecar = PathBuf::from(sidecar); + if std::fs::symlink_metadata(&sidecar).is_ok() { + return Some(sidecar); + } + } + let name = path + .file_name() + .map_or_else(|| path.as_os_str().to_os_string(), |name| name.to_os_string()); + let mut prefix = name; + prefix.push("-mj"); + let prefix = prefix.to_string_lossy().into_owned(); + if let Some(parent) = path.parent() { + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&prefix) { + return Some(entry.path()); + } + } + } + } + None +} + +/// §7 step 7's fence before the rename: no sidecar, the same `G`, the same +/// catalog identity and the sealed `(size, mtime, change counter)`. +fn install_fence(facts: &SealedFacts, tuple: &MarkTuple) -> HistoryResult<()> { + // §10: the path must still name the sealed file. A replacement is + // `StaleStore` before the rename could install over a different Store. + if let Some(sealed) = facts.identity { + match sealed.names(&facts.path) { + Ok(true) => {} + Ok(false) => return Err(HistoryError::StaleStore), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(HistoryError::StaleStore); + } + Err(_) => return Err(HistoryError::Integrity("Store identity at install")), + } + } + for path in [&facts.path, &compact_path(&facts.path)] { + if sidecar_present(path).is_some() { + return Err(HistoryError::Conflict("Store sidecar at install")); + } + } + let connection = open_immutable(&facts.path)?; + let current = file_tuple(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + if ¤t != tuple { + return Err(HistoryError::Conflict("the sealed Store changed")); + } + let (bytes, modified, change_counter) = source_state(&facts.path)?; + if bytes != facts.bytes || modified != facts.modified || change_counter != facts.change_counter + { + return Err(HistoryError::Conflict("the sealed Store changed")); + } + Ok(()) +} + +/// §7 step 7: `chmod 0600` and `sync_all` before the rename. +fn prepare_target(target: &Path) -> HistoryResult<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(target, std::fs::Permissions::from_mode(0o600)).map_err(io_error)?; + } + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(target) + .map_err(io_error)?; + file.sync_all().map_err(io_error)?; + Ok(()) +} + +fn sync_directory(store: &Path) -> HistoryResult<()> { + if let Some(parent) = store.parent() { + let directory = std::fs::File::open(parent).map_err(io_error)?; + directory.sync_all().map_err(io_error)?; + } + Ok(()) +} + +/// Removes the fixed artifact set and reports only what remains, so a +/// successful settle has a zero residue (§11). +fn clear_artifacts(store: &Path) -> CompactResidue { + let mut bytes = 0u64; + let mut left = Vec::new(); + for path in artifact_paths(store) { + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + if std::fs::remove_file(&path).is_err() && std::fs::symlink_metadata(&path).is_ok() { + // §11: residue is what the settle left behind. An artifact that + // was removed is not residue, so its bytes are not counted. + bytes = bytes.saturating_add(metadata.len()); + left.push( + path.file_name() + .map_or_else(|| path.display().to_string(), |name| { + name.to_string_lossy().into_owned() + }), + ); + } + } + CompactResidue { + bytes, + artifacts: left, + reason: None, + } +} + +/// Settles one run: the artifact set is removed first, the op record last +/// (§9: the op is settled only after removal). +fn settle(store: &Path, request: &CompactRequest, op: &RunOp) -> HistoryResult { + let mut residue = clear_artifacts(store); + // §9: the op settles only after the artifact set is gone. A crash here + // leaves the installed `G + 1` and the record for the next run. + stop_at(request, CompactFailpoint::DuringSettle, op, store)?; + let record_path = op_path(store); + if std::fs::symlink_metadata(&record_path).is_ok() { + let bytes = std::fs::metadata(&record_path).map_or(0, |metadata| metadata.len()); + if std::fs::remove_file(&record_path).is_err() + && std::fs::symlink_metadata(&record_path).is_ok() + { + residue.bytes = residue.bytes.saturating_add(bytes); + residue.artifacts.push(record_path.file_name().map_or_else( + || record_path.display().to_string(), + |name| name.to_string_lossy().into_owned(), + )); + } + } + if !residue.artifacts.is_empty() && residue.reason.is_none() { + residue.reason = Some("the artifact set could not be removed".to_owned()); + } + Ok(residue) +} + +/// Drops every artifact and the op record after a definite failure before the +/// rename; the live Store is unchanged apart from the seal's checkpoint. +fn abort_before_install(store: &Path) { + let residue = clear_artifacts(store); + if residue.artifacts.is_empty() { + let _ = std::fs::remove_file(op_path(store)); + } +} + +/// Releases the admission intent of a definite failure before the seal and +/// then removes the artifact set and the op record. When the release itself +/// fails the op record is kept as the next run's recovery breadcrumb (§9): +/// deleting it would leave a live intent that no later call could settle. +fn abort_run(history: &crate::HistoryProvider, store: &Path, op_id: &[u8; 16]) { + if history.gc_release(op_id).is_ok() { + abort_before_install(store); + } +} + +/// Judges and removes a leftover an earlier run recorded (§9): the artifact set +/// it names and the admission intent it took. The caller already holds the +/// writer lane, so a leftover owner is gone. +fn settle_leftover( + history: &crate::HistoryProvider, + store: &Path, + _generation: u64, +) -> HistoryResult<()> { + let Some(op) = RunOp::read(store) else { + if artifact_paths(store) + .iter() + .any(|path| std::fs::symlink_metadata(path).is_ok()) + { + return Err(HistoryError::Exists("unrecorded compaction artifact")); + } + return Ok(()); + }; + // Adopt the recorded owner's admission intent before its artifact is gone. + history.gc_release(&op.op_id)?; + let residue = clear_artifacts(store); + if !residue.artifacts.is_empty() { + return Err(HistoryError::Integrity( + "recorded compaction artifact could not be removed", + )); + } + if std::fs::symlink_metadata(op_path(store)).is_ok() { + std::fs::remove_file(op_path(store)).map_err(io_error)?; + } + Ok(()) +} + +/// A writer held across the seal, install and settle, released on every exit. +struct WriterGuard(Option); + +impl WriterGuard { + fn new(writer: super::lane::writer_admission::Writer) -> Self { + Self(Some(writer)) + } + + fn take(&mut self) -> super::lane::writer_admission::Writer { + self.0.take().expect("a compaction holds its writer") + } + + fn put(&mut self, writer: super::lane::writer_admission::Writer) { + self.0 = Some(writer); + } + + fn release(&mut self) -> HistoryResult<()> { + match self.0.take() { + Some(writer) => writer.release().map_err(|_| HistoryError::UnknownOutcome), + None => Ok(()), + } + } +} + +impl Drop for WriterGuard { + fn drop(&mut self) { + if let Some(writer) = self.0.take() { + let _ = writer.release(); + } + } +} + +/// A test-only crash stop: records the phase and leaves the state a crash +/// would leave. +fn stop_at( + request: &CompactRequest, + point: CompactFailpoint, + op: &RunOp, + store: &Path, +) -> HistoryResult<()> { + if request.failpoint != Some(point) { + return Ok(()); + } + let mut stopped = op.clone(); + stopped.phase = point.phase().to_owned(); + stopped.write(store)?; + Err(HistoryError::UnknownOutcome) +} + +fn immutable_tuple(path: &Path) -> HistoryResult { + let connection = open_immutable(path)?; + let tuple = file_tuple(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(tuple) +} + +fn immutable_digest(path: &Path) -> HistoryResult<[u8; 32]> { + let connection = open_immutable(path)?; + let digest = file_heads_digest(&connection)?; + connection + .close() + .map_err(|(_, error)| db_error(error))?; + Ok(digest) +} + +/// §11: a zero field always carries one word of the closed vocabulary. +#[allow(clippy::too_many_arguments)] +fn receipt( + path: CompactPath, + generation_before: u64, + generation_after: u64, + dead_bytes: u64, + logical_discarded: u64, + backend_reusable: u64, + physical_shrink: u64, + kept_waste: CompactKeptWaste, + peak_temp_bytes: u64, + residue: CompactResidue, + total_stored: u64, + history_rows: u64, +) -> CompactReceipt { + let mut zero_reasons = Vec::new(); + if dead_bytes == 0 { + let reason = if total_stored == 0 { + "empty-store" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "deadBytesBound", + reason, + }); + zero_reasons.push(CompactZeroReason { + field: "deadBytesRefined", + reason, + }); + } + if logical_discarded == 0 { + let reason = if history_rows == 0 { + "no-history" + } else if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "logicalDiscarded", + reason, + }); + } + if physical_shrink == 0 { + zero_reasons.push(CompactZeroReason { + field: "physicalShrink", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + if backend_reusable == 0 { + zero_reasons.push(CompactZeroReason { + field: "backendReusable", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + if kept_waste.is_empty() { + zero_reasons.push(CompactZeroReason { + field: "keptWaste", + reason: "no-waste", + }); + } + if peak_temp_bytes == 0 { + zero_reasons.push(CompactZeroReason { + field: "peakTempBytes", + reason: "none-path", + }); + } + CompactReceipt { + path, + generation_before, + generation_after, + dead_bytes_bound: dead_bytes, + dead_bytes_refined: dead_bytes, + logical_discarded, + backend_reusable, + physical_shrink, + kept_waste, + peak_temp_bytes, + residue, + estimate: true, + zero_reasons, + } +} + +impl SqliteHandles { + /// Runs one §7 SQLite Store compaction on this Store's own machine. + /// + /// The call consumes the handles like [`Self::seal`]: it checkpoints and + /// closes the sole session, and a rebuild installs a new file at `G + 1`. + /// It takes the Store's exclusive writer lane before its admission intent, + /// so no admitted writer can interleave between the mark, the seal and the + /// rename. The caller reopens the Store afterwards. + /// + /// A definite failure before the rename removes the artifact set and leaves + /// the old Store unchanged apart from the seal's checkpoint. A rename or + /// directory-fsync failure is installed-but-uncertain: the new file and the + /// op record are kept for the next `store.compact` to judge (§9). + pub fn compact(self, request: &CompactRequest) -> HistoryResult { + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = request; + return Err(HistoryError::Unsupported("SQLite Store compaction")); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let SqliteHandles { + storage, + history, + acquisition, + } = self; + drop(acquisition); + let session = storage.session.clone(); + let store = session.path.clone(); + let profile = session.profile.clone(); + let layout = session.layout; + let acquisition_schema = session.acquisition; + let synchronous = profile.synchronous; + + // §7 step 1: the exclusive lane first, so the generation this run + // records cannot move between the mark, the seal and the rename. + let lane = session.hold_lane().map_err(HistoryError::from)?; + // §7 step 1: the sealed source's WAL is emptied before the closure. + // The lane is already held, so no writer can append meanwhile. + session.checkpoint_holding().map_err(HistoryError::from)?; + let live = live_tuple(&session)?; + let generation = u64::try_from(live.generation) + .map_err(|_| HistoryError::Integrity("Store generation"))?; + settle_leftover(&history, &store, generation)?; + + let op_id = request.op_id; + let mut op = RunOp { + op_id, + generation: live.generation, + phase: "admission".to_owned(), + started_ms: now_ms(), + owner_pid: request.owner_pid, + owner_pid_start: request.owner_pid_start, + owner_pid_boot: request.owner_pid_boot, + tuple: live.clone(), + live_heads_digest: [0u8; 32], + }; + op.write(&store)?; + let intent = GcIntent::operation( + op_id, + GcIntentKind::Compaction, + GcScopeKind::Store, + history.catalog_id.as_slice().to_vec(), + ) + .owned( + request.owner_runtime.as_deref(), + request.owner_boot.as_deref(), + ) + .with_phase(Some("mark")); + history.gc_register(&intent)?; + // §8: the registration, the mark and the totals read are one + // admitted sequence. A definite failure here releases the intent + // before the artifact set and the op record go, so the Store is + // left exactly as it was and no later call meets BusyOp. + let admitted = + (|| -> HistoryResult<(MarkResult, MarkTuple, u64, u64, CompactKeptWaste, u64)> { + let mark = history.mark()?; + let tuple = live_tuple(&session)?; + op.tuple = tuple.clone(); + op.live_heads_digest = mark.live_heads_digest(); + op.phase = "mark".to_owned(); + op.write(&store)?; + let lengths = pack_lengths(&session)?; + let total = lengths.values().copied().fold(0u64, u64::saturating_add); + let mut live_stored = 0u64; + for pack in mark.objects().packs() { + let length = lengths + .get(pack) + .ok_or(HistoryError::Integrity("mark pack"))?; + live_stored = live_stored.saturating_add(*length); + } + let dead = total.saturating_sub(live_stored); + let kept_waste = CompactKeptWaste::from_mark(mark.objects().waste()); + let history_rows = u64::try_from(tuple.history_rows).unwrap_or(0); + Ok((mark, tuple, total, dead, kept_waste, history_rows)) + })(); + let (mark, tuple, total, dead, kept_waste, history_rows) = match admitted { + Ok(admitted) => admitted, + Err(error) => { + abort_run(&history, &store, &op_id); + return Err(error); + } + }; + let rebuild = total > 0 && dead.saturating_mul(10) >= total; + + if !rebuild { + // §3 `none`: mark only. The intent is cleared and the op record + // removed; nothing is rebuilt and the generation is unchanged. + let (free, page) = live_page_metrics(&session)?; + history.gc_release(&op_id)?; + let _ = std::fs::remove_file(op_path(&store)); + drop(lane); + return Ok(receipt( + CompactPath::None, + generation, + generation, + dead, + 0, + free.saturating_mul(page), + 0, + kept_waste, + 0, + CompactResidue::default(), + total, + history_rows, + )); + } + + // §7 step 2: a stop here leaves the open session, the writer lane, + // the admission intent and the op record; the live Store is at `G`. + stop_at(request, CompactFailpoint::BeforeSeal, &op, &store)?; + + // §7 step 2 onward: seal and rebuild. Only the session remains. + let object_packs = match object_packs(&session) { + Ok(packs) => packs, + Err(error) => { + abort_run(&history, &store, &op_id); + return Err(error); + } + }; + // §8/§9: release the admission intent while the Store is still + // reachable. Every failure below runs after the seal, when this + // process can no longer run SQL on the Store; without this release + // a live intent would refuse every later GC operation. The held + // writer lane still excludes writers through the install, and a + // crash leaves the op record for the next run to settle. + if let Err(error) = history.gc_release(&op_id) { + return Err(error); + } + drop(history); + drop(storage); + let mut guard = WriterGuard::new(lane.into_writer()); + let session = Arc::try_unwrap(session).map_err(|_| HistoryError::Busy)?; + let writer = guard.take(); + let (facts, writer) = match session.seal_holding(writer) { + Ok(sealed) => sealed, + Err(error) => return Err(seal_error(error)), + }; + guard.put(writer); + // §7 step 1.5: the sealed file is about to be read back. + stop_at(request, CompactFailpoint::BeforeSealRead, &op, &store)?; + + // §7 step 1.5's fence: the seal must not have moved the generation + // or the heads the mark recorded. + let sealed_tuple = match immutable_tuple(&facts.path) { + Ok(tuple) => tuple, + Err(error) => { + abort_before_install(&store); + return Err(error); + } + }; + if sealed_tuple != tuple { + abort_before_install(&store); + return Err(HistoryError::Conflict("the Store changed after the mark")); + } + match immutable_digest(&facts.path) { + Ok(digest) if digest == mark.live_heads_digest() => {} + Ok(_) => { + abort_before_install(&store); + return Err(HistoryError::Conflict( + "the Store heads changed after the mark", + )); + } + Err(error) => { + abort_before_install(&store); + return Err(error); + } + } + stop_at(request, CompactFailpoint::AfterSeal, &op, &store)?; + + op.phase = "build".to_owned(); + op.write(&store)?; + let capacity = request.failpoint == Some(CompactFailpoint::CapacityAtBuild); + let built = match build( + &facts, + &mark, + &tuple, + &object_packs, + layout, + synchronous, + capacity, + request, + &op, + ) { + Ok(built) => built, + // A stop point leaves the crash state on purpose: never abort + // an artifact set a recorded run names (§9). + Err(HistoryError::UnknownOutcome) => return Err(HistoryError::UnknownOutcome), + Err(error) => { + abort_before_install(&store); + return Err(error); + } + }; + stop_at(request, CompactFailpoint::AfterBuild, &op, &store)?; + stop_at(request, CompactFailpoint::AfterWalSwitch, &op, &store)?; + + op.phase = "verify".to_owned(); + if let Err(error) = op.write(&store) { + abort_before_install(&store); + return Err(error); + } + if let Err(error) = verify( + &store, + &built, + &mark, + &tuple, + layout, + acquisition_schema, + request, + &op, + ) { + if matches!(error, HistoryError::UnknownOutcome) { + return Err(error); + } + abort_before_install(&store); + return Err(error); + } + stop_at(request, CompactFailpoint::AfterVerify, &op, &store)?; + + if let Some(hook) = request.before_install_hook { + hook(&facts.path); + } + if let Err(error) = install_fence(&facts, &tuple) { + abort_before_install(&store); + return Err(error); + } + if let Err(error) = prepare_target(&built.target) { + abort_before_install(&store); + return Err(error); + } + stop_at(request, CompactFailpoint::BeforeInstall, &op, &store)?; + + if std::fs::rename(&built.target, &facts.path).is_err() { + // Installed-but-uncertain: never delete the new Store as if + // nothing happened (§7 step 8, §9). + op.phase = "install".to_owned(); + let _ = op.write(&store); + return Err(HistoryError::UnknownOutcome); + } + // §7 step 7: the rename happened; a crash before the directory + // fsync leaves an installed-but-uncertain `G + 1`. + stop_at(request, CompactFailpoint::DuringRename, &op, &store)?; + if let Err(_error) = sync_directory(&facts.path) { + op.phase = "install".to_owned(); + let _ = op.write(&store); + return Err(HistoryError::UnknownOutcome); + } + op.phase = "install".to_owned(); + let _ = op.write(&store); + stop_at(request, CompactFailpoint::AfterInstall, &op, &store)?; + guard.release()?; + let residue = settle(&store, request, &op)?; + let logical_discarded = u64::try_from(tuple.logical) + .unwrap_or(0) + .saturating_sub(built.logical); + let physical_shrink = facts.bytes.saturating_sub(built.target_bytes); + Ok(receipt( + CompactPath::Rebuild, + generation, + generation.saturating_add(1), + dead, + logical_discarded, + built.free_bytes, + physical_shrink, + kept_waste, + built.peak_temp_bytes(), + residue, + total, + history_rows, + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::io_error; + use layerfs_history::HistoryError; + + /// §7 step 3 on a bounded filesystem: `ENOSPC`, `EDQUOT` and `EFBIG` are + /// the report's `Capacity` refusal, never a corruption verdict. + #[test] + fn bounded_filesystem_errors_are_capacity() { + for code in [28, 122, 27, 69] { + let error = io_error(std::io::Error::from_raw_os_error(code)); + assert!( + matches!(error, HistoryError::Capacity(_)), + "errno {code}: {error:?}" + ); + } + } + + /// Every other I/O failure keeps the integrity classification. + #[test] + fn another_io_error_is_integrity() { + let error = io_error(std::io::Error::from_raw_os_error(5)); + assert!(matches!(error, HistoryError::Integrity(_)), "{error:?}"); + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs index 1c31440b0b..3d58695311 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs @@ -38,6 +38,17 @@ impl Session { } } +impl LaneHold { + /// Takes the cross-process writer lock out of the hold and releases the + /// connection turn, so a seal can consume the session while the lane is + /// still held (§7 step 1). The caller becomes the lane's owner. + pub(crate) fn into_writer(mut self) -> Writer { + self.writer + .take() + .expect("a held Store lane owns its writer") + } +} + impl Drop for LaneHold { fn drop(&mut self) { if let Some(writer) = self.writer.take() { diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs new file mode 100644 index 0000000000..43eac9148c --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs @@ -0,0 +1,51 @@ +//! §10 Store-file identity: the `(dev, ino)` fence a SQLite handle applies +//! before it reads or writes, and the install fence re-applies to the sealed +//! source. +use std::path::Path; + +/// The `(device, inode)` pair naming one Store file at one moment (§10). +/// +/// A transaction compares the pair recorded at open with the pair the Store +/// path names now; a replacement or a move is `StaleStore`. Together with the +/// held writer lane this is the SQLite fence of §10: an admitted writer cannot +/// be in flight at install, and a connection that bypassed admission is caught +/// at its next transaction. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct FileIdentity { + /// The filesystem device holding the file. + pub(crate) dev: u64, + /// The file's inode number on that device. + pub(crate) ino: u64, +} + +impl FileIdentity { + /// Captures the identity of the file `path` currently names. + /// + /// `Ok(None)` means this platform has no `(dev, ino)` pair; the fence then + /// has nothing to compare and callers keep their other facts. `Err` is a + /// path that could not be inspected at all. + pub(crate) fn capture(path: &Path) -> Result, std::io::Error> { + let metadata = std::fs::metadata(path)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + Ok(Some(Self { + dev: metadata.dev(), + ino: metadata.ino(), + })) + } + #[cfg(not(unix))] + { + let _ = metadata; + Ok(None) + } + } + + /// §10: whether `path` still names this exact file. + pub(crate) fn names(&self, path: &Path) -> Result { + match Self::capture(path)? { + Some(current) => Ok(current == *self), + None => Ok(true), + } + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs index 1895897203..a23a4ce8cb 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod file_control; #[cfg(any(target_os = "macos", target_os = "linux"))] pub(crate) mod headroom; +pub(crate) mod identity; pub(crate) mod reclamation; pub(crate) mod seal; #[cfg(target_os = "macos")] diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index f9a02f37c0..6370074a43 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -1,12 +1,95 @@ //! Sole-owner host seal; never available through the daemon's Store ports. +use crate::backend::sqlite::lane::writer_admission::Writer; +use crate::backend::sqlite::lifecycle::identity::FileIdentity; use crate::backend::sqlite::session::connection::Session; use crate::backend::sqlite::session::query; use crate::backend::sqlite::session::rows; use crate::{backend::records::BackendError, SealedStore}; use layerfs_storage::port::PersistenceError; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +/// The identity of one closed Store file at the moment its last connection +/// closed, captured under the writer lane (§7 step 1). +#[derive(Clone, Debug)] +pub(crate) struct SealedFacts { + /// The closed database file; no sidecar was present when it was captured. + pub(crate) path: PathBuf, + /// Provisioning profile the file was closed under. + pub(crate) profile: crate::SqlitePersistenceProfile, + /// SQLite version that created/sealed the file. + pub(crate) sqlite_version: String, + /// Main-file bytes after the WAL was checkpointed and the connection closed. + pub(crate) bytes: u64, + /// Main-file modification time at the same moment. + pub(crate) modified: SystemTime, + /// The database header change counter (big-endian bytes 24..28). + pub(crate) change_counter: u32, + /// The file's §10 `(dev, ino)` while the lane was still held. + pub(crate) identity: Option, +} + +impl SealedFacts { + /// The public sealed-Store handoff this identity describes. + pub(crate) fn into_store(self) -> SealedStore { + SealedStore { + path: self.path, + profile: self.profile, + sqlite_version: self.sqlite_version, + bytes: self.bytes, + } + } +} + +fn read_change_counter(path: &Path) -> Result { + let refused = |error: std::io::Error| PersistenceError::Refused { + status: error.to_string(), + }; + let mut file = std::fs::File::open(path).map_err(refused)?; + let mut bytes = [0u8; 4]; + file.seek(SeekFrom::Start(24)).map_err(refused)?; + file.read_exact(&mut bytes).map_err(refused)?; + Ok(u32::from_be_bytes(bytes)) +} impl Session { - pub(crate) fn seal(mut self) -> Result { + /// Consumes the sole session, checkpoints once, closes and verifies one file. + /// macOS then releases unused allocation beyond the original file's logical + /// end, without copying its contents or changing its identity or length. + /// + /// A retained provider/Storage/Reader/Save refuses with `Busy` before any + /// checkpoint. Another process can obstruct checkpoint or keep sidecars; + /// neither case triggers a retry or deletion. Provisioning must prevent + /// new openers throughout sealing and handoff. This is not live Store + /// reclamation. Uncertain transfer/temporary-close outcomes retain the owned + /// `.layerfs-allocation-*` sibling; no automatic retry or deletion follows. + pub(crate) fn seal(self) -> Result { + let writer = { + let mut state = self + .state + .lock() + .map_err(|_| PersistenceError::Uncertain)?; + state.writer_hold.take() + }; + let writer = match writer { + Some(writer) => writer, + None => Writer::acquire(&self.path)?, + }; + let (facts, writer) = self.seal_holding(writer)?; + writer.release().map_err(|_| PersistenceError::Uncertain)?; + Ok(facts.into_store()) + } + + /// [`Self::seal`] under a caller-held writer lane (§7 step 1: the guard is + /// passed into the seal, never acquired a second time). The lane is returned + /// so a compaction can keep it through the rename. The checkpoint, checked + /// close, sidecar check and the sealed `(size, mtime, change counter)` + /// capture all run while the lane is held. + pub(crate) fn seal_holding( + mut self, + writer: Writer, + ) -> Result<(SealedFacts, Writer), PersistenceError> { if !self.writable { return Err(BackendError::ReadOnly.into()); } @@ -19,13 +102,6 @@ impl Session { if state.quarantined || !state.connection.is_autocommit() { return Err(PersistenceError::Uncertain); } - // The same advisory lane as a write transaction: a seal is the sole - // owner's offline handoff, so no other opener may write while the - // checkpoint, close and sidecar removal below run. - if state.writer_hold.is_none() { - state.writer_hold = - Some(crate::backend::sqlite::lane::writer_admission::Writer::acquire(&self.path)?); - } #[cfg(target_os = "macos")] let identity = super::seal_allocation::Identity::capture(&self.path)?; #[cfg(target_os = "macos")] @@ -68,22 +144,39 @@ impl Session { } } } - if let Some(writer) = state.writer_hold.take() { - writer.release().map_err(|_| PersistenceError::Uncertain)?; - } #[cfg(target_os = "macos")] - let bytes = super::seal_allocation::release(&self.path, identity)?; + let released = super::seal_allocation::release(&self.path, identity)?; #[cfg(not(target_os = "macos"))] - let bytes = std::fs::metadata(&self.path) + let released = 0u64; + let metadata = std::fs::metadata(&self.path).map_err(|error| PersistenceError::Refused { + status: error.to_string(), + })?; + let modified = metadata + .modified() .map_err(|error| PersistenceError::Refused { status: error.to_string(), - })? - .len(); - Ok(SealedStore { - path: self.path, - profile: self.profile.persistence, - sqlite_version: self.profile.sqlite_version, - bytes, - }) + })?; + let bytes = if released == 0 { + metadata.len() + } else { + released + }; + let change_counter = read_change_counter(&self.path)?; + let identity = + FileIdentity::capture(&self.path).map_err(|error| PersistenceError::Refused { + status: error.to_string(), + })?; + Ok(( + SealedFacts { + path: self.path.clone(), + profile: self.profile.persistence, + sqlite_version: self.profile.sqlite_version.clone(), + bytes, + modified, + change_counter, + identity, + }, + writer, + )) } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs index 2cb0925bbd..d79e929597 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs @@ -1,6 +1,7 @@ //! SQLite-only connection, transaction, query, schema and physical mechanics, //! grouped by role (#506 A1). pub(crate) mod acquisition; +pub(crate) mod compact; pub(crate) mod heads; pub(crate) mod lane; pub(crate) mod lifecycle; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs index 151d5e3da3..ae2af52d8f 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs @@ -68,6 +68,40 @@ impl Session { } } +impl Session { + /// One `PRAGMA wal_checkpoint(TRUNCATE)` for a caller that already holds the + /// Store's writer lane (§7 step 1). No lane and no turn are taken here; a + /// nonzero `busy` is reported, never retried. + pub(crate) fn checkpoint_holding(&self) -> Result<(), BackendError> { + if !self.writable { + return Err(BackendError::ReadOnly); + } + let _pass = self.background.as_ref().map(|pass| pass.hold()); + let mut state = self.state.lock().map_err(|_| BackendError::Unknown)?; + super::thread_work::attributed(&mut state, |state| { + if state.quarantined || !state.connection.is_autocommit() { + state.quarantined = true; + return Err(BackendError::Unknown); + } + let rows = query::run( + &state.connection, + "PRAGMA wal_checkpoint(TRUNCATE)", + vec![], + &state.work, + )?; + if !state.connection.is_autocommit() { + state.quarantined = true; + return Err(BackendError::Unknown); + } + let busy = rows.first().ok_or(BackendError::Integrity)?.get::(0)?; + if busy != 0 { + return Err(BackendError::Busy); + } + Ok(()) + }) + } +} + /// The length of the Store's `-wal` file; zero when there is none. pub(crate) fn wal_bytes(path: &std::path::Path) -> Result { let mut wal = path.as_os_str().to_os_string(); diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index c080697dae..608efa6d47 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -3,6 +3,7 @@ use super::profile; use super::query; use super::rows; use crate::backend::records::BackendError; +use crate::backend::sqlite::lifecycle::identity::FileIdentity; use crate::SqlitePersistenceProfile; use rusqlite::{limits::Limit, Connection, OpenFlags}; use std::{ @@ -165,6 +166,9 @@ pub(crate) struct Session { pub(crate) profile: ConnectionProfile, pub(crate) layout: crate::SqlitePackLayout, pub(crate) acquisition: crate::SqliteAcquisitionSchema, + /// The `(dev, ino)` identity the path named when this handle opened (§10). + /// `None` on a platform without one; the fence then has nothing to compare. + pub(crate) identity: Option, pub(crate) path: PathBuf, } impl Session { @@ -331,6 +335,8 @@ impl Session { let background = (writable && selected == SqlitePersistenceProfile::Disposable) .then(|| super::background::Checkpointer::start(path)) .transpose()?; + // §10: the record every later transaction is fenced against. + let identity = FileIdentity::capture(path).map_err(|_| BackendError::Integrity)?; Ok(Self { diagnostic, background, @@ -347,6 +353,7 @@ impl Session { profile, layout, acquisition, + identity, }) } /// Stored schema version: the pack layout's, offset by the acquisition tables. @@ -361,4 +368,19 @@ impl Session { let w = *s.work.borrow(); Ok(w) } + /// §10: the Store path must still name the file this handle opened, or the + /// call is `StaleStore` before it reads or writes anything. + pub(crate) fn check_identity(&self) -> Result<(), BackendError> { + let Some(expected) = self.identity else { + return Ok(()); + }; + match expected.names(&self.path) { + Ok(true) => Ok(()), + Ok(false) => Err(BackendError::Stale), + // A removed Store is as stale as a replaced one: nothing this + // handle opened is left at the path to act on. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Err(BackendError::Stale), + Err(_) => Err(BackendError::Integrity), + } + } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index c9f6f349f8..bd64d7a177 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -249,6 +249,9 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "gc_branch_stack_any" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_branch_stack_any.sql" )), + "gc_epoch_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_epoch_read.sql" + )), "gc_epoch_share" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_epoch_share.sql" )), @@ -267,6 +270,9 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "gc_inode_blocks" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_inode_blocks.sql" )), + "gc_tuple_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_tuple_read.sql" + )), "kept_node_clear" => Ok(include_str!( "../../../../sql/sqlite/queries/history/kept_node_clear.sql" )), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs index 65a1fb1cbd..b06988dae6 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs @@ -148,6 +148,7 @@ impl Session { if state.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // An earlier uncertain transaction may still exist. It cannot be // inherited by an independently owned snapshot or guessed complete. if !state.connection.is_autocommit() { @@ -207,6 +208,7 @@ impl Session { if state.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // An earlier uncertain transaction cannot be inherited or guessed done. if !state.connection.is_autocommit() { state.quarantined = true; @@ -249,6 +251,7 @@ impl Session { if s.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // A held publication lane already owns the writer for its nested work. if writable && !turn.nested { // State owns this before BEGIN, including across a poisoned unwind. diff --git a/engine/crates/layerfs-persistence/src/history/commit.rs b/engine/crates/layerfs-persistence/src/history/commit.rs index fc7120f3bc..351c0e533b 100644 --- a/engine/crates/layerfs-persistence/src/history/commit.rs +++ b/engine/crates/layerfs-persistence/src/history/commit.rs @@ -95,6 +95,9 @@ pub(crate) fn commit_branch( if stack.profile != request.profile || stack.scope != request.scope { return Err(HistoryError::Integrity("Commit context")); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, branch.stack)?; let moved = branch.base_layer != request.expected_base || branch.head_commit != request.expected_head; if moved && (!request.force || branch.base_layer != request.expected_base) { @@ -162,6 +165,7 @@ pub(crate) fn commit_branch( if advanced != 1 { return Err(HistoryError::Integrity("Branch advance")); } + fence.confirm(tx)?; let displaced = branch.head_commit.filter(|_| moved); Ok(CommitOutcome::Committed { record: Box::new(record), diff --git a/engine/crates/layerfs-persistence/src/history/epoch.rs b/engine/crates/layerfs-persistence/src/history/epoch.rs new file mode 100644 index 0000000000..1df4cb903d --- /dev/null +++ b/engine/crates/layerfs-persistence/src/history/epoch.rs @@ -0,0 +1,69 @@ +//! The `project_epoch` writer fence (spec §5). +//! +//! Every operation that creates a Commit or Layer reads the project's epoch at +//! the start of the write transaction that will insert it and re-checks that +//! same row before the transaction commits. Each removal marker bumps the epoch +//! in its own marker transaction (§4), so a value that changed between the two +//! reads means a removal raced this write: the writer refuses +//! `Conflict("project epoch")` and the whole write transaction rolls back, so +//! no Commit or Layer is left behind. A missing row (a project deleted and +//! compacted away, §4.1/§6) is the same `Conflict`, never "unchanged". +//! +//! The refusal code is `Conflict`: §10's table names it for the sealed tuple +//! and for a missing `project_epoch` row, and §12's marker-epoch case says a +//! one-shot write racing `branch.remove`/`project.delete` gets `Conflict` and +//! leaves no Commit. It carries no detail (§10; gc-spec-gaps G1 fixes the +//! empty detail for every `Conflict`). +//! +//! On SQLite both reads run inside the one `BEGIN IMMEDIATE` transaction that +//! also writes the Commit/Layer, so a marker can never interleave between them +//! (its `BEGIN IMMEDIATE` is `Busy` until this writer finishes). PostgreSQL +//! makes the re-check the writer's last statement before `COMMIT` with +//! `SELECT ... FOR SHARE` (`gc_epoch_share`), which also blocks the marker's +//! `UPDATE project_epoch SET epoch = epoch + 1` until this transaction resolves: +//! a removal that committed first is seen as a changed value, and one still in +//! flight is serialized after this writer. + +use super::rows::{cell, one}; +use super::transaction::Transaction; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::identity::LayerStackId; + +/// Unlocked read at the start of the write transaction. +const READ: &str = "gc_epoch_read"; +/// Re-read before `COMMIT`; `FOR SHARE` on PostgreSQL, a plain read on SQLite. +const SHARE: &str = "gc_epoch_share"; + +/// The epoch one writer observed when its write transaction opened. +#[derive(Clone, Copy)] +pub(crate) struct Fence { + stack: LayerStackId, + epoch: i64, +} + +/// Reads the project epoch at the start of the write transaction. +/// +/// A missing row is `Conflict`: after `project.delete` and compaction there is +/// no fence left to honour, so the write is refused rather than treated as +/// unchanged. +pub(crate) fn open(tx: &Transaction<'_>, stack: LayerStackId) -> HistoryResult { + let epoch = read(tx, READ, stack)?.ok_or(HistoryError::Conflict("project epoch"))?; + Ok(Fence { stack, epoch }) +} + +impl Fence { + /// Re-checks the epoch as the last statement before `COMMIT`; a changed or + /// removed row is `Conflict` and the caller's transaction writes nothing. + pub(crate) fn confirm(&self, tx: &Transaction<'_>) -> HistoryResult<()> { + let current = + read(tx, SHARE, self.stack)?.ok_or(HistoryError::Conflict("project epoch"))?; + if current != self.epoch { + return Err(HistoryError::Conflict("project epoch")); + } + Ok(()) + } +} + +fn read(tx: &Transaction<'_>, statement: &str, stack: LayerStackId) -> HistoryResult> { + one(tx, statement, [stack.as_slice()], |row| cell::(row, 0)) +} diff --git a/engine/crates/layerfs-persistence/src/history/layerstack.rs b/engine/crates/layerfs-persistence/src/history/layerstack.rs index 17ecc6ddeb..ef5f4e4015 100644 --- a/engine/crates/layerfs-persistence/src/history/layerstack.rs +++ b/engine/crates/layerfs-persistence/src/history/layerstack.rs @@ -352,6 +352,9 @@ pub(crate) fn add_layer( if commit.stack != request.stack { return Err(HistoryError::Integrity("Commit ownership")); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, request.stack)?; if let Some(id) = published_source(tx, request.branch, request.commit)? { let stored = layer(tx, id)?.ok_or(HistoryError::Integrity("Layer provenance"))?; if stored.stack != request.stack @@ -455,6 +458,7 @@ pub(crate) fn add_layer( actual: LayerId::from_slice(&actual)?, }); } + fence.confirm(tx)?; Ok(AddLayerOutcome::Added { layer }) } diff --git a/engine/crates/layerfs-persistence/src/history/merge.rs b/engine/crates/layerfs-persistence/src/history/merge.rs index fc074b4e61..59c9ca97f4 100644 --- a/engine/crates/layerfs-persistence/src/history/merge.rs +++ b/engine/crates/layerfs-persistence/src/history/merge.rs @@ -77,6 +77,9 @@ pub(crate) fn merge_branch( if let Some(state) = super::branch::name_taken(tx, request.stack, &request.name)? { return Err(HistoryError::NameTaken(Box::new(state))); } + // Spec 5: read the project epoch at the start of this write transaction; + // `confirm` below re-checks it before the transaction commits. + let fence = super::epoch::open(tx, request.stack)?; let (base, head) = match &request.head { MergeHead::Node(NodeId::Layer(layer)) => (*layer, None), MergeHead::Node(NodeId::Commit(commit)) => { @@ -132,6 +135,8 @@ pub(crate) fn merge_branch( ], ) .map_err(sql)?; - super::branch::branch_snapshot(tx, request.branch)? - .ok_or(HistoryError::Integrity("Branch insertion")) + let snapshot = super::branch::branch_snapshot(tx, request.branch)? + .ok_or(HistoryError::Integrity("Branch insertion"))?; + fence.confirm(tx)?; + Ok(snapshot) } diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index 7e4c26e712..219d7b16e8 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -7,6 +7,7 @@ mod branch; mod catalog; mod commit; mod digests; +mod epoch; mod hints; mod kept; mod layerstack; @@ -21,8 +22,8 @@ mod removal; mod rows; mod transaction; pub use kept::{KeptHistory, KeptNode, KeptNodeId}; -pub use objects::{KeptObjects, KeptWaste, MarkResult}; pub(crate) use lineage_backfill::backfill_lineage; +pub use objects::{GcTuple, KeptObjects, KeptWaste, MarkResult}; pub use provider::HistoryProvider; pub(crate) use rows::{ branch_row as diagnostic_branch, commit_graph_row as diagnostic_commit_graph, diff --git a/engine/crates/layerfs-persistence/src/history/objects.rs b/engine/crates/layerfs-persistence/src/history/objects.rs index 5a2af90032..30dd25402c 100644 --- a/engine/crates/layerfs-persistence/src/history/objects.rs +++ b/engine/crates/layerfs-persistence/src/history/objects.rs @@ -37,6 +37,7 @@ const LOCATIONS: &str = "gc_object_locations"; const LIVE_STACKS: &str = "gc_live_stacks"; const LIVE_BRANCHES: &str = "gc_live_branches"; const INODE_BLOCKS: &str = "gc_inode_blocks"; +const TUPLE: &str = "gc_tuple_read"; /// The bytes a mark keeps that no retained root actually needs (§6.2, §11). /// @@ -153,6 +154,76 @@ impl MarkResult { } } +/// The Store tuple a compaction records as its mark's first statement and +/// re-checks before the swap (§8 Phase B): the generation, the serial and pack +/// allocators, the four Store totals, the signature stamp and the history row +/// count the mark ran at. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct GcTuple { + /// The catalog incarnation (the Store generation G). + pub generation: i64, + /// The next history serial the mark saw. + pub next_history_serial: i64, + /// store_policy.next_pack_id at the mark. + pub next_pack_id: i64, + /// store_policy.next_ordinal at the mark. + pub next_ordinal: i64, + /// store_totals.logical at the mark. + pub logical: i64, + /// store_totals.canonical at the mark. + pub canonical: i64, + /// store_totals.stored at the mark. + pub stored: i64, + /// store_totals.metadata at the mark. + pub metadata: i64, + /// The highest content_signature stamp at the mark. + pub stamp: i64, + /// count(commit) + count(layer) at the mark. + pub history_rows: i64, +} + +impl HistoryProvider { + /// The T7.2 mark with the §8 mark tuple read as its first statement. + /// + /// The tuple and the closure are one write transaction: a publication that + /// lands before the tuple is read is part of the mark, and one that lands + /// after it moves at least one tuple field, so Phase B abandons. + pub fn mark_gc(&self) -> HistoryResult<(MarkResult, GcTuple)> { + self.write(mark_gc) + } + + /// The §6.4 live-heads digest of the Store as it is now, without a mark. + pub fn live_heads_digest(&self) -> HistoryResult<[u8; 32]> { + self.read(live_heads_digest) + } +} + +fn mark_gc(tx: &Transaction<'_>) -> HistoryResult<(MarkResult, GcTuple)> { + let tuple = gc_tuple(tx)?; + let mark = mark(tx)?; + Ok((mark, tuple)) +} + +fn gc_tuple(tx: &Transaction<'_>) -> HistoryResult { + let rows = tx.query(TUPLE, Vec::::new())?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("Store tuple")); + } + let row = &rows[0]; + Ok(GcTuple { + generation: cell(row, 0)?, + next_history_serial: cell(row, 1)?, + next_pack_id: cell(row, 2)?, + next_ordinal: cell(row, 3)?, + logical: cell(row, 4)?, + canonical: cell(row, 5)?, + stored: cell(row, 6)?, + metadata: cell(row, 7)?, + stamp: cell(row, 8)?, + history_rows: cell(row, 9)?, + }) +} + impl HistoryProvider { /// Recomputes the §6.1 history closure, then the §6.2 object closure over /// its retained Commit and Layer roots, and the §6.4 live-heads digest. diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index bbf172860e..fe158ec820 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -8,6 +8,7 @@ mod storage; mod store; pub(crate) use store::config; mod history; +mod report; #[cfg(feature = "postgresql")] pub use backend::postgresql::access::{access as postgresql_access, PostgresAccess}; #[cfg(feature = "postgresql")] @@ -15,7 +16,14 @@ pub use backend::postgresql::status::{ status as postgresql_status, PostgresStatus, PostgresStatusError, }; pub use backend::sqlite::acquisition::AcquisitionProvider; +pub use backend::sqlite::compact::{ + compact_op_status, CompactFailpoint, CompactKeptWaste, CompactOpStatus, CompactPath, + CompactReceipt, CompactRequest, CompactResidue, CompactZeroReason, +}; pub use backend::sqlite::heads::{heads_digest, read_heads}; +pub use report::{ + CompactGeneration, CompactReport, CompactReportSource, CompactStatus, SQLITE_COMPACT_PHASES, +}; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use backend::sqlite::lifecycle::headroom::STORE_HEADROOM_BYTES; pub use backend::sqlite::lifecycle::reclamation::{SpaceReclamation, RECLAMATION_PAGE_LIMIT}; diff --git a/engine/crates/layerfs-persistence/src/report.rs b/engine/crates/layerfs-persistence/src/report.rs new file mode 100644 index 0000000000..0c6a1fa80e --- /dev/null +++ b/engine/crates/layerfs-persistence/src/report.rs @@ -0,0 +1,124 @@ +//! The §11 compaction report and the `store.status` compaction projection. +//! +//! One shape serves every backend. A SQLite [`CompactReceipt`] projects into +//! [`CompactReport`] through [`CompactReportSource`], and T9's PostgreSQL +//! rebuild implements that trait for its own receipt, so both paths share one +//! wire shape and one serializer. §11 names are logical; the rendered keys are +//! snake_case (§10). +use crate::backend::sqlite::compact::{ + CompactKeptWaste, CompactPath, CompactReceipt, CompactResidue, CompactZeroReason, +}; + +/// The phases a SQLite compaction passes, in order (§11): a run that crashes +/// leaves its record at the phase it reached. +pub const SQLITE_COMPACT_PHASES: [&str; 4] = ["mark", "build", "verify", "install"]; + +/// The generation a compaction started at and, when it installed a rebuild, +/// the one it left (`none` keeps both equal). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompactGeneration { + /// The generation the mark ran at. + pub before: u64, + /// The installed generation; equal to `before` on the `none` path. + pub after: u64, +} + +/// The §11 report `store.compact` returns, backend-neutral. +/// +/// SQLite fills it from [`CompactReceipt`]; T9's PostgreSQL rebuild fills the +/// same fields from its own receipt. A backend-only field is deliberately not +/// defined here: PostgreSQL reports its own zeros through the same keys. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactReport { + /// `none` or `rebuild`. + pub path: CompactPath, + /// The generation before and after. + pub generation: CompactGeneration, + /// Bytes in wholly dead packs after the metadata walk. + pub dead_bytes_bound: u64, + /// The same bytes after the payload sweep. + pub dead_bytes_refined: u64, + /// The collected history rows' `logical` bytes. + pub logical_discarded: u64, + /// Free pages × page size: the build's artifact, or the live `none` Store. + pub backend_reusable: u64, + /// Old file size minus the new one. + pub physical_shrink: u64, + /// Bytes the mark keeps that no retained root needs. + pub kept_waste: CompactKeptWaste, + /// Peak temporary bytes the run needed. + pub peak_temp_bytes: u64, + /// The artifact set left after settle; empty on success. + pub residue: CompactResidue, + /// §13: the report is an estimate, not a measurement. + pub estimate: bool, + /// One reason per zero report field, from the closed vocabulary. + pub zero_reasons: Vec, +} + +/// A backend receipt that projects to the one §11 [`CompactReport`]. +/// +/// SQLite's [`CompactReceipt`] implements it here. T9's PostgreSQL rebuild +/// implements it for the receipt of its Phase M/A/B/C run, which keeps the +/// PostgreSQL path on this shared shape without a PostgreSQL-only field. +pub trait CompactReportSource { + /// Projects this receipt as the §11 report. + fn compact_report(&self) -> CompactReport; +} + +impl CompactReportSource for CompactReceipt { + fn compact_report(&self) -> CompactReport { + CompactReport { + path: self.path, + generation: CompactGeneration { + before: self.generation_before, + after: self.generation_after, + }, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + logical_discarded: self.logical_discarded, + backend_reusable: self.backend_reusable, + physical_shrink: self.physical_shrink, + kept_waste: self.kept_waste.clone(), + peak_temp_bytes: self.peak_temp_bytes, + residue: self.residue.clone(), + estimate: self.estimate, + zero_reasons: self.zero_reasons.clone(), + } + } +} + +impl From<&CompactReceipt> for CompactReport { + fn from(receipt: &CompactReceipt) -> Self { + receipt.compact_report() + } +} + +impl From for CompactReport { + fn from(receipt: CompactReceipt) -> Self { + receipt.compact_report() + } +} + +/// The operation `store.status` shows while a compaction runs, or after a dead +/// owner left one (§11): `mark`, `build`, `verify` or `install`. +/// +/// It is read from `.compact-op` alone; the Store database, its +/// artifacts and the directory listing are never opened for it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactStatus { + /// The operation identity as `op-`. + pub id: String, + /// The call that owns it: `store.compact`. + pub name: &'static str, + /// The phase the run recorded. + pub phase: String, + /// When the run started, in Unix milliseconds. + pub started: i64, + /// Whether the outcome is unknown: a settled run removes its record. + pub uncertain: bool, + /// `busy` while the owner lives, `unresolved` once it is gone. + pub state: &'static str, + /// The backend's phases, in order, for a progress view. + pub phases: &'static [&'static str], +} diff --git a/engine/crates/layerfs-persistence/tests/epoch_fence.rs b/engine/crates/layerfs-persistence/tests/epoch_fence.rs new file mode 100644 index 0000000000..48dabe8a29 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/epoch_fence.rs @@ -0,0 +1,442 @@ +//! #41 T6.5: the writer-side `project_epoch` fence (spec §5). Every operation +//! that creates a Commit or Layer — `commit_branch`, `add_layer`, +//! `merge_branch`, and the Workspace/file Commit path that calls +//! `commit_branch` — reads the project epoch in its write transaction and +//! re-checks it before that transaction commits. A change means a removal +//! marker (which bumps the epoch in its own transaction) raced the write; a +//! missing row means the project was deleted and compacted away. Both are +//! `Conflict`, and the whole transaction rolls back. +mod support; + +use layerfs_history::error::{HistoryError, Missing}; +use layerfs_history::records::{ + AddLayerOutcome, AddLayerRequest, BranchMergeRequest, CommitMeta, CommitOutcome, CommitRecord, + CommitRequest, CommitVia, ForkRequest, ForkSource, MergeCommit, MergeHead, MergeSide, + StackInitialization, +}; +use layerfs_history::{BranchId, HistoryCatalog, HistoryCatalogConfig, LayerStackId, NodeId}; +use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; +use support::{branch_id, name, root, stack, Temp}; + +const BINDING: &[u8] = b"epoch-fence"; +const CURSOR: [u8; 32] = [53; 32]; + +fn config(path: &std::path::Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn catalog(temp: &Temp) -> layerfs_persistence::HistoryProvider { + support::sqlite::create( + &temp.join("store.sqlite"), + &HistoryCatalogConfig { + cursor_key: CURSOR, + binding_key: BINDING.to_vec(), + incarnation: 3, + }, + ) + .expect("catalog") +} + +fn writable(path: &std::path::Path) -> layerfs_persistence::HistoryProvider { + SqliteHandles::open_writable(config(path), BINDING, CURSOR) + .expect("writable catalog") + .history +} + +/// Project `proj` with one Branch `work`, at its genesis Layer. +fn project(history: &dyn HistoryCatalog) -> (LayerStackId, BranchId, layerfs_history::LayerId) { + let record = history + .initialize_layerstack(&StackInitialization { + logical: 1, + stack: stack(1), + name: name("proj"), + scope: root(200), + profile: root(201), + genesis_root: root(1), + }) + .unwrap(); + let branch = branch_id(1); + history + .fork(&ForkRequest { + stack: stack(1), + branch, + name: name("work"), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); + (stack(1), branch, record.head_layer) +} + +fn commit_request(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRequest { + let record = history.branch(branch).unwrap().unwrap(); + let expected_root = match record.head_commit { + Some(head) => history.commit(head).unwrap().unwrap().root, + None => history.layer(record.base_layer).unwrap().unwrap().root, + }; + CommitRequest { + branch, + expected_head: record.head_commit, + expected_base: record.base_layer, + expected_root, + candidate_root: root(tag), + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + } +} + +fn commit(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRecord { + match history + .commit_branch(&commit_request(history, branch, tag)) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +fn layer_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, +) -> AddLayerRequest { + let record = history.branch(branch).unwrap().unwrap(); + let head = history.layer_stack(stack).unwrap().unwrap(); + AddLayerRequest { + stack, + branch, + commit: record.head_commit.expect("a committed Branch"), + expected_stack_head: head.head_layer, + expected_branch_base: record.base_layer, + } +} + +fn publish( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, +) -> layerfs_history::LayerRecord { + match history + .add_layer(&layer_request(history, stack, branch)) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +/// A merge of `work`'s Commit with a second Branch still at the genesis Layer: +/// one new Commit, second parent the genesis node, root `tag`. +fn merge_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, + commit: layerfs_history::CommitId, + genesis: layerfs_history::LayerId, + tag: u8, +) -> BranchMergeRequest { + let side = branch_id(2); + history + .fork(&ForkRequest { + stack, + branch: side, + name: name("side"), + source: ForkSource::Layer(genesis), + }) + .unwrap(); + BranchMergeRequest { + stack, + branch: branch_id(9), + name: name("merged"), + sides: [ + (MergeSide::Branch(branch), NodeId::Commit(commit)), + (MergeSide::Branch(side), NodeId::Layer(genesis)), + ], + head: MergeHead::Commit(Box::new(MergeCommit { + base: genesis, + parent: Some(commit), + second_parent: NodeId::Layer(genesis), + root: root(tag), + meta: CommitMeta::new(CommitVia::Merge), + })), + } +} + +fn epoch(path: &std::path::Path, stack: LayerStackId) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .query_row( + "SELECT epoch FROM project_epoch WHERE layer_stack_id = ?1", + [stack.as_slice()], + |row| row.get(0), + ) + .unwrap() +} + +/// `(commit rows, layer rows)` in the Store file, to prove a refusal wrote +/// nothing. +fn counts(path: &std::path::Path) -> (i64, i64) { + let connection = rusqlite::Connection::open(path).unwrap(); + let count = |table: &str| -> i64 { + connection + .query_row(&format!("SELECT count(*) FROM \"{table}\""), [], |row| { + row.get(0) + }) + .unwrap() + }; + (count("commit"), count("layer")) +} + +fn trigger(path: &std::path::Path, table: &str) { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .execute_batch(&format!( + "CREATE TRIGGER t65_epoch_race BEFORE INSERT ON \"{table}\" BEGIN \ + UPDATE project_epoch SET epoch = epoch + 1 \ + WHERE layer_stack_id = NEW.layer_stack_id; END;" + )) + .unwrap(); +} + +fn drop_trigger(path: &std::path::Path) { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .execute_batch("DROP TRIGGER t65_epoch_race") + .unwrap(); +} + +#[test] +fn a_normal_commit_publish_and_merge_leave_the_epoch_alone() { + let temp = Temp::new("epoch-fence-normal"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + + let first = commit(&history, work, 5); + assert_eq!(epoch(&path, shop), 0, "a Commit does not bump the epoch"); + + let layer = publish(&history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!(epoch(&path, shop), 0, "a published Layer does not bump it"); + + let merge = history + .merge_branch(&merge_request(&history, shop, work, first.id, genesis, 60)) + .unwrap(); + assert_eq!(merge.branch.name.as_str(), "merged"); + assert_eq!(epoch(&path, shop), 0, "a merge does not bump it"); + assert_eq!((counts(&path).0, counts(&path).1), (2, 2)); +} + +#[test] +fn a_missing_epoch_row_refuses_every_writer_with_conflict() { + let temp = Temp::new("epoch-fence-missing"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let first = commit(&history, work, 5); + let side_request = merge_request(&history, shop, work, first.id, genesis, 61); + let before = counts(&path); + + let connection = rusqlite::Connection::open(&path).unwrap(); + connection + .execute( + "DELETE FROM project_epoch WHERE layer_stack_id = ?1", + [shop.as_slice()], + ) + .unwrap(); + drop(connection); + + // No fence row is left after `project.delete` + compaction: every writer + // refuses `Conflict`, never "unchanged". + let refused = history + .commit_branch(&commit_request(&history, work, 6)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = history + .add_layer(&layer_request(&history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = history.merge_branch(&side_request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + + // No partial write: no Commit, no Layer, and the Branch head is unchanged. + assert_eq!(counts(&path), before); + let snapshot = history.branch_snapshot(work).unwrap().unwrap(); + assert_eq!(snapshot.branch.head_commit, Some(first.id)); + assert_eq!( + history.layer_stack(shop).unwrap().unwrap().head_layer, + genesis + ); + assert!(history.branch(branch_id(9)).unwrap().is_none()); +} + +#[test] +fn a_removal_first_refuses_the_later_commit_and_writes_nothing() { + let temp = Temp::new("epoch-fence-removal-first"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + let _first = commit(&history, work, 5); + let before = counts(&path); + + // Capture the request while the Branch is still visible; the removal then + // wins, and the filtered Branch resolution hides the retired Branch before + // the fence opens, so the write is refused and leaves no Commit behind. + let request = commit_request(&history, work, 6); + history.mark_branch_removed(work, 1_000).unwrap(); + assert_eq!(epoch(&path, shop), 1, "the marker bumped the epoch once"); + + let refused = history.commit_branch(&request).unwrap_err(); + assert!( + matches!( + refused, + HistoryError::Missing(Missing::Branch) | HistoryError::Conflict("project epoch") + ), + "{refused:?}" + ); + assert_eq!(counts(&path), before); +} + +#[test] +fn a_concurrent_commit_and_removal_have_one_winner() { + let temp = Temp::new("epoch-fence-race"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + drop(history); + + let left = writable(&path); + let right = writable(&path); + let request = commit_request(&left, work, 7); + let (committed, removed) = std::thread::scope(|scope| { + let committed = scope.spawn(move || left.commit_branch(&request)); + let removed = scope.spawn(move || right.mark_branch_removed(work, 2_000)); + (committed.join().unwrap(), removed.join().unwrap()) + }); + + // The two writes serialize: a Commit first, then the removal retires the + // Branch with it (or is `Busy` and retries); or the removal first, and the + // Commit is refused, leaving no row behind. + match (&committed, &removed) { + (Ok(_), Ok(_)) => { + assert_eq!(epoch(&path, shop), 1); + assert_eq!(counts(&path).0, 1, "the Commit stays under the removal"); + } + (Ok(_), Err(HistoryError::Busy)) => { + assert_eq!(epoch(&path, shop), 0, "the removal has to retry"); + assert_eq!(counts(&path).0, 1); + } + ( + Err( + HistoryError::Busy + | HistoryError::Missing(Missing::Branch) + | HistoryError::Conflict(_), + ), + Ok(_), + ) => { + assert_eq!(epoch(&path, shop), 1); + assert_eq!(counts(&path).0, 0, "the refused Commit left nothing"); + } + other => panic!("unexpected outcome: {other:?}"), + } +} + +#[test] +fn an_epoch_change_during_a_commit_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-commit"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, _genesis) = project(&history); + + // A trigger stands in for the removal marker that bumps the epoch after + // the writer's start read: it fires inside the writer's own transaction, + // before its re-check. + trigger(&path, "commit"); + let refused = history + .commit_branch(&commit_request(&history, work, 5)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path).0, 0, "the Commit rolled back"); + assert_eq!(epoch(&path, shop), 0, "the fake bump rolled back too"); + assert!(history.branch(work).unwrap().unwrap().head_commit.is_none()); + + // Without the interleaved change the same Commit lands. + drop_trigger(&path); + let record = commit(&history, work, 5); + assert_eq!(record.stack, shop); + assert_eq!(epoch(&path, shop), 0); + assert_eq!( + history.branch(work).unwrap().unwrap().head_commit, + Some(record.id) + ); +} + +#[test] +fn an_epoch_change_during_a_publish_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-layer"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let _first = commit(&history, work, 5); + let before = counts(&path); + + trigger(&path, "layer"); + let refused = history + .add_layer(&layer_request(&history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path), before, "the Layer rolled back"); + assert_eq!(epoch(&path, shop), 0); + assert_eq!( + history.layer_stack(shop).unwrap().unwrap().head_layer, + genesis + ); + + drop_trigger(&path); + let layer = publish(&history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!(counts(&path).1, before.1 + 1); +} + +#[test] +fn an_epoch_change_during_a_merge_is_detected_and_rolled_back() { + let temp = Temp::new("epoch-fence-trigger-merge"); + let path = temp.join("store.sqlite"); + let history = catalog(&temp); + let (shop, work, genesis) = project(&history); + let first = commit(&history, work, 5); + let request = merge_request(&history, shop, work, first.id, genesis, 62); + let before = counts(&path); + + trigger(&path, "commit"); + let refused = history.merge_branch(&request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&path), before, "the merge Commit rolled back"); + assert_eq!(epoch(&path, shop), 0); + assert!(history.branch(branch_id(9)).unwrap().is_none()); + + drop_trigger(&path); + let merge = history.merge_branch(&request).unwrap(); + assert_eq!(merge.branch.name.as_str(), "merged"); + assert!(merge.branch.head_commit.is_some()); +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json new file mode 100644 index 0000000000..c67cf099d2 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json @@ -0,0 +1,407 @@ +{ + "digestVector": { + "absentHeadDigest": "6864c4f336ed6ccfb6ea864bd565b23b97fbbff571e6b627d0043bce7218a724", + "commit": [ + "121c28e0f8ea5f06e9f3a5ad08f7f2bd7b2ad1afd677d57099991f41678ab19ec1" + ], + "retiredBranchDigest": "dafcdd21820991184e6f21e7c80cbdd0f034fdc1d62ebf9502aeae381770c857", + "retiredStack": [ + "3107070707070707070707070707070707" + ] + }, + "emptyNone": { + "mark": { + "liveHeadsDigest": "3556f439f3ea4d21d189c484e3c117492779e5c62249481882cd6011cf1a13a9", + "objectCount": 0, + "packs": [] + }, + "receipt": { + "backendReusable": 0, + "deadBytesBound": 0, + "deadBytesRefined": 0, + "estimate": true, + "generationAfter": 7, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 0, + "deltaBases": { + "bytes": 0, + "references": [] + }, + "inodeBlocksBytes": 0 + }, + "logicalDiscarded": 0, + "path": "none", + "peakTempBytes": 0, + "physicalShrink": 0, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "deadBytesBound", + "reason": "empty-store" + }, + { + "field": "deadBytesRefined", + "reason": "empty-store" + }, + { + "field": "logicalDiscarded", + "reason": "no-history" + }, + { + "field": "physicalShrink", + "reason": "none-path" + }, + { + "field": "backendReusable", + "reason": "none-path" + }, + { + "field": "keptWaste", + "reason": "no-waste" + }, + { + "field": "peakTempBytes", + "reason": "none-path" + } + ] + }, + "totalsAfter": { + "canonical": 0, + "logical": 0, + "metadata": 0, + "stored": 0 + }, + "totalsBefore": { + "canonical": 0, + "logical": 0, + "metadata": 0, + "stored": 0 + } + }, + "noDeadNone": { + "commit": [ + "120dee8be25262ff5277cf6682a1b66b1157c2f1aa1ed551e3a3c140ec4d41e250" + ], + "layer": [ + "3228ecf26544a2fea9ffeecc6829ab584e2a4fa4f5e7be920696d98ed8d843e749" + ], + "mark": { + "liveHeadsDigest": "b04ee485d7c6002f22a8ab6d12c206ad84254435fb1067e952883c8f3012128d", + "objectCount": 8, + "packs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ] + }, + "receipt": { + "backendReusable": 32768, + "deadBytesBound": 0, + "deadBytesRefined": 0, + "estimate": true, + "generationAfter": 7, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 0, + "deltaBases": { + "bytes": 0, + "references": [] + }, + "inodeBlocksBytes": 190 + }, + "logicalDiscarded": 0, + "path": "none", + "peakTempBytes": 0, + "physicalShrink": 0, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "deadBytesBound", + "reason": "no-waste" + }, + { + "field": "deadBytesRefined", + "reason": "no-waste" + }, + { + "field": "logicalDiscarded", + "reason": "none-path" + }, + { + "field": "physicalShrink", + "reason": "none-path" + }, + { + "field": "peakTempBytes", + "reason": "none-path" + } + ] + }, + "totalsAfter": { + "canonical": 890, + "logical": 410, + "metadata": 808, + "stored": 936 + }, + "totalsBefore": { + "canonical": 890, + "logical": 410, + "metadata": 808, + "stored": 936 + } + }, + "rebuild": { + "after": { + "branchRetired": [ + "1101010101010101010101010101010101" + ], + "deadPacksGone": true, + "fileDigestCollected": false, + "fileDigestRetained": true, + "fullDoctor": "clean", + "layerStackRetired": 0, + "liveHeadsDigestUnchanged": true, + "liveReadable": true + }, + "fixture": { + "branches": { + "collected": [ + "1103030303030303030303030303030303", + "1104040404040404040404040404040404" + ], + "live": [ + "1102020202020202020202020202020202" + ], + "retainedRetired": [ + "1101010101010101010101010101010101" + ] + }, + "commits": { + "collected": [ + "12504118308814458073cf694c832277a9a3edf25438d6c29033749d7d4b6851e9", + "12f41169281d83c82e136fb7a69850b3d72ea90ef637dd4145bdbf1077cbafbb45" + ], + "retained": [ + "12ae744d795440a3f12341548e1f1ae118e384450d2454aa6e1bf952fc5fa5b9e5", + "129c35a564ebb33405e16e5ebc548aa5e01640914517db7a7a2948c0c22e8600e6", + "128512651dcea9f37272d401d22f19aefcac65d46f4a66dcf9596078ce4f1efb2f" + ] + }, + "deadBytesBound": 182635, + "deltaBaseReferences": [ + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85" + ], + "deltaChainPackIds": [ + 25, + 26, + 27 + ], + "inodeBlocksBytes": 665, + "layers": { + "collected": [ + "32c89ce4fa2bda2b0fc9ef449faa224fe05895352e5333ec7fe9fb4e35fb8d5409", + "32e7c02e338721609c58801a6195a8f9ececb7875b3715a6356f811534f9185dda" + ], + "retained": [ + "32e2fce8a15669a223b821c5a12720915f959a3bd8b678843a2d4c836f49973d79", + "3228fe19ff26677849bef9617a26fba8c9d70b6bb0f30f4ceb35dac89164bba157", + "32219e07e8572a40e056647a2db0febee7cff5e951a91974ce130bf7534d4f5ed9" + ] + }, + "objects": { + "dead": [ + "04da3ee0d5c26526b82eaacaa3611fb5ee51b53938b15b9a6e0c2ce23fac79aa", + "03cdb1082aefd8630de9ded4491f70107d1194879c3725f47604e61d6ce8464f", + "b41d84c180ddbf811874a45b54f6bb5929351b9d922e3eae16030b93ef0c0465", + "93934455f81c303589782508b25a6fc6cb5577041d09f9997dbdf9a4443da90e", + "1ccae865928d331b9f40f6b9781de1a8f2fc131c79a46b18de977a92e1fc41f6", + "d330e6950c00b9c6e5214a911a0b7438590ce84c9f732d9a5dcf0337a3ee3b3c", + "9e1ae9f693108d70236c4b834eae7066e95a1c6c88319598521b3a30178b335b", + "9b4f54e459f725c48c96be90af12c6918e3135964d38629313fe09fa548c2c63", + "c260b43bb74755139dd041a0906db269d0ed8039181fcb0d1ec6226ae1e1638e", + "1d0d2e8c5f735a3437e84a60119ae1daa08a34d769969c460a7a7695361bd88b", + "aa2d83fe8c3e636f1c8086bcf96bdee0081bd564ed730f627e1bf130d3f6725b", + "c7885caa9ac150fca12ca3352cec6e1a9ab8d585332e3def1ccb084cdb710368", + "8e65a309ef2a363965849a9b19bb3b005e565d4a678f09893d721a8031d9977e", + "cc9d4bace21b47206e4cacdacc1dcce819210b79787c33a71c1bdfd553bca024", + "1352f9c655388061372fe949b37530b1284bc50c4cd7f979ff25d848e42a43ac", + "968c266c9fd518965edb975d40714441b48b142ea1bef48ad6164da39f307a80", + "773f0e9363afec9a9bc0294e492e84e9c2985de50fcb21294b6762774309e5b5", + "a96b6772f5b6c00682d3e8545e583c6f99f3e404015efb471a8c5ccb4d42f1e4", + "90bd063fc258fce027928d7d05334f076868c9fa5effe998c09a18c79b9b3c1b" + ], + "live": [ + "bea0e9f6fdfdb07ff631ac8213c85caf98cba1ff2aeb342fe7f99ff210b1498e", + "f7b81686c824c3de2c23077b02f554936618e20621719a0682c9430358290f73", + "4ed7a5abe47b79b9c281d7bbd914e858162d5d005c3aff0e05c5b1bf484e3403", + "3df6d13237972839afaa61737e92f27e60793cc25d61b7f8515b6f8374d3dc2d", + "43652a535c7f61a5caef1d1574a661b4c19b02d848d6463430d933920c148152", + "b57fc5ee3a4f8270aecd33c137a39b68336669fa1d06f11acb8f02e6ff6e6849", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85", + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "3806265ed5cd11005a9918e03c34377954012805cdfe908877678c1395111776", + "01fd2163213bdea11e96e3fb9f27ea76a6112497895d7e4751f541509d5b7e25" + ] + }, + "packs": { + "dead": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 19, + 20, + 21, + 22, + 23, + 24, + 29, + 30, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52 + ], + "live": [ + 13, + 14, + 15, + 16, + 17, + 18, + 25, + 26, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43 + ], + "shared": 28, + "sharedDeadGroupBytes": 265, + "sharedLength": 562 + }, + "stacks": { + "collected": [ + "3102020202020202020202020202020202" + ], + "live": [ + "3101010101010101010101010101010101" + ] + } + }, + "mark": { + "liveHeadsDigest": "d3a3eadd25ce23512bc358a91b3195ee8d29ee027f83840d567dc6540f47bab9", + "objectCount": 23, + "packs": [ + 13, + 14, + 15, + 16, + 17, + 18, + 25, + 26, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43 + ] + }, + "receipt": { + "backendReusable": 0, + "deadBytesBound": 182635, + "deadBytesRefined": 182635, + "estimate": true, + "generationAfter": 8, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 265, + "deltaBases": { + "bytes": 571, + "references": [ + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85" + ] + }, + "inodeBlocksBytes": 665 + }, + "logicalDiscarded": 160, + "path": "rebuild", + "peakTempBytes": 389120, + "physicalShrink": 221184, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "backendReusable", + "reason": "no-waste" + } + ] + }, + "totalsAfter": { + "canonical": 134988, + "logical": 1090, + "metadata": 2024, + "stored": 134670 + }, + "totalsBefore": { + "canonical": 317453, + "logical": 1250, + "metadata": 3644, + "stored": 317305 + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs new file mode 100644 index 0000000000..acdcebce6a --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs @@ -0,0 +1,582 @@ +//! #41 T9: the §8 PostgreSQL rebuild. A compaction takes the Store-scoped +//! exclusive intent, records the mark tuple, builds the retained payload in a +//! scratch schema, swaps `pack`/`object_location` at `G -> G + 1`, deletes only +//! the persisted `gc_dead` set, settles the totals once, and leaves no +//! scratch schema or op row. A leftover at `G` is removed; one at `G + 1` +//! finishes Phase C (lost-reply resume). A conflicting lock is `Busy` (55P03). +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRequest, + CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, LayerStackId, + LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use support::{branch_id, name, stack, BINDING}; + +/// Drops the test's schema when the test ends, pass or fail. +struct Schema { + url: String, + selection: PostgresSelection, +} +impl Schema { + fn new(url: &str, tag: &str) -> Self { + let name = format!("lfs_t9_{tag}_{}", std::process::id()); + let selection = PostgresSelection::new(url, name); + let schema = Self { + url: url.to_owned(), + selection, + }; + schema.drop_schema(); + schema + } + fn open(&self) -> PostgresHandles { + PostgresHandles::open_writable(&self.selection, BINDING, [71; 32]).expect("Store open") + } + fn create(&self) -> PostgresHandles { + PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) + .expect("Store creation") + } + fn client(&self) -> postgres::Client { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); + client + .batch_execute(&format!("SET search_path TO {}", self.selection.schema)) + .expect("search path"); + client + } + fn count(&self, table: &str) -> i64 { + let sql = format!("SELECT count(*) FROM {}.{}", self.selection.schema, table); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn count_where(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn scalar(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("scalar").get(0) + } + fn exists(&self, sql: &str) -> bool { + self.scalar(sql) != 0 + } + fn scratch_exists(&self, op: u8) -> bool { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + self.exists(&format!( + "SELECT (CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname='{scratch}') THEN 1 ELSE 0 END)::bigint" + )) + } + fn drop_scratch(&self, op: u8) { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!("DROP SCHEMA IF EXISTS {scratch} CASCADE")); + } + } + fn drop_schema(&self) { + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!( + "DROP SCHEMA IF EXISTS {} CASCADE", + self.selection.schema + )); + } + } +} +impl Drop for Schema { + fn drop(&mut self) { + self.drop_schema(); + } +} + +fn catalog() -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + } +} + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &PostgresHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &PostgresHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: Vec::new(), + }) + .unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &PostgresHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: stack(tag), + name: name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, +) -> layerfs_history::CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn located(handles: &PostgresHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn generation(handles: &PostgresHandles) -> u64 { + handles.history.incarnation() +} + +/// The live generation from the Store row (the open session caches the value +/// it opened at, as it does on SQLite). +fn db_generation(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT catalog_incarnation FROM {s}.history_meta WHERE id=1")) + .unwrap() +} + +fn clean(handles: &PostgresHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +fn assert_settled(schema: &Schema, op: u8) { + assert_eq!(schema.count("gc_op"), 0, "gc_op is cleared on settle"); + assert_eq!(schema.count("gc_dead"), 0, "gc_dead is cleared on settle"); + assert_eq!( + schema.count_where("SELECT count(*) FROM {s}.gc_intent WHERE kind='compaction'"), + 0, + "the compaction intent is released" + ); + assert!( + !schema.scratch_exists(op), + "the recorded scratch schema is dropped" + ); + let logical = schema.scalar("SELECT logical FROM {s}.store_totals WHERE id=1"); + let sum = schema.scalar( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.layer)", + ); + assert_eq!(logical, sum, "store_totals.logical tracks the history rows"); + let canonical = schema.scalar("SELECT canonical FROM {s}.store_totals WHERE id=1"); + let canonical_sum = + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"); + assert_eq!(canonical, canonical_sum, "store_totals.canonical"); + let stored = schema.scalar("SELECT stored FROM {s}.store_totals WHERE id=1"); + let stored_sum = schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack"); + assert_eq!(stored, stored_sum, "store_totals.stored"); + let metadata = schema.scalar("SELECT metadata FROM {s}.store_totals WHERE id=1"); + let metadata_sum = + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"); + assert_eq!(metadata, metadata_sum, "store_totals.metadata"); +} + +#[test] +fn an_empty_store_takes_the_none_path() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "empty"); + let handles = schema.create(); + let before = generation(&handles); + let receipt = handles.compact(&request(1)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "backendReusable" && zero.reason == "postgresql-in-place")); + assert_settled(&schema, 1); + assert_eq!(generation(&handles), before); + handles.history.mark().unwrap(); +} + +#[test] +fn a_live_store_takes_the_none_path() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "live"); + let handles = schema.create(); + let genesis = filesystem_root(&handles, 1); + initialize(&handles, 1, genesis); + let before = generation(&handles); + let receipt = handles.compact(&request(2)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.physical_shrink, 0); + assert_settled(&schema, 2); + assert!(located(&handles, genesis)); +} + +#[test] +fn a_wholly_dead_pack_is_rebuilt_away_and_totals_settle() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "rebuild"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"nobody reaches this object"); + assert!(located(&handles, dead)); + let before = generation(&handles); + let receipt = handles.compact(&request(3)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.dead_bytes_bound > 0); + assert_eq!(receipt.backend_reusable, 0); + assert_settled(&schema, 3); + assert_eq!(db_generation(&schema), before + 1); + assert!(!located(&handles, dead), "the dead object is collected"); + assert!(located(&handles, live), "the retained object survives"); + clean(&schema.open()); +} + +#[test] +fn retained_roots_survive_a_rebuild_that_collects_a_deleted_project() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "retained"); + let handles = schema.create(); + let live_genesis = filesystem_root(&handles, 1); + let live_commit = filesystem_root(&handles, 2); + let live = initialize(&handles, 1, live_genesis); + let work = branch_id(1); + fork(&handles, live.id, work, 1); + let retained_commit = commit(&handles, work, live_commit); + publish(&handles, live.id, work); + + let dead_genesis = filesystem_root(&handles, 3); + let dead_commit = filesystem_root(&handles, 4); + let gone = initialize(&handles, 2, dead_genesis); + let dead_branch = branch_id(2); + fork(&handles, gone.id, dead_branch, 2); + let collected_commit = commit(&handles, dead_branch, dead_commit); + publish(&handles, gone.id, dead_branch); + handles.history.mark_stack_deleted(gone.id, 5_000).unwrap(); + + let before = generation(&handles); + let receipt = handles.compact(&request(4)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert!( + receipt.logical_discarded > 0, + "the deleted project's rows are collected" + ); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 4); + // The retained project and its history survive. + assert!(handles.history.layer_stack(live.id).unwrap().is_some()); + assert!(handles.history.branch(work).unwrap().is_some()); + assert!(handles.history.commit(retained_commit).unwrap().is_some()); + assert!(located(&handles, live_genesis)); + assert!(located(&handles, live_commit)); + // The deleted project's history and objects are gone. + assert!(handles.history.layer_stack(gone.id).unwrap().is_none()); + assert!(handles.history.commit(collected_commit).unwrap().is_none()); + assert!(!located(&handles, dead_genesis)); + assert!(!located(&handles, dead_commit)); + clean(&schema.open()); +} + +#[test] +fn a_lost_swap_reply_is_resumed_by_the_next_compact() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "lost"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack collected by the first round"); + let before = generation(&handles); + let error = handles + .compact(&request_at(5, CompactFailpoint::PostgresAfterSwap)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + // The swap committed: the Store is at G + 1 and the op still names Phase C. + assert_eq!(db_generation(&schema), before + 1); + assert_eq!(schema.count("gc_op"), 1); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + let receipt = handles.compact(&request(6)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 6); + assert_eq!(db_generation(&schema), before + 1); + clean(&schema.open()); +} + +#[test] +fn the_same_session_resumes_an_uncertain_round() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "same"); + schema.drop_scratch(11); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the resumed round collects"); + let before = generation(&handles); + let error = handles + .compact(&request_at(11, CompactFailpoint::PostgresAfterMark)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + // The same session is the recorded owner and is idle; it adopts its own + // uncertain round instead of terminating itself. + let receipt = handles.compact(&request(12)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 12); + clean(&schema.open()); +} + +#[test] +fn a_stale_owner_backend_is_terminated_before_the_next_round() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "terminate"); + schema.drop_scratch(9); + let first = schema.create(); + let live = filesystem_root(&first, 1); + initialize(&first, 1, live); + let _dead = whole_file(&first, b"a pack the adopted round collects"); + let before = generation(&first); + let error = first + .compact(&request_at(9, CompactFailpoint::PostgresAfterMark)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + let pid = schema.scalar("SELECT pid FROM {s}.gc_op WHERE id=1"); + assert!(pid > 0, "the op records its owner backend"); + // The recorded owner is alive but idle: the next round terminates that + // (pid, backend_start) and adopts the op as a dead owner (§9, §12). + let second = schema.open(); + let receipt = second.compact(&request(10)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_eq!(db_generation(&schema), before + 1); + assert_settled(&schema, 10); + let mut gone = false; + for _ in 0..50 { + if schema.scalar(&format!( + "SELECT count(*) FROM pg_stat_activity WHERE pid={pid}" + )) == 0 + { + gone = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + assert!(gone, "the stale owner backend is terminated"); + clean(&schema.open()); +} + +#[test] +fn an_exclusive_table_lock_makes_the_swap_busy() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "busy"); + schema.drop_scratch(7); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + // Phase A never touches store_totals; the swap's ACCESS EXCLUSIVE NOWAIT + // does, so this lock is met exactly at the swap (SQLSTATE 55P03). + let mut blocker = schema.client(); + blocker + .batch_execute(&format!( + "BEGIN; LOCK TABLE {}.binding_hint_block IN ACCESS EXCLUSIVE MODE;", + schema.selection.schema + )) + .unwrap(); + let error = handles.compact(&request(7)).unwrap_err(); + blocker.batch_execute("ROLLBACK").unwrap(); + assert!( + matches!(error, layerfs_history::HistoryError::Busy), + "a 55P03 lock at the swap is Busy, got {error:?}" + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs new file mode 100644 index 0000000000..8445f4f917 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -0,0 +1,865 @@ +//! #41 T8: the §7 SQLite rebuild. A compaction takes the writer lane, runs the +//! mark, chooses `none`/`rebuild` from the wholly dead packs, builds and +//! verifies `.compact`, installs it at `G → G+1`, and leaves no +//! artifact; a recorded leftover is judged by the generation and removed by +//! the next `store.compact`. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, + CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, + HistoryError, LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + compact_op_status, CompactFailpoint, CompactPath, CompactReport, CompactReportSource, + CompactRequest, PersistenceConfig, SqliteAcquisitionSchema, SqliteHandles, SqlitePackLayout, + SqlitePersistenceProfile, +}; +use layerfs_storage::{ + port::acquisition::{Acquisition, Begin}, + port::PackPersistence, Storage, StoragePolicy, +}; +use std::cell::RefCell; +use std::path::{Path, PathBuf}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-test-binding"; + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn compact_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact"); + PathBuf::from(name) +} + +fn op_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact-op"); + PathBuf::from(name) +} + +fn visited_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact-visited"); + PathBuf::from(name) +} + + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &SqliteHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new( + profile_id(), + scope_for_seed([1; 32]), + 1, + inode_id, + ) + .unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit(handles: &SqliteHandles, branch: BranchId, candidate_root: ObjectId) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root, + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +#[test] +fn an_empty_store_takes_the_none_path() { + let (_temp, handles, path) = create_store(); + let before = generation(&handles); + let receipt = handles.compact(&request(1)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before); + reopened.history.mark().unwrap(); +} + +#[test] +fn a_store_without_wholly_dead_packs_takes_the_none_path() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let before = generation(&handles); + let receipt = handles.compact(&request(2)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!( + receipt.dead_bytes_refined, receipt.dead_bytes_bound, + "the none path does not refine" + ); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.physical_shrink, 0); + assert!(!compact_path(&path).exists()); + assert!(!op_path(&path).exists()); +} + +#[test] +fn a_wholly_dead_pack_is_rebuilt_away() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"nobody reaches this object"); + let before = generation(&handles); + let receipt = handles.compact(&request(3)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + for suffix in ["-wal", "-shm", "-journal"] { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + assert!(!PathBuf::from(name).exists(), "sidecar {suffix} remains"); + } + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before + 1); + assert!(located(&reopened, live), "the retained object survives"); + assert!(!located(&reopened, dead), "the dead object is collected"); + clean(&reopened); +} + +#[test] +fn retained_history_and_its_objects_survive_a_rebuild() { + let (_temp, handles, path) = create_store(); + let genesis = filesystem_root(&handles, 1); + let committed = filesystem_root(&handles, 2); + let stack = initialize(&handles, 1, genesis); + let work = support::branch_id(1); + fork(&handles, stack.id, work, 1); + let commit_id = commit(&handles, work, committed); + publish(&handles, stack.id, work); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let before = generation(&handles); + let receipt = handles.compact(&request(4)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before + 1); + assert!(reopened.history.commit(commit_id).unwrap().is_some()); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); + for id in [genesis, committed] { + assert!(located(&reopened, id), "object {id} survives"); + } + assert!(!located(&reopened, dead)); + clean(&reopened); +} + +#[test] +fn a_stray_staged_name_is_refused_and_not_deleted() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let stray = compact_path(&path); + std::fs::write(&stray, b"not a Store").unwrap(); + let error = handles.compact(&request(5)).unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::Exists(_)), + "{error:?}" + ); + assert!(stray.exists(), "a name no record names is not deleted"); +} + +#[test] +fn a_build_failpoint_leaves_a_restartable_artifact() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&request_at(6, CompactFailpoint::AfterBuild)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + assert!(compact_path(&path).exists(), "the staged Store remains"); + assert!(visited_path(&path).exists(), "the visited scratch remains"); + assert!(op_path(&path).exists(), "the op record remains"); + + let reopened = reopen(&path); + assert_eq!( + generation(&reopened), + before, + "the old Store is still the live one" + ); + let receipt = reopened.compact(&request(7)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let final_handles = reopen(&path); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +#[test] +fn an_installed_but_unsettled_run_is_judged_at_g_plus_one() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&request_at(8, CompactFailpoint::AfterInstall)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + assert!(op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!( + generation(&reopened), + before + 1, + "the new Store is already installed" + ); + let receipt = reopened.compact(&request(9)).unwrap(); + assert_eq!(receipt.generation_before, before + 1); + assert!(!op_path(&path).exists()); + assert!(!compact_path(&path).exists()); +} + +#[test] +fn the_report_projects_every_section_11_field() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let receipt = handles.compact(&request(30)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let report = CompactReport::from(&receipt); + assert_eq!(report.path, CompactPath::Rebuild); + assert_eq!(report.generation.before, before); + assert_eq!(report.generation.after, before + 1); + assert_eq!(report.dead_bytes_bound, receipt.dead_bytes_bound); + assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); + assert_eq!(report.logical_discarded, receipt.logical_discarded); + assert_eq!(report.backend_reusable, receipt.backend_reusable); + assert_eq!(report.physical_shrink, receipt.physical_shrink); + assert_eq!(report.kept_waste, receipt.kept_waste); + assert_eq!(report.peak_temp_bytes, receipt.peak_temp_bytes); + assert_eq!(report.residue, receipt.residue); + assert!(report.estimate); + assert_eq!(report.zero_reasons, receipt.zero_reasons); + assert!( + report + .zero_reasons + .iter() + .all(|zero| zero.wire_field() != "dead_bytes_bound"), + "a rebuild with dead bytes does not explain a zero" + ); + // `From` and the shared source trait project the same report. + assert_eq!(report, receipt.compact_report()); +} + +#[test] +fn the_empty_store_report_explains_its_zeros() { + let (_temp, handles, path) = create_store(); + let receipt = handles.compact(&request(31)).unwrap(); + let report = CompactReport::from(&receipt); + assert_eq!(report.path, CompactPath::None); + assert_eq!(report.generation.before, report.generation.after); + assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); + assert!(report.estimate); + assert!( + report.zero_reasons.iter().any(|zero| { + zero.field == "deadBytesBound" + && zero.wire_field() == "dead_bytes_bound" + && zero.reason == "empty-store" + }), + "the empty store names its zero: {:?}", + report.zero_reasons + ); + assert!(report.residue.artifacts.is_empty()); + assert!(compact_op_status(&path).is_none(), "none settles its op"); +} + +#[test] +fn a_leftover_op_reports_its_phase_and_state() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&CompactRequest { + op_id: [21; 16], + owner_pid: Some(std::process::id()), + failpoint: Some(CompactFailpoint::AfterBuild), + ..CompactRequest::default() + }) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + let op = compact_op_status(&path).expect("the leftover op record is readable"); + assert_eq!(op.op_id, [21; 16]); + assert_eq!(op.generation, i64::try_from(before).unwrap()); + assert_eq!(op.phase, "build"); + assert_eq!(op.owner_pid, Some(std::process::id())); + assert!(op.started_ms > 0); + + let busy = op.status(true); + assert_eq!(busy.id, "op-15151515151515151515151515151515"); + assert_eq!(busy.name, "store.compact"); + assert_eq!(busy.phase, "build"); + assert_eq!(busy.started, op.started_ms); + assert!(busy.uncertain, "an unsettled record is uncertain"); + assert_eq!(busy.state, "busy"); + assert_eq!( + busy.phases, + &["mark", "build", "verify", "install"][..], + "the SQLite phase order" + ); + + let leftover = op.status(false); + assert!(leftover.uncertain); + assert_eq!(leftover.state, "unresolved"); + assert_eq!(leftover.phase, "build"); + + // The next run settles the recorded leftover: artifact, intent and op go. + let reopened = reopen(&path); + let receipt = reopened.compact(&request(22)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert!(compact_op_status(&path).is_none(), "the op record is settled"); + assert!(receipt.residue.artifacts.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.reason.is_none()); + assert!(!compact_path(&path).exists()); +} + +#[test] +fn generation_increases_once_per_installed_rebuild() { + let (_temp, handles, path) = create_store(); + let first = filesystem_root(&handles, 1); + initialize(&handles, 1, first); + let _ = whole_file(&handles, b"first dead root"); + let start = generation(&handles); + assert_eq!( + handles.compact(&request(10)).unwrap().generation_after, + start + 1 + ); + let reopened = reopen(&path); + let _ = whole_file(&reopened, b"second dead root"); + let receipt = reopened.compact(&request(11)).unwrap(); + assert_eq!(receipt.generation_before, start + 1); + assert_eq!(receipt.generation_after, start + 2); + let final_handles = reopen(&path); + assert_eq!(generation(&final_handles), start + 2); + assert!(located(&final_handles, first)); + clean(&final_handles); +} + +// --- Fences, thresholds, grouped and acquisition layouts (T8.1) --- + +thread_local! { + /// The Store file the install-fence test renames over the sealed path. + static REPLACEMENT: RefCell> = const { RefCell::new(None) }; +} + +/// The test-only hook CompactRequest::before_install_hook runs: replace the +/// Store path with the prepared file after the seal and before the rename. +fn replace_store(path: &Path) { + let replacement = REPLACEMENT + .with(|slot| slot.borrow_mut().take()) + .expect("a prepared replacement"); + std::fs::rename(&replacement, path).expect("replace the sealed Store"); +} + +/// The stored length of every pack, read after the Store is closed. +fn pack_lengths(path: &Path) -> Vec { + let connection = + rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("read-only Store"); + let mut statement = connection + .prepare("SELECT length FROM pack ORDER BY pack_id") + .expect("pack rows"); + let rows = statement + .query_map([], |row| row.get::<_, i64>(0)) + .expect("pack lengths"); + rows.map(|row| u64::try_from(row.expect("a pack length")).expect("nonnegative")) + .collect() +} + +/// Every acquisition working row the rebuild must clear. +fn acquisition_rows(path: &Path) -> i64 { + let connection = + rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("read-only Store"); + ["init_operation", "init_entry", "init_host_file"] + .iter() + .map(|table| { + connection + .query_row(&format!("SELECT count(*) FROM {table}"), [], |row| { + row.get::<_, i64>(0) + }) + .expect("acquisition row count") + }) + .sum() +} + +fn create_with( + layout: SqlitePackLayout, + acquisition: SqliteAcquisitionSchema, +) -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-layout"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_pack_layout(layout) + .with_sqlite_acquisition(acquisition), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +/// One Store whose live whole-file roots are retained and whose dead +/// whole-file packs no retained root reaches. +/// +/// Every object has the same canonical length, so every pack has the same +/// stored length: the wholly dead byte ratio is dead / (live + dead) exactly. +fn ratio_store(live: usize, dead: usize) -> (Temp, SqliteHandles, PathBuf) { + let (temp, handles, path) = create_store(); + for index in 0..live { + let root = whole_file(&handles, format!("live-{index:06}").as_bytes()); + initialize(&handles, u8::try_from(index + 1).unwrap(), root); + } + for index in 0..dead { + let _ = whole_file(&handles, format!("dead-{index:06}").as_bytes()); + } + (temp, handles, path) +} + +#[test] +fn nine_percent_wholly_dead_bytes_take_the_none_path() { + let (_temp, handles, path) = ratio_store(91, 9); + let before = generation(&handles); + let receipt = handles.compact(&request(20)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_after, before); + assert!(!compact_path(&path).exists()); + assert!(!op_path(&path).exists()); + let lengths = pack_lengths(&path); + assert_eq!(lengths.len(), 100, "one pack per object: {lengths:?}"); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 9 * unit); + assert!( + 9 * unit * 10 < 100 * unit, + "9% wholly dead bytes are below the threshold" + ); +} + +#[test] +fn ten_percent_wholly_dead_bytes_rebuild_at_the_threshold() { + let (_temp, handles, path) = ratio_store(90, 10); + let before = generation(&handles); + let receipt = handles.compact(&request(21)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + let lengths = pack_lengths(&path); + assert_eq!( + lengths.len(), + 90, + "ten dead packs are collected: {lengths:?}" + ); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 10 * unit); + assert!( + 10 * unit * 10 >= (90 + 10) * unit, + "exactly 10% wholly dead bytes take the rebuild path" + ); +} + +#[test] +fn thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold() { + let (_temp, handles, path) = ratio_store(70, 30); + let before = generation(&handles); + let receipt = handles.compact(&request(22)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let lengths = pack_lengths(&path); + assert_eq!(lengths.len(), 70, "thirty dead packs are collected"); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 30 * unit); + assert!(30 * unit * 10 > (70 + 30) * unit); +} + +#[test] +fn a_store_replaced_between_seal_and_install_is_stale() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"a pack the rebuild would collect"); + + // A second valid Store at the same generation: replacing the path with it + // changes the (dev, ino) pair and leaves size, mtime and counter unrelated. + let (_replacement_temp, replacement_handles, replacement_path) = create_store(); + let replacement_generation = generation(&replacement_handles); + replacement_handles.seal().unwrap(); + REPLACEMENT.with(|slot| *slot.borrow_mut() = Some(replacement_path)); + + let request = CompactRequest { + op_id: [23; 16], + before_install_hook: Some(replace_store), + ..CompactRequest::default() + }; + let error = handles.compact(&request).unwrap_err(); + assert!(matches!(error, HistoryError::StaleStore), "{error:?}"); + + // No install: every artifact is gone and the replacement is untouched. + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), replacement_generation); +} + +#[test] +fn a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + let (_replacement_temp, replacement_handles, replacement_path) = create_store(); + replacement_handles.seal().unwrap(); + std::fs::rename(&replacement_path, &path).unwrap(); + + // The handle still owns the old inode; the fence refuses before anything runs. + let error = handles.history.mark().unwrap_err(); + assert!(matches!(error, HistoryError::StaleStore), "{error:?}"); +} + +#[test] +fn a_capacity_failure_leaves_the_store_unchanged() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let error = handles + .compact(&request_at(24, CompactFailpoint::CapacityAtBuild)) + .unwrap_err(); + assert!(matches!(error, HistoryError::Capacity(_)), "{error:?}"); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before, "the old Store is unchanged"); + assert!(located(&reopened, dead), "nothing was collected"); + let receipt = reopened.compact(&request(25)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +fn layout_round_trip(layout: SqlitePackLayout) { + let (_temp, handles, path) = create_with(layout, SqliteAcquisitionSchema::Absent); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + // The none path: the stored layout opens and a mark-only run leaves it. + let receipt = handles.compact(&request(26)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + let reopened = reopen(&path); + assert_eq!(reopened.profile().pack_layout, layout); + + // A rebuild collects a wholly dead pack and preserves the layout. + let dead = whole_file(&reopened, b"a wholly dead pack in a grouped layout"); + let before = generation(&reopened); + let receipt = reopened.compact(&request(27)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let final_handles = reopen(&path); + assert_eq!(final_handles.profile().pack_layout, layout); + assert!(located(&final_handles, live)); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +#[test] +fn group_rows_layout_opens_compacts_and_takes_none() { + layout_round_trip(SqlitePackLayout::GroupRows); +} + +#[test] +fn group_rows_indexed_layout_opens_compacts_and_takes_none() { + layout_round_trip(SqlitePackLayout::GroupRowsIndexed); +} + +#[test] +fn acquisition_layout_opens_compacts_and_clears_working_rows() { + let (_temp, handles, path) = + create_with(SqlitePackLayout::GroupRows, SqliteAcquisitionSchema::Tables); + assert_eq!( + handles.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + // A live acquisition operation: the rebuild must clear its working rows. + let _owner = handles + .acquisition + .begin(&Begin { + source_device: 1, + source_inode: 2, + stack: [3; 16], + scope: ObjectId::for_bytes(b"acquisition scope"), + }) + .unwrap(); + + // The none path leaves the acquisition tables and their working rows. + let receipt = handles.compact(&request(28)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + let reopened = reopen(&path); + assert_eq!( + reopened.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + + // A rebuild clears the working rows and keeps the acquisition tables. + let dead = whole_file(&reopened, b"a wholly dead pack with acquisition tables"); + let before = generation(&reopened); + let receipt = reopened.compact(&request(29)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let final_handles = reopen(&path); + assert_eq!( + final_handles.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + assert!(located(&final_handles, live)); + assert!(!located(&final_handles, dead)); + assert_eq!(acquisition_rows(&path), 0, "working rows are cleared"); + clean(&final_handles); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs new file mode 100644 index 0000000000..79751dd869 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs @@ -0,0 +1,577 @@ +//! #41 T12.2: the §9 crash matrix for SQLite `store.compact`. +//! +//! Each case aborts at one named failpoint, reopens the Store and asserts the +//! documented crash state: wholly old at `G` or wholly new at `G + 1`, the +//! exact artifact set, the live objects still readable, the admission record +//! and the op/phase/uncertain `store.status` projection. It then runs the next +//! `store.compact` and asserts it settles (installed) or rebuilds (pre-install) +//! and leaves a clean Store. +//! +//! The expectations below are backend-neutral data; a PostgreSQL harness can +//! reuse the same table with its own failpoints and its own artifact set. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CatalogId, CommitId, CommitMeta, CommitOutcome, + CommitRequest, CommitVia, ForkRequest, ForkSource, GcIntentKind, GcScopeKind, HistoryCatalog, + HistoryCatalogConfig, LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + compact_op_status, CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, + SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use std::path::{Path, PathBuf}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-crash-binding"; + +/// One crash point: the failpoint, the phase `store.status` must show, whether +/// the rebuild is already installed at `G + 1`, and which staged artifacts the +/// crash leaves beside the Store. +#[derive(Clone, Copy, Debug)] +struct Case { + failpoint: CompactFailpoint, + phase: &'static str, + installed: bool, + target: bool, + visited: bool, +} + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-crash"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: [u8; 16], failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: op, + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact") +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-op") +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-visited") +} + +fn catalog_id() -> CatalogId { + CatalogId::derive(BINDING).expect("catalog identity") +} + +fn intents(handles: &SqliteHandles) -> Vec { + handles + .history + .gc_intents(GcScopeKind::Store, catalog_id().as_slice()) + .unwrap() +} + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + save.accept(object.clone()).unwrap(); + save.finish().unwrap(); + object.id() +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: Vec::new(), + }) + .unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + let root_id = object.id(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + for object in [directory, inode, object] { + save.accept(object).unwrap(); + } + save.finish().unwrap(); + root_id +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +#[allow(dead_code)] +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +#[allow(dead_code)] +fn commit(handles: &SqliteHandles, branch: BranchId, candidate_root: ObjectId) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +#[allow(dead_code)] +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn object_bytes(handles: &SqliteHandles, id: ObjectId) -> Vec { + let storage = Storage::new(handles.storage.clone()).unwrap(); + storage + .reader() + .unwrap() + .read_objects(&[id]) + .unwrap() + .into_iter() + .next() + .expect("one object") +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +fn assert_no_sidecars(store: &Path, case: Case) { + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(store, suffix).exists(), + "sidecar {suffix} beside {} at {case:?}", + store.display() + ); + } +} + +fn exercise(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + let op = [41u8; 16]; + let error = handles + .compact(&request_at(op, case.failpoint)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + + // `store.status` (T10.1) projects the recorded operation, its phase and + // its uncertain outcome; the engine reads them from the op record. + let status = compact_op_status(&path).expect("the op record remains"); + assert_eq!(status.op_id, op); + assert_eq!(status.generation, i64::try_from(before).unwrap()); + assert_eq!(status.phase, case.phase, "phase at {case:?}"); + assert!( + status.uncertain, + "an unfinished run is uncertain at {case:?}" + ); + + // The exact artifact set the point leaves beside the live Store. + assert_eq!( + compact_path(&path).exists(), + case.target, + "staged Store at {case:?}" + ); + assert_eq!( + visited_path(&path).exists(), + case.visited, + "visited scratch at {case:?}" + ); + assert!(op_path(&path).exists(), "op record at {case:?}"); + assert_no_sidecars(&path, case); + assert_no_sidecars(&compact_path(&path), case); + + let reopened = reopen(&path); + + // The admission intent is released before the seal (T-G5), so only a + // crash before that release still leaves it; a rebuilt Store was copied + // without it (`build` clears `gc_intent`). + let listed = intents(&reopened); + if case.failpoint == CompactFailpoint::BeforeSeal { + assert_eq!(listed.len(), 1, "admission intent at {case:?}"); + assert_eq!(listed[0].op_id, op); + assert_eq!(listed[0].kind, GcIntentKind::Compaction); + assert_eq!(listed[0].scope_kind, GcScopeKind::Store); + } else { + assert!(listed.is_empty(), "the intent was released at {case:?}"); + } + + // Wholly old or wholly new: never a mixed Store. + let expected_generation = if case.installed { before + 1 } else { before }; + assert_eq!( + generation(&reopened), + expected_generation, + "generation at {case:?}" + ); + assert!(located(&reopened, live), "live root at {case:?}"); + assert_eq!( + object_bytes(&reopened, live), + live_bytes, + "live bytes readable at {case:?}" + ); + assert_eq!( + located(&reopened, dead), + !case.installed, + "the dead pack at {case:?}" + ); + assert!( + reopened.history.layer_stack(stack.id).unwrap().is_some(), + "retained history readable at {case:?}" + ); + + // A crash during the copy can tear the staged file. The next run must + // remove the recorded set instead of trusting it. + if case.failpoint == CompactFailpoint::DuringCopy { + let target = compact_path(&path); + std::fs::OpenOptions::new() + .write(true) + .open(&target) + .unwrap() + .set_len(17) + .unwrap(); + std::fs::write(with_suffix(&target, "-journal"), b"torn copy journal").unwrap(); + } + // A crash partway through settle can leave a subset of the recorded set. + // Put one back to prove the next run removes the whole set. + if case.failpoint == CompactFailpoint::DuringSettle { + std::fs::write( + with_suffix(&compact_path(&path), "-journal"), + b"partial settle", + ) + .unwrap(); + std::fs::write(visited_path(&path), b"partial settle").unwrap(); + } + + // The next run judges the leftover and reaches a clean state: a rebuild + // from `G` before install, a settle at `G + 1` after it. + let receipt = reopened.compact(&request(42)).unwrap(); + if case.installed { + assert_eq!(receipt.path, CompactPath::None, "settle at {case:?}"); + assert_eq!(receipt.generation_before, before + 1); + assert_eq!(receipt.generation_after, before + 1); + } else { + assert_eq!(receipt.path, CompactPath::Rebuild, "rebuild at {case:?}"); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + } + assert!( + receipt.residue.artifacts.is_empty(), + "residue at {case:?}: {:?}", + receipt.residue + ); + + assert!(compact_op_status(&path).is_none(), "op settled at {case:?}"); + assert!(!compact_path(&path).exists(), "target removed at {case:?}"); + assert!(!visited_path(&path).exists(), "visited removed at {case:?}"); + assert!(!op_path(&path).exists(), "op removed at {case:?}"); + assert_no_sidecars(&path, case); + + let final_handles = reopen(&path); + assert_eq!(generation(&final_handles), before + 1, "final generation"); + assert!(located(&final_handles, live), "final live root"); + assert_eq!( + object_bytes(&final_handles, live), + live_bytes, + "final bytes" + ); + assert!(!located(&final_handles, dead), "the dead pack is collected"); + assert!(intents(&final_handles).is_empty(), "the intent is released"); + // Closure and the full doctor both pass on the settled Store. + final_handles.history.mark().unwrap(); + clean(&final_handles); +} + +macro_rules! crash_at { + ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal) => { + #[test] + fn $name() { + exercise(Case { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + }); + } + }; +} + +crash_at!( + crash_before_seal, + CompactFailpoint::BeforeSeal, + "mark", + false, + false, + false +); +crash_at!( + crash_before_seal_read, + CompactFailpoint::BeforeSealRead, + "mark", + false, + false, + false +); +crash_at!( + crash_after_seal, + CompactFailpoint::AfterSeal, + "mark", + false, + false, + false +); +crash_at!( + crash_before_copy, + CompactFailpoint::BeforeCopy, + "build", + false, + false, + false +); +crash_at!( + crash_during_copy, + CompactFailpoint::DuringCopy, + "build", + false, + true, + true +); +crash_at!( + crash_after_build, + CompactFailpoint::AfterBuild, + "build", + false, + true, + true +); +crash_at!( + crash_after_wal_switch, + CompactFailpoint::AfterWalSwitch, + "build", + false, + true, + true +); +crash_at!( + crash_during_verify, + CompactFailpoint::DuringVerify, + "verify", + false, + true, + true +); +crash_at!( + crash_after_verify, + CompactFailpoint::AfterVerify, + "verify", + false, + true, + true +); +crash_at!( + crash_before_install, + CompactFailpoint::BeforeInstall, + "verify", + false, + true, + true +); +crash_at!( + crash_during_rename, + CompactFailpoint::DuringRename, + "install", + true, + false, + true +); +crash_at!( + crash_after_install, + CompactFailpoint::AfterInstall, + "install", + true, + false, + true +); +crash_at!( + crash_during_settle, + CompactFailpoint::DuringSettle, + "install", + true, + false, + false +); diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs new file mode 100644 index 0000000000..3107ef5ccc --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs @@ -0,0 +1,1597 @@ +//! #41 T12.1: SQLite `store.compact` fixtures and the golden receipt. +//! +//! One transcript pins the §11 report for all three outcomes: an empty Store +//! (`none`/`empty-store`), a Store with no wholly dead pack (`none`/`no-waste`) +//! and a Store with retained history plus known dead objects, packs and delta +//! bases (`rebuild`). The fixture is built only through the public APIs; the +//! golden JSON is compared byte for byte, so a change to any reported value is +//! deliberate. The same file pins the §6.4 live-heads digest vector (fixed ids, +//! a retired Branch and stack, an absent head). +//! +//! Covered matrix rows (SQLite half where the row is cross-backend): +//! AC-04 (deterministic arm), AC-05, AC-17, AC-18, AC-25 (Branch/stack markers), +//! AC-38 (successful settle), AC-40 (SQLite half), AC-47 (`none`/success), +//! AC-48 (empty Store and rebuild arms), AC-52, AC-56 (SQLite half), AC-58. +//! +//! Update the golden deliberately with `UPDATE_GOLDEN=1 cargo test -p +//! layerfs-persistence --test sqlite_compact_fixtures`; the second run checks +//! the new file. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{ + encode_whole_file_payload, + file::mapping::{self, ExtentNode, ExtentSlice, FileState}, + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + root::profile_id, + scope_for_seed, FilesystemRoot, PathName, + }, + inode_leaf::{ + encode_inode_value, InodeKind, InodeLeaf, InodeLeafRow, InodeMetadata, InodeValue, + LEAF_ROW_BYTES, + }, + FinalizedObject, ObjectId, ObjectRole, +}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, HistoryCatalog, LayerId, + LayerRecord, LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + diagnostic::{Observation, Options, Outcome}, + CompactPath, CompactReceipt, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_storage::{ + encoding::{encode_full, encode_prefix, raw_payload, CompressionWorkspace}, + location::{ObjectLocation, PackDomain}, + pack::{assemble, build_group, layout, PackLane}, + port::{PackPersistence, Publication, PublishedPack, Reserve}, + save::SaveProfile, + Storage, StorageCapacities, +}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use support::doctor::Fixture; +use support::Temp; + +const BINDING: &[u8] = b"doctor-adversarial"; +const CURSOR: [u8; 32] = [71; 32]; +const SCOPE_SEED: [u8; 32] = [11; 32]; +const GOLDEN: &str = include_str!("fixtures/sqlite_compact_golden.json"); + +// A deterministic pseudo-random payload: the dead packs must not compress away +// to nothing, and the same bytes must produce the same receipt on every run. +fn pseudo_bytes(len: usize, seed: u8) -> Vec { + let mut state = u64::from(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.to_le_bytes()); + } + out.truncate(len); + out +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + CURSOR, + ) + .unwrap() +} + +fn store_path(temp: &Temp) -> PathBuf { + temp.join("store.sqlite") +} + +fn sidecar(store: &Path, suffix: &str) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +// ---- fixture objects ------------------------------------------------------- + +fn whole_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + f.put( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .id() +} + +/// One chunked file's content root (a FileState): the representation a file +/// at or above the small-file threshold takes, and the only kind whose +/// file_digest row the Store records. +fn chunked_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + let mut extents = Vec::new(); + for part in bytes.chunks(32_768) { + let chunk = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(part).unwrap(), + ) + .id(); + extents.push(ExtentSlice::new(chunk, 0, part.len() as u32).unwrap()); + } + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: bytes.len() as u64, + extents, + }, + true, + ) + .unwrap(), + ) + .id(); + f.put( + ObjectRole::FileState, + mapping::encode_file_state(FileState { + logical_len: bytes.len() as u64, + extent_count: (bytes.len() as u64).div_ceil(32_768), + tree_level: 0, + profile_id: mapping::profile_id(), + mapping_root: leaf, + }) + .unwrap(), + ) + .id() +} + +/// One valid filesystem tree: a root directory whose only entry is a regular +/// file whose content root is `content`. The inode leaf is pooled, so the +/// fixture also owns `inode_value_block` rows that every rebuild keeps. +fn tree(f: &Fixture, content: ObjectId, seed: u8) -> ObjectId { + let directory = f + .put( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: vec![(PathName::new(&format!("file-{seed}")).unwrap(), 2)], + }) + .unwrap(), + ) + .id(); + let mut values = Vec::new(); + let mut rows = Vec::new(); + for (serial, kind, count, root, mode) in [ + (1u64, InodeKind::Directory, 0u64, directory, 0o755), + (2u64, InodeKind::RegularFile, 1u64, content, 0o644), + ] { + let value = encode_inode_value(InodeValue { + kind, + namespace_ref_count: count, + content_root: root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode, + mtime_seconds: 0, + mtime_nanoseconds: 0, + }), + }); + values.push(value); + rows.push(InodeLeafRow { serial, value }); + } + let inode = f.inode( + InodeLeaf { + subtree_bytes: LEAF_ROW_BYTES as u64 * rows.len() as u64, + rows, + } + .encode() + .unwrap(), + values, + ); + f.put( + ObjectRole::FilesystemRoot, + FilesystemRoot::new(profile_id(), scope_for_seed(SCOPE_SEED), 1, inode) + .unwrap() + .encode() + .unwrap(), + ) + .id() +} + +/// Publishes one pack holding each object as its own group, so a retained pack +/// can hold one reached and one unreached record. +fn publish_groups(f: &Fixture, objects: &[FinalizedObject]) -> (i64, Vec) { + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let mut workspace = CompressionWorkspace::new().unwrap(); + let mut profile = SaveProfile::default(); + let groups: Vec<_> = objects + .iter() + .map(|object| { + let record = encode_full( + object.canonical(), + object.role(), + &capacities, + &mut workspace, + &mut profile, + ) + .unwrap(); + build_group(PackLane::WholeFile, &[record.record], None).unwrap() + }) + .collect(); + let body = assemble(PackLane::WholeFile, &groups).unwrap(); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .unwrap(); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body.clone()), + )], + objects: objects + .iter() + .enumerate() + .map(|(group, object)| ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: group, + record_number: 0, + }) + .collect(), + ..Publication::default() + }) + .unwrap(); + (reserved.first_pack_id, body) +} + +/// Publishes `target` as a PREFIX record over `base`, asserting the encoder +/// kept the delta (a full record would hide the base from the mark). +fn prefix_record(f: &Fixture, target: &FinalizedObject, base: &FinalizedObject, base_id: ObjectId) { + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let record = encode_prefix( + target.canonical(), + target.role(), + base_id, + raw_payload(base.canonical(), base.role()).unwrap(), + &capacities, + &mut CompressionWorkspace::new().unwrap(), + &mut SaveProfile::default(), + ) + .unwrap(); + assert_eq!( + record.base, + Some(base_id), + "the fixture needs a PREFIX record, not a full one" + ); + f.record(target, record.lane, record.record, Vec::new()); +} + +// ---- fixture history ------------------------------------------------------ + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: scope_for_seed(SCOPE_SEED).object(), + profile: profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit( + handles: &SqliteHandles, + branch: BranchId, + candidate_root: ObjectId, + logical: u64, +) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + let meta = CommitMeta { + statistics: CommitStatistics { + logical, + ..CommitStatistics::ZERO + }, + ..CommitMeta::new(CommitVia::Files) + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta, + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +// ---- raw Store facts ------------------------------------------------------ + +#[derive(Clone, Copy, Debug, Default)] +struct Totals { + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, +} + +fn totals(path: &Path) -> Totals { + let connection = rusqlite::Connection::open(path).unwrap(); + let (logical, canonical, stored, metadata): (i64, i64, i64, i64) = connection + .query_row( + "SELECT logical,canonical,stored,metadata FROM store_totals WHERE id=1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + Totals { + logical: logical as u64, + canonical: canonical as u64, + stored: stored as u64, + metadata: metadata as u64, + } +} + +#[derive(Debug)] +struct PackFacts { + total: u64, + live_bytes: u64, + dead_bytes: u64, + live: Vec, + dead: Vec, +} + +fn pack_facts(path: &Path, live_packs: &[i64]) -> PackFacts { + let connection = rusqlite::Connection::open(path).unwrap(); + let mut statement = connection + .prepare("SELECT pack_id, length FROM pack ORDER BY pack_id") + .unwrap(); + let rows = statement + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) + .unwrap(); + let mut facts = PackFacts { + total: 0, + live_bytes: 0, + dead_bytes: 0, + live: Vec::new(), + dead: Vec::new(), + }; + for row in rows { + let (pack, length) = row.unwrap(); + facts.total += length as u64; + if live_packs.contains(&pack) { + facts.live.push(pack); + facts.live_bytes += length as u64; + } else { + facts.dead.push(pack); + } + } + facts.dead_bytes = facts.total - facts.live_bytes; + facts +} + +fn scalar(path: &Path, sql: &str, param: impl rusqlite::ToSql) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .query_row(sql, [param], |row| row.get(0)) + .unwrap() +} + +fn scalar0(path: &Path, sql: &str) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection.query_row(sql, [], |row| row.get(0)).unwrap() +} + +fn pack_of(path: &Path, id: ObjectId) -> Option { + let connection = rusqlite::Connection::open(path).unwrap(); + match connection.query_row( + "SELECT pack_id FROM object_location WHERE object_id=?1", + [id.as_bytes().as_slice()], + |row| row.get::<_, i64>(0), + ) { + Ok(pack) => Some(pack), + Err(rusqlite::Error::QueryReturnedNoRows) => None, + Err(error) => panic!("object location: {error}"), + } +} + +fn object_count(path: &Path, id: ObjectId) -> i64 { + scalar( + path, + "SELECT count(*) FROM object_location WHERE object_id=?1", + id.as_bytes().as_slice(), + ) +} + +fn pack_length(path: &Path, pack: i64) -> u64 { + scalar(path, "SELECT length FROM pack WHERE pack_id=?1", pack) as u64 +} + +fn pack_row(path: &Path, pack: i64) -> i64 { + scalar(path, "SELECT count(*) FROM pack WHERE pack_id=?1", pack) +} + +fn branch_row(path: &Path, id: BranchId) -> i64 { + scalar( + path, + "SELECT count(*) FROM branch WHERE branch_id=?1", + id.as_slice(), + ) +} + +fn stack_row(path: &Path, id: LayerStackId) -> i64 { + scalar( + path, + "SELECT count(*) FROM layer_stack WHERE layer_stack_id=?1", + id.as_slice(), + ) +} + +fn retired_branches(path: &Path) -> Vec { + let connection = rusqlite::Connection::open(path).unwrap(); + let mut statement = connection + .prepare("SELECT branch_id FROM branch_retired ORDER BY branch_id") + .unwrap(); + let rows = statement + .query_map([], |row| row.get::<_, Vec>(0)) + .unwrap(); + rows.map(|row| BranchId::from_slice(&row.unwrap()).unwrap()) + .collect() +} + +fn retired_stacks(path: &Path) -> i64 { + scalar0(path, "SELECT count(*) FROM layer_stack_retired") +} + +fn digest_rows(path: &Path, digest: &[u8; 32]) -> i64 { + scalar( + path, + "SELECT count(*) FROM file_digest WHERE digest=?1", + digest.as_slice(), + ) +} + +fn inode_block_bytes(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(length(body)),0) FROM inode_value_block", + ) as u64 +} + +fn sum_pack_lengths(path: &Path) -> u64 { + scalar0(path, "SELECT COALESCE(SUM(length),0) FROM pack") as u64 +} + +fn sum_canonical(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(canonical_length),0) FROM object_location", + ) as u64 +} + +fn sum_metadata_packs(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0", + ) as u64 +} + +fn sum_history_logical(path: &Path) -> u64 { + (scalar0(path, "SELECT COALESCE(SUM(logical),0) FROM [commit]") + + scalar0(path, "SELECT COALESCE(SUM(logical),0) FROM layer")) as u64 +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn audit(handles: &SqliteHandles) -> Vec { + let mut out = Vec::new(); + handles + .diagnose( + Options { + quick: false, + ..Options::default() + }, + &mut |observation| out.push(observation), + ) + .unwrap(); + out +} + +fn assert_doctor_clean(handles: &SqliteHandles) { + let out = audit(handles); + assert!( + !out.iter() + .any(|observation| matches!(observation.outcome, Outcome::Fail | Outcome::Unknown)), + "the compacted Store must pass the full doctor: {out:?}" + ); +} + +fn assert_no_artifacts(store: &Path) { + for suffix in [ + ".compact", + ".compact-journal", + ".compact-wal", + ".compact-shm", + ".compact-visited", + ".compact-visited-journal", + ".compact-vacuum", + ".compact-op", + ".compact-op.tmp", + ] { + assert!( + !sidecar(store, suffix).exists(), + "compaction artifact {suffix} remains" + ); + } + let name = store.file_name().unwrap().to_string_lossy().to_string(); + for entry in std::fs::read_dir(store.parent().unwrap()).unwrap() { + let entry = entry.unwrap(); + let file = entry.file_name().to_string_lossy().to_string(); + if file == name { + continue; + } + let rest = file.strip_prefix(&name).unwrap_or(&file); + assert!( + matches!(rest, "-wal" | "-shm" | "-journal"), + "unexpected file beside the Store: {file}" + ); + } +} + +// ---- JSON ----------------------------------------------------------------- + +enum Json { + Arr(Vec), + Bool(bool), + Null, + Num(u64), + Obj(Vec<(&'static str, Json)>), + Str(String), +} + +impl Json { + fn render(&self) -> String { + let mut out = String::new(); + self.write(&mut out, 0); + out.push('\n'); + out + } + + fn write(&self, out: &mut String, depth: usize) { + match self { + Self::Bool(value) => out.push_str(if *value { "true" } else { "false" }), + Self::Null => out.push_str("null"), + Self::Num(value) => out.push_str(&value.to_string()), + Self::Str(value) => { + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + character if character.is_control() => { + out.push_str(&format!("\\u{:04x}", character as u32)); + } + character => out.push(character), + } + } + out.push('"'); + } + Self::Arr(items) => { + if items.is_empty() { + out.push_str("[]"); + return; + } + out.push_str("[\n"); + for (index, item) in items.iter().enumerate() { + pad(out, depth + 1); + item.write(out, depth + 1); + if index + 1 < items.len() { + out.push(','); + } + out.push('\n'); + } + pad(out, depth); + out.push(']'); + } + Self::Obj(fields) => { + if fields.is_empty() { + out.push_str("{}"); + return; + } + let mut sorted: Vec<_> = fields.iter().collect(); + sorted.sort_by(|left, right| left.0.cmp(right.0)); + out.push_str("{\n"); + for (index, (key, value)) in sorted.iter().enumerate() { + pad(out, depth + 1); + out.push('"'); + out.push_str(key); + out.push_str("\": "); + value.write(out, depth + 1); + if index + 1 < sorted.len() { + out.push(','); + } + out.push('\n'); + } + pad(out, depth); + out.push('}'); + } + } + } +} + +fn pad(out: &mut String, depth: usize) { + for _ in 0..depth { + out.push_str(" "); + } +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(DIGITS[(byte >> 4) as usize] as char); + out.push(DIGITS[(byte & 15) as usize] as char); + } + out +} + +fn strings(items: impl IntoIterator) -> Json { + Json::Arr(items.into_iter().map(Json::Str).collect()) +} + +fn object_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_bytes()))) +} + +fn byte_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(&id))) +} + +fn branch_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn commit_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn layer_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn stack_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn numbers(items: impl IntoIterator) -> Json { + Json::Arr( + items + .into_iter() + .map(|item| Json::Num(item as u64)) + .collect(), + ) +} + +fn totals_json(total: Totals) -> Json { + Json::Obj(vec![ + ("canonical", Json::Num(total.canonical)), + ("logical", Json::Num(total.logical)), + ("metadata", Json::Num(total.metadata)), + ("stored", Json::Num(total.stored)), + ]) +} + +fn receipt_json(receipt: &CompactReceipt) -> Json { + Json::Obj(vec![ + ("backendReusable", Json::Num(receipt.backend_reusable)), + ("deadBytesBound", Json::Num(receipt.dead_bytes_bound)), + ("deadBytesRefined", Json::Num(receipt.dead_bytes_refined)), + ("estimate", Json::Bool(receipt.estimate)), + ("generationAfter", Json::Num(receipt.generation_after)), + ("generationBefore", Json::Num(receipt.generation_before)), + ( + "keptWaste", + Json::Obj(vec![ + ( + "deadInRetainedPacksBytes", + Json::Num(receipt.kept_waste.dead_in_retained_packs_bytes), + ), + ( + "deltaBases", + Json::Obj(vec![ + ("bytes", Json::Num(receipt.kept_waste.delta_bases_bytes)), + ( + "references", + byte_ids(receipt.kept_waste.delta_base_references.iter().copied()), + ), + ]), + ), + ( + "inodeBlocksBytes", + Json::Num(receipt.kept_waste.inode_blocks_bytes), + ), + ]), + ), + ("logicalDiscarded", Json::Num(receipt.logical_discarded)), + ("path", Json::Str(receipt.path.as_str().to_owned())), + ("peakTempBytes", Json::Num(receipt.peak_temp_bytes)), + ("physicalShrink", Json::Num(receipt.physical_shrink)), + ( + "residue", + Json::Obj(vec![ + ( + "artifacts", + strings(receipt.residue.artifacts.iter().cloned()), + ), + ("bytes", Json::Num(receipt.residue.bytes)), + ( + "reason", + match &receipt.residue.reason { + Some(reason) => Json::Str(reason.clone()), + None => Json::Null, + }, + ), + ]), + ), + ( + "zeroReasons", + Json::Arr( + receipt + .zero_reasons + .iter() + .map(|zero| { + Json::Obj(vec![ + ("field", Json::Str(zero.field.to_owned())), + ("reason", Json::Str(zero.reason.to_owned())), + ]) + }) + .collect(), + ), + ), + ]) +} + +fn mark_json(digest: [u8; 32], objects: usize, packs: &[i64]) -> Json { + Json::Obj(vec![ + ("liveHeadsDigest", Json::Str(hex(&digest))), + ("objectCount", Json::Num(objects as u64)), + ("packs", numbers(packs.iter().copied())), + ]) +} + +fn check_golden(actual: &Json) { + let rendered = actual.render(); + if std::env::var_os("UPDATE_GOLDEN").is_some() { + let path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/sqlite_compact_golden.json"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, &rendered).unwrap(); + eprintln!("updated golden transcript {}", path.display()); + return; + } + assert_eq!( + rendered, GOLDEN, + "the SQLite compaction golden transcript changed. Review every value; if \ + the change is deliberate, rerun with UPDATE_GOLDEN=1." + ); +} + +/// The §6.4 live-heads vector, written out independently of the mark: the +/// fixed prefix, the non-retired stacks in identity order as `(17, 33)`, then +/// the marker-free Branches in identity order as `(17, 33-or-33-zero, 33)`. +fn expected_heads_digest( + stacks: &[(LayerStackId, LayerId)], + branches: &[(BranchId, Option, LayerId)], +) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let mut stacks = stacks.to_vec(); + stacks.sort_by(|left, right| left.0.as_slice().cmp(right.0.as_slice())); + for (stack, head) in stacks { + assert_eq!(stack.as_slice().len(), 17, "stack id width"); + assert_eq!(head.as_slice().len(), 33, "layer id width"); + hasher.update(stack.as_slice()); + hasher.update(head.as_slice()); + } + let mut branches = branches.to_vec(); + branches.sort_by(|left, right| left.0.as_slice().cmp(right.0.as_slice())); + for (branch, commit, base) in branches { + assert_eq!(branch.as_slice().len(), 17, "branch id width"); + assert_eq!(base.as_slice().len(), 33, "base layer id width"); + hasher.update(branch.as_slice()); + match commit { + Some(commit) => { + assert_eq!(commit.as_slice().len(), 33, "commit id width"); + hasher.update(commit.as_slice()); + } + None => { + hasher.update(&[0u8; 33]); + } + } + hasher.update(base.as_slice()); + } + *hasher.finalize().as_bytes() +} + +// ---- cases ---------------------------------------------------------------- + +fn empty_none_case() -> Json { + let f = Fixture::new(); + let Fixture { temp, handles, .. } = f; + let path = store_path(&temp); + let before = totals(&path); + assert_eq!( + ( + before.logical, + before.canonical, + before.stored, + before.metadata + ), + (0, 0, 0, 0) + ); + let generation_before = generation(&handles); + let receipt = handles.compact(&request(0x70)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, 0); + assert_eq!(receipt.logical_discarded, 0); + assert_eq!(receipt.physical_shrink, 0); + assert_eq!(receipt.peak_temp_bytes, 0); + assert!(receipt.estimate); + assert!(receipt.kept_waste.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesRefined" && zero.reason == "empty-store")); + assert_no_artifacts(&path); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before); + assert_doctor_clean(&reopened); + let digest = reopened.history.mark().unwrap().live_heads_digest(); + let after = totals(&path); + Json::Obj(vec![ + ("mark", mark_json(digest, 0, &[])), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +fn no_dead_none_case() -> Json { + let f = Fixture::new(); + let genesis_content = whole_file(&f, b"the genesis tree content"); + let genesis = tree(&f, genesis_content, 7); + let stack = initialize(&f.handles, 1, genesis); + let branch = support::branch_id(1); + fork(&f.handles, stack.id, branch, 1); + let commit_content = whole_file(&f, b"a retained commit tree content"); + let commit_root = tree(&f, commit_content, 8); + let commit_id = commit(&f.handles, branch, commit_root, 200); + let layer = publish(&f.handles, stack.id, branch); + + let Fixture { temp, handles, .. } = f; + let path = store_path(&temp); + let before = totals(&path); + let generation_before = generation(&handles); + let mark = handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + assert!(mark.objects().waste().inode_blocks_bytes() > 0); + let facts = pack_facts(&path, mark.objects().packs()); + assert_eq!(facts.dead_bytes, 0, "no wholly dead pack exists"); + let digest = mark.live_heads_digest(); + let live_packs = mark.objects().packs().to_vec(); + let receipt = handles.compact(&request(0x71)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert_eq!( + receipt.logical_discarded, 0, + "the none path collects nothing" + ); + assert_eq!(receipt.physical_shrink, 0); + assert_eq!(receipt.peak_temp_bytes, 0); + assert!(receipt.estimate); + assert!(receipt.residue.artifacts.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "no-waste")); + assert_no_artifacts(&path); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before); + assert_doctor_clean(&reopened); + assert!(reopened.history.commit(commit_id).unwrap().is_some()); + assert!(reopened.history.layer(layer.id).unwrap().is_some()); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + digest, + "the none path leaves the heads unchanged" + ); + let after = totals(&path); + Json::Obj(vec![ + ("commit", commit_ids([commit_id])), + ("layer", layer_ids([layer.id])), + ("mark", mark_json(digest, mark.objects().len(), &live_packs)), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +struct FixtureIds { + stacks_live: Vec, + stacks_collected: Vec, + branches_live: Vec, + branches_retained_retired: Vec, + branches_collected: Vec, + commits_retained: Vec, + commits_collected: Vec, + layers_retained: Vec, + layers_collected: Vec, + objects_live: Vec, + objects_dead: Vec, + delta_bases_sorted: Vec, + delta_chain: Vec<(ObjectId, i64)>, + delta_target: ObjectId, + live_group: ObjectId, + dead_group: ObjectId, + shared_pack: i64, + shared_pack_length: u64, + shared_dead_group: u64, + digest_collected: [u8; 32], + digest_retained: [u8; 32], +} + +fn rebuild_fixture() -> (Temp, SqliteHandles, FixtureIds) { + let f = Fixture::new(); + + // The dead packs: unreferenced whole files, each in its own pack. + let mut objects_dead = Vec::new(); + for index in 0..12u8 { + let payload = pseudo_bytes(4096, 0xA0 + index); + objects_dead.push(whole_file(&f, &payload)); + } + // AC-18 needs chunked file roots: the digest floor is the small-file + // threshold, so a WholeFile root is below it and records no row. + let retained_chunked_payload = pseudo_bytes(131_072, 0x77); + let retained_chunked = chunked_file(&f, &retained_chunked_payload); + let collected_chunked_payload = pseudo_bytes(131_072, 0x88); + let collected_chunked = chunked_file(&f, &collected_chunked_payload); + let digest_collected = *blake3::hash(&collected_chunked_payload).as_bytes(); + let digest_retained = *blake3::hash(&retained_chunked_payload).as_bytes(); + + // A depth-2 delta chain: base2 <- base1 <- target. Only the target is a + // retained root, so both bases are kept only as delta bases. + let base2_payload = pseudo_bytes(512, 0x11); + let base1_payload = [base2_payload.as_slice(), b" v2"].concat(); + let target_payload = [base1_payload.as_slice(), b" v3"].concat(); + let base2 = whole_file(&f, &base2_payload); + let base2_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base2_payload).unwrap(), + ) + .unwrap(); + let base1_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base1_payload).unwrap(), + ) + .unwrap(); + prefix_record(&f, &base1_object, &base2_object, base2); + let base1 = base1_object.id(); + let target_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&target_payload).unwrap(), + ) + .unwrap(); + prefix_record(&f, &target_object, &base1_object, base1); + let delta_target = target_object.id(); + + // One retained pack holding one reached and one unreached record. + let live_payload = pseudo_bytes(256, 0x55); + let dead_payload = pseudo_bytes(256, 0x66); + let live_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&live_payload).unwrap(), + ) + .unwrap(); + let dead_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&dead_payload).unwrap(), + ) + .unwrap(); + let (shared_pack, shared_body) = publish_groups(&f, &[live_group.clone(), dead_group.clone()]); + // The shared pack's live object is the genesis tree's file content. + let shared_header = layout::parse_header(&shared_body).unwrap(); + let dead_view = layout::group_view(&shared_body, shared_header, 1).unwrap(); + let shared_dead_group = (dead_view.end - dead_view.start) as u64; + let shared_pack_length = pack_length(&store_path(&f.temp), shared_pack); + + // Content objects of the trees that are expected to be collected. + let collected_content_a4 = collected_chunked; + let collected_content_b0 = whole_file(&f, b"deleted project genesis content"); + let collected_content_b1 = whole_file(&f, b"deleted project commit content"); + let content_c3 = whole_file(&f, b"live branch commit content"); + + let root_a0 = tree(&f, live_group.id(), 0); + let root_a1 = tree(&f, delta_target, 1); + let root_a2 = tree(&f, retained_chunked, 2); + let root_a3 = tree(&f, content_c3, 3); + let root_a4 = tree(&f, collected_content_a4, 4); + let root_b0 = tree(&f, collected_content_b0, 5); + let root_b1 = tree(&f, collected_content_b1, 6); + + let stack_a = initialize(&f.handles, 1, root_a0); + let branch_published = support::branch_id(1); + fork(&f.handles, stack_a.id, branch_published, 1); + let commit_1 = commit(&f.handles, branch_published, root_a1, 120); + let layer_1 = publish(&f.handles, stack_a.id, branch_published); + let commit_2 = commit(&f.handles, branch_published, root_a2, 240); + let layer_2 = publish(&f.handles, stack_a.id, branch_published); + + let branch_live = support::branch_id(2); + fork(&f.handles, stack_a.id, branch_live, 2); + let commit_3 = commit(&f.handles, branch_live, root_a3, 360); + + let branch_retired = support::branch_id(3); + fork(&f.handles, stack_a.id, branch_retired, 3); + let commit_4 = commit(&f.handles, branch_retired, root_a4, 70); + + let stack_b = initialize(&f.handles, 2, root_b0); + let branch_b = support::branch_id(4); + fork(&f.handles, stack_b.id, branch_b, 4); + let commit_5 = commit(&f.handles, branch_b, root_b1, 40); + let layer_b = publish(&f.handles, stack_b.id, branch_b); + + // A retired Branch a retained Layer still names keeps its row and marker. + f.handles + .history + .mark_branch_removed(branch_published, 1_000) + .unwrap(); + // A retired Branch nothing retained reaches is collected whole. + f.handles + .history + .mark_branch_removed(branch_retired, 1_500) + .unwrap(); + // A deleted project goes whole, marker included. + f.handles + .history + .mark_stack_deleted(stack_b.id, 2_000) + .unwrap(); + + // AC-18: one digest root no retained root reaches, one it does. + f.handles + .history + .record_file_digests(&[ + FileDigest { + root: *collected_chunked.as_bytes(), + length: collected_chunked_payload.len() as u64, + digest: digest_collected, + }, + FileDigest { + root: *retained_chunked.as_bytes(), + length: retained_chunked_payload.len() as u64, + digest: digest_retained, + }, + ]) + .unwrap(); + + let path = store_path(&f.temp); + let delta_chain: Vec<(ObjectId, i64)> = [base2, base1, delta_target] + .iter() + .map(|id| (*id, pack_of(&path, *id).expect("a delta-chain pack"))) + .collect(); + let mut delta_bases_sorted = vec![base1, base2]; + delta_bases_sorted.sort(); + let mut objects_dead = objects_dead; + objects_dead.push(dead_group.id()); + objects_dead.extend([ + root_a4, + root_b0, + root_b1, + collected_content_a4, + collected_content_b0, + collected_content_b1, + ]); + let objects_live = vec![ + root_a0, + root_a1, + root_a2, + root_a3, + live_group.id(), + retained_chunked, + base2, + base1, + delta_target, + content_c3, + ]; + let ids = FixtureIds { + stacks_live: vec![stack_a.id], + stacks_collected: vec![stack_b.id], + branches_live: vec![branch_live], + branches_retained_retired: vec![branch_published], + branches_collected: vec![branch_retired, branch_b], + commits_retained: vec![commit_1, commit_2, commit_3], + commits_collected: vec![commit_4, commit_5], + layers_retained: vec![stack_a.head_layer, layer_1.id, layer_2.id], + layers_collected: vec![stack_b.head_layer, layer_b.id], + objects_live, + objects_dead, + delta_bases_sorted, + delta_chain, + delta_target, + live_group: live_group.id(), + dead_group: dead_group.id(), + shared_pack, + shared_pack_length, + shared_dead_group, + digest_collected, + digest_retained, + }; + let Fixture { temp, handles, .. } = f; + (temp, handles, ids) +} + +fn rebuild_case() -> Json { + let (temp, handles, ids) = rebuild_fixture(); + let path = store_path(&temp); + let before = totals(&path); + let generation_before = generation(&handles); + + let mark = handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + let expected_digest = expected_heads_digest( + &[(ids.stacks_live[0], *ids.layers_retained.last().unwrap())], + &[( + ids.branches_live[0], + Some(ids.commits_retained[2]), + *ids.layers_retained.last().unwrap(), + )], + ); + assert_eq!( + mark.live_heads_digest(), + expected_digest, + "the mark hashes the §6.4 vector" + ); + let live_packs = mark.objects().packs().to_vec(); + let facts = pack_facts(&path, &live_packs); + let inode_blocks = inode_block_bytes(&path); + assert_eq!(mark.objects().waste().inode_blocks_bytes(), inode_blocks); + assert_eq!( + mark.objects().waste().dead_in_retained_packs_bytes(), + ids.shared_dead_group, + "the unreached group of the retained pack is keptWaste" + ); + assert_eq!( + mark.objects().waste().delta_base_references(), + ids.delta_bases_sorted.as_slice(), + "a depth-2 chain reports both bases, in identity order" + ); + let expected_bases: Vec<[u8; 32]> = ids + .delta_bases_sorted + .iter() + .map(|id| *id.as_bytes()) + .collect(); + assert!( + mark.objects().waste().delta_bases_bytes() > 0, + "the bases are kept only as bases" + ); + for id in &ids.objects_live { + assert!(mark.objects().contains(*id), "live object {id} is retained"); + } + for id in &ids.objects_dead { + assert!( + !mark.objects().contains(*id), + "dead object {id} is collected" + ); + } + assert_eq!(facts.dead_bytes, facts.total - facts.live_bytes); + assert!( + facts.dead_bytes * 10 >= facts.total, + "the fixture must take the rebuild path: {} dead of {}", + facts.dead_bytes, + facts.total + ); + + let receipt = handles.compact(&request(0x72)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before + 1); + assert_eq!(receipt.dead_bytes_bound, facts.dead_bytes); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + assert!(receipt.estimate); + assert_eq!( + receipt.kept_waste.dead_in_retained_packs_bytes, + ids.shared_dead_group + ); + assert_eq!(receipt.kept_waste.inode_blocks_bytes, inode_blocks); + assert_eq!(receipt.kept_waste.delta_base_references, expected_bases); + assert!(receipt.kept_waste.delta_bases_bytes > 0); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert_no_artifacts(&path); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before + 1); + let after = totals(&path); + assert_eq!( + receipt.logical_discarded, + before.logical - after.logical, + "logicalDiscarded is the history logical delta" + ); + assert_eq!(before.logical - after.logical, 160); // C4 + stack B genesis + C5 + Lb + assert_eq!(after.stored, sum_pack_lengths(&path)); + assert_eq!(after.canonical, sum_canonical(&path)); + assert_eq!(after.metadata, sum_metadata_packs(&path)); + assert_eq!(after.logical, sum_history_logical(&path)); + + // Live objects are readable, including the delta target through its chain. + let storage = Storage::new(reopened.storage.clone()).unwrap(); + let reader = storage.reader().unwrap(); + let canonicals = reader + .read_objects(&[ids.delta_target, ids.live_group]) + .unwrap(); + assert_eq!(canonicals.len(), 2); + assert!(!canonicals[0].is_empty()); + assert!(!canonicals[1].is_empty()); + for id in &ids.objects_live { + assert!(located(&reopened, *id), "live object {id} survives"); + } + for id in &ids.objects_dead { + assert!(!located(&reopened, *id), "dead object {id} is gone"); + } + for pack in &facts.dead { + assert_eq!(pack_row(&path, *pack), 0, "dead pack {pack} is removed"); + } + for pack in &facts.live { + assert_eq!(pack_row(&path, *pack), 1, "live pack {pack} stays"); + } + + // The retained pack keeps the unreached group: repacking is out of 2.0. + assert_eq!(pack_of(&path, ids.live_group), Some(ids.shared_pack)); + assert_eq!(object_count(&path, ids.live_group), 1); + assert_eq!(object_count(&path, ids.dead_group), 0); + assert_eq!(pack_length(&path, ids.shared_pack), ids.shared_pack_length); + // AC-17: a copied pack keeps its identity. + for (id, pack) in &ids.delta_chain { + assert_eq!( + pack_of(&path, *id), + Some(*pack), + "pack_id of {id} is preserved" + ); + } + + // Retained history and markers survive; collected rows take their markers. + for id in &ids.commits_retained { + assert!( + reopened.history.commit(*id).unwrap().is_some(), + "commit {id}" + ); + } + for id in &ids.commits_collected { + assert!( + reopened.history.commit(*id).unwrap().is_none(), + "commit {id}" + ); + } + for id in &ids.layers_retained { + assert!(reopened.history.layer(*id).unwrap().is_some(), "layer {id}"); + } + for id in &ids.layers_collected { + assert!(reopened.history.layer(*id).unwrap().is_none(), "layer {id}"); + } + for id in &ids.stacks_live { + assert!(reopened.history.layer_stack(*id).unwrap().is_some()); + assert_eq!(stack_row(&path, *id), 1); + } + for id in &ids.stacks_collected { + assert!(reopened.history.layer_stack(*id).unwrap().is_none()); + assert_eq!(stack_row(&path, *id), 0); + } + for id in &ids.branches_live { + assert!(reopened.history.branch(*id).unwrap().is_some()); + } + assert_eq!(retired_branches(&path), ids.branches_retained_retired); + for id in &ids.branches_collected { + assert_eq!(branch_row(&path, *id), 0, "collected branch row {id}"); + } + assert_eq!(retired_stacks(&path), 0); + + // AC-18: a digest row survives only when a retained root reaches the root. + assert_eq!(digest_rows(&path, &ids.digest_retained), 1); + assert_eq!(digest_rows(&path, &ids.digest_collected), 0); + + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + mark.live_heads_digest(), + "S9: the live-heads digest matches before and after" + ); + + let fixture = Json::Obj(vec![ + ( + "branches", + Json::Obj(vec![ + ( + "collected", + branch_ids(ids.branches_collected.iter().copied()), + ), + ("live", branch_ids(ids.branches_live.iter().copied())), + ( + "retainedRetired", + branch_ids(ids.branches_retained_retired.iter().copied()), + ), + ]), + ), + ( + "commits", + Json::Obj(vec![ + ( + "collected", + commit_ids(ids.commits_collected.iter().copied()), + ), + ("retained", commit_ids(ids.commits_retained.iter().copied())), + ]), + ), + ("deadBytesBound", Json::Num(facts.dead_bytes)), + ( + "deltaBaseReferences", + object_ids(ids.delta_bases_sorted.iter().copied()), + ), + ( + "deltaChainPackIds", + numbers(ids.delta_chain.iter().map(|(_, pack)| *pack)), + ), + ("inodeBlocksBytes", Json::Num(inode_blocks)), + ( + "layers", + Json::Obj(vec![ + ("collected", layer_ids(ids.layers_collected.iter().copied())), + ("retained", layer_ids(ids.layers_retained.iter().copied())), + ]), + ), + ( + "objects", + Json::Obj(vec![ + ("dead", object_ids(ids.objects_dead.iter().copied())), + ("live", object_ids(ids.objects_live.iter().copied())), + ]), + ), + ( + "packs", + Json::Obj(vec![ + ("dead", numbers(facts.dead.iter().copied())), + ("live", numbers(facts.live.iter().copied())), + ("shared", Json::Num(ids.shared_pack as u64)), + ("sharedDeadGroupBytes", Json::Num(ids.shared_dead_group)), + ("sharedLength", Json::Num(ids.shared_pack_length)), + ]), + ), + ( + "stacks", + Json::Obj(vec![ + ("collected", stack_ids(ids.stacks_collected.iter().copied())), + ("live", stack_ids(ids.stacks_live.iter().copied())), + ]), + ), + ]); + let after_state = Json::Obj(vec![ + ("branchRetired", branch_ids(retired_branches(&path))), + ("deadPacksGone", Json::Bool(true)), + ("fileDigestCollected", Json::Bool(false)), + ("fileDigestRetained", Json::Bool(true)), + ("layerStackRetired", Json::Num(retired_stacks(&path) as u64)), + ("liveReadable", Json::Bool(true)), + ("liveHeadsDigestUnchanged", Json::Bool(true)), + ("fullDoctor", Json::Str("clean".to_owned())), + ]); + Json::Obj(vec![ + ("after", after_state), + ("fixture", fixture), + ( + "mark", + mark_json(mark.live_heads_digest(), mark.objects().len(), &live_packs), + ), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +fn digest_vector_case() -> Json { + let f = Fixture::new(); + // Out-of-order creation: the digest orders by identity, not creation. + let root_9 = tree(&f, whole_file(&f, b"digest stack nine content"), 9); + let stack_9 = initialize(&f.handles, 9, root_9); + let root_3 = tree(&f, whole_file(&f, b"digest stack three content"), 10); + let stack_3 = initialize(&f.handles, 3, root_3); + let root_7 = tree(&f, whole_file(&f, b"digest retired stack content"), 11); + let stack_7 = initialize(&f.handles, 7, root_7); + f.handles + .history + .mark_stack_deleted(stack_7.id, 7_000) + .unwrap(); + + let branch_9 = support::branch_id(9); + fork(&f.handles, stack_9.id, branch_9, 9); + let branch_3 = support::branch_id(3); + fork(&f.handles, stack_3.id, branch_3, 3); + let branch_5 = support::branch_id(5); + fork(&f.handles, stack_9.id, branch_5, 5); + let commit_root = tree(&f, whole_file(&f, b"digest commit content"), 12); + let commit_id = commit(&f.handles, branch_3, commit_root, 90); + + // branch_5 has never committed: an absent head is hashed as 33 zero bytes. + let with_empty_head = f.handles.history.mark().unwrap().live_heads_digest(); + assert_eq!( + with_empty_head, + f.handles.history.mark().unwrap().live_heads_digest(), + "the fixed fixture pins one digest" + ); + assert_eq!( + with_empty_head, + expected_heads_digest( + &[ + (stack_3.id, stack_3.head_layer), + (stack_9.id, stack_9.head_layer) + ], + &[ + (branch_3, Some(commit_id), stack_3.head_layer), + (branch_5, None, stack_9.head_layer), + (branch_9, None, stack_9.head_layer), + ], + ), + "an absent head is 33 zero bytes" + ); + // A retired Branch is absent, not zeroed. + f.handles + .history + .mark_branch_removed(branch_5, 5_000) + .unwrap(); + let retired_branch = f.handles.history.mark().unwrap().live_heads_digest(); + assert_ne!( + with_empty_head, retired_branch, + "a retired Branch leaves the digest instead of hashing zeros" + ); + assert_eq!( + retired_branch, + expected_heads_digest( + &[ + (stack_3.id, stack_3.head_layer), + (stack_9.id, stack_9.head_layer) + ], + &[ + (branch_3, Some(commit_id), stack_3.head_layer), + (branch_9, None, stack_9.head_layer), + ], + ), + "a retired Branch row is absent, not zeroed" + ); + assert_eq!( + retired_branch, + f.handles.history.mark().unwrap().live_heads_digest() + ); + Json::Obj(vec![ + ("absentHeadDigest", Json::Str(hex(&with_empty_head))), + ("commit", commit_ids([commit_id])), + ("retiredBranchDigest", Json::Str(hex(&retired_branch))), + ("retiredStack", stack_ids([stack_7.id])), + ]) +} + +#[test] +fn the_sqlite_compaction_receipts_match_the_golden_transcript() { + let transcript = Json::Obj(vec![ + ("digestVector", digest_vector_case()), + ("emptyNone", empty_none_case()), + ("noDeadNone", no_dead_none_case()), + ("rebuild", rebuild_case()), + ]); + check_golden(&transcript); +} diff --git a/engine/crates/layerfs-store-service/src/history/admission.rs b/engine/crates/layerfs-store-service/src/history/admission.rs index 283ed668a3..84b421dca8 100644 --- a/engine/crates/layerfs-store-service/src/history/admission.rs +++ b/engine/crates/layerfs-store-service/src/history/admission.rs @@ -49,8 +49,16 @@ pub fn project_removal(stack: &LayerStackRecord) -> GcIntent { /// A `project.delete` intent when only the stack id is known. pub fn project_removal_from_id(stack: LayerStackId) -> GcIntent { + project_removal_at(stack, op_id()) +} + +/// A `project.delete` intent under an operation identity the caller already +/// registered. The control layer admits the project scope before its sidecar +/// and marker and reuses the identity there, so one row is taken and released +/// once, never two competing rows. +pub fn project_removal_at(stack: LayerStackId, op: [u8; 16]) -> GcIntent { GcIntent::operation( - op_id(), + op, GcIntentKind::Removal, GcScopeKind::Project, stack.as_slice(), diff --git a/engine/crates/layerfs-store-service/src/history/deletion.rs b/engine/crates/layerfs-store-service/src/history/deletion.rs index c4bd8d6e1a..3fb3d38710 100644 --- a/engine/crates/layerfs-store-service/src/history/deletion.rs +++ b/engine/crates/layerfs-store-service/src/history/deletion.rs @@ -10,7 +10,7 @@ use crate::history::lookup; use crate::store::Store; use layerfs_bridge::control::SdkRefusal; -use layerfs_history::records::GcDeletion; +use layerfs_history::records::{GcDeletion, GcIntent}; use layerfs_history::LayerStackId; use std::sync::Arc; @@ -51,10 +51,23 @@ pub fn delete_stack( stack: LayerStackId, deleted_ms: i64, ) -> Result { - let history = store.history(); let intent = crate::history::admission::project_removal_from_id(stack); + delete_stack_with(store, stack, deleted_ms, &intent) +} + +/// [`delete_stack`] under an admission intent the caller already registered. +/// The marker's own admission scan excludes this operation identity, so the +/// caller's row is reused and released once instead of being seen as another +/// removal. A provider without admission writes the marker as before. +pub fn delete_stack_with( + store: &Arc, + stack: LayerStackId, + deleted_ms: i64, + intent: &GcIntent, +) -> Result { + let history = store.history(); match history - .gc_delete_stack(stack, deleted_ms, &intent) + .gc_delete_stack(stack, deleted_ms, intent) .map_err(lookup::history)? { GcDeletion::Retired { deleted_ms } => Ok(Deletion::Retired { deleted_ms }), diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index a37773f2d0..c197adebc4 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -57,6 +57,7 @@ pub fn history(error: HistoryError) -> SdkRefusal { HistoryError::NameTaken(_) => "NameTaken", HistoryError::Removed(_) => "Removed", HistoryError::Conflict(_) => "Conflict", + HistoryError::Exists(_) => "Exists", HistoryError::StaleStore => "StaleStore", HistoryError::Unsupported(_) => "Unsupported", HistoryError::UnknownOutcome => "Unknown", diff --git a/engine/crates/layerfs-store-service/src/methods/branch/mod.rs b/engine/crates/layerfs-store-service/src/methods/branch/mod.rs index 63177f6be8..92fc82863e 100644 --- a/engine/crates/layerfs-store-service/src/methods/branch/mod.rs +++ b/engine/crates/layerfs-store-service/src/methods/branch/mod.rs @@ -7,6 +7,7 @@ mod get_commit; mod list_commits; mod merge; mod publish; +mod remove; pub use commits::commits; pub use diff_branches::diff_branches; pub use diff_commits::diff_commits; @@ -15,3 +16,4 @@ pub use get_commit::get_commit; pub use list_commits::list_commits; pub use merge::merge; pub use publish::publish; +pub use remove::remove; diff --git a/engine/crates/layerfs-store-service/src/methods/branch/remove.rs b/engine/crates/layerfs-store-service/src/methods/branch/remove.rs new file mode 100644 index 0000000000..1bcdf5c80b --- /dev/null +++ b/engine/crates/layerfs-store-service/src/methods/branch/remove.rs @@ -0,0 +1,10 @@ +//! `branch.remove`. +use crate::methods::args::{project, text, Answer}; +use crate::{history::removal, StoreService}; +use layerfs_bridge::control::SdkValue; + +/// `remove({ project, branch })`: hide one Branch by its Store marker. +pub fn remove(service: &StoreService, args: &SdkValue) -> Answer { + let stack = project(service, args)?; + removal::remove(&service.store, &stack, text(args, "branch")?) +} diff --git a/engine/crates/layerfs-store-service/src/methods/dispatch.rs b/engine/crates/layerfs-store-service/src/methods/dispatch.rs index 73e1cf034a..5c96f216f9 100644 --- a/engine/crates/layerfs-store-service/src/methods/dispatch.rs +++ b/engine/crates/layerfs-store-service/src/methods/dispatch.rs @@ -29,6 +29,7 @@ pub fn call(service: &StoreService, call: &SdkCall) -> Answer { "branch.get_commit" => branch::get_commit(service, args), "branch.merge" => branch::merge(service, args), "branch.publish" => branch::publish(service, args), + "branch.remove" => branch::remove(service, args), "branch.diff_commits" => branch::diff_commits(service, args), "branch.diff_branches" => branch::diff_branches(service, args), "fs.read" => fs::read(service, args), diff --git a/engine/crates/layerfs-store-service/src/methods/file/glob.rs b/engine/crates/layerfs-store-service/src/methods/file/glob.rs index bde2596c77..065a2e1bd6 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/glob.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/glob.rs @@ -17,6 +17,8 @@ pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { return Err(refusal); } } + // The page-independent `removed_count`: the hidden entries this pattern + // would return anywhere under `path`, not only direct children. let removed = seen.hidden_matches(&reader, &snap, &path, pattern)?; if args.get("page").is_some() { return paged_reads::pin( diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs index ef60b2cf4c..f388a1b1a5 100644 --- a/engine/crates/layerfs-store-service/src/read/visibility.rs +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -132,7 +132,10 @@ impl Visibility { /// The number of hidden entries directly inside `dir` whose entries exist /// at this snapshot: the `removed_count` of a listing, independent of the - /// page. A marker on a path below a direct child is not counted here. + /// page. A listing returns `dir`'s direct entries, so a marker below one + /// of them is not a hidden result and is not counted; a glob counts the + /// matched hidden entries at any depth (`hidden_matches`). Each call + /// therefore counts exactly the entries its own result would omit. pub fn hidden_children( &self, reader: &StorePorts, @@ -168,8 +171,12 @@ impl Visibility { } /// The number of hidden entries at or below `path` that the glob `pattern` - /// matches: the `removed_count` of a glob, independent of the page. Only - /// hidden subtrees are walked, so a snapshot with no marker counts nothing. + /// matches: the `removed_count` of a glob, independent of the page. It + /// counts every hidden entry the glob would otherwise return, at any depth + /// under the query root, not only the root's direct children: the pattern + /// decides which descendants are results. A listing's `hidden_children` + /// does the same for its direct entries. Only hidden subtrees are walked, + /// so a snapshot with no marker counts nothing. pub fn hidden_matches( &self, reader: &StorePorts, @@ -212,6 +219,10 @@ impl Visibility { found.insert(full.clone()); } if entry.value.kind == InodeKind::Directory { + // A hidden directory's descendants are hidden by the same + // governing frontier and are glob results whenever the pattern + // reaches them, so walk the hidden subtree and count each + // match; `found` deduplicates a marker on a descendant. walked.push(relative.to_owned()); walk::walk_within(reader, snap, &full, None, |child, _| { if rule.matches_path(child) { diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index 281c3f0de8..36925dcd95 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -123,6 +123,12 @@ pub fn commit( crate::store::CommitError::History(layerfs_history::HistoryError::Busy) => { "Busy" } + // Spec 5: the project epoch moved (a removal raced this + // write) or is gone after compaction; the Commit is + // refused with the spec's `Conflict`, nothing written. + crate::store::CommitError::History( + layerfs_history::HistoryError::Conflict(_), + ) => "Conflict", crate::store::CommitError::History( layerfs_history::HistoryError::Capacity(_), ) => "Capacity", @@ -201,6 +207,9 @@ fn change( // already whiteouts a hidden path, so its absence there is ambiguous. // The base view distinguishes a hidden path (a `Removed` refusal for // `edit` and a cp source) from a genuinely absent one (`NoSuchPath`). + // `rm` and `mv`-from read only the private overlay, where a hidden path is + // already a whiteout, so they report it as absent: `NoSuchPath`, this + // method table's wire spelling of §4.3's `NotFound` for those two calls. let base = workspace .snapshot() .map_err(crate::write::mutation::failure)?; @@ -234,6 +243,7 @@ fn change( }, Change::Mkdir { path, mode } => mutation.mkdir(path, *mode), Change::Chmod { path, mode } => mutation.chmod(path, *mode), + // `rm` treats a hidden path as absent for the same reason. Change::Rm { path } => mutation.remove(path), Change::Cp { from, to } => match mutation.copy(from, to, index as u64) { Err(refusal) if refusal.code == "NoSuchPath" => match removed(from)? { @@ -242,6 +252,9 @@ fn change( }, other => other, }, + // A hidden `from` is absent to the overlay, so `mv`-from answers + // `NoSuchPath` (see the `NotFound` mapping above); unlike + // `edit`/`cp`-from it does not consult the base view. Change::Mv { from, to } => mutation.rename(from, to), })() .map_err(|mut e| { diff --git a/engine/crates/layerfs-store-service/tests/epoch_fence.rs b/engine/crates/layerfs-store-service/tests/epoch_fence.rs new file mode 100644 index 0000000000..41674a4421 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/epoch_fence.rs @@ -0,0 +1,77 @@ +//! #41 T6.5: the writer-side `project_epoch` fence on the Workspace Commit +//! path. `fs.write` (and the SDK `commit` method) go through +//! `files::commit` -> `BoundWorkspace::commit` -> `HistoryCatalog::commit_branch`, +//! so a fence refusal must arrive as the spec's `Conflict`, not as a generic +//! failure, and an ordinary write must be unaffected. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_store_service::history::lookup; +use std::fs; +use support::Fixture; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn write(fixture: &Fixture, content: &str) -> Result { + fixture.try_call( + "fs.write", + vec![ + ("branch", text("work")), + ("path", text("/a.txt")), + ("content", text(content)), + ], + ) +} + +fn commits(fixture: &Fixture) -> usize { + support::items(&fixture.call("branch.list_commits", vec![("branch", text("work"))])).len() +} + +#[test] +fn a_missing_epoch_row_is_conflict_on_the_workspace_commit_path() { + let fixture = Fixture::new("epoch-fence-service", |source| { + fs::write(source.join("a.txt"), b"a\n").unwrap(); + 1 + }); + fixture.fork("work"); + + // The ordinary path still commits and leaves the epoch row alone. + write(&fixture, "b\n").expect("an ordinary write commits"); + let epoch_row = |path: &std::path::Path, stack: &[u8]| -> i64 { + rusqlite::Connection::open(path) + .unwrap() + .query_row( + "SELECT epoch FROM project_epoch WHERE layer_stack_id = ?1", + [stack], + |row| row.get(0), + ) + .unwrap() + }; + let stack = lookup::project(&fixture.service.store, "installed").expect("project"); + assert_eq!(epoch_row(&fixture.host.config.path, stack.id.as_slice()), 0); + + // Remove the fence row behind the service's back: the next Commit's + // start-of-transaction read finds no epoch to honour. + rusqlite::Connection::open(&fixture.host.config.path) + .unwrap() + .execute( + "DELETE FROM project_epoch WHERE layer_stack_id = ?1", + [stack.id.as_slice()], + ) + .unwrap(); + + let before = commits(&fixture); + let refused = write(&fixture, "c\n").expect_err("the fence refuses the commit"); + assert_eq!(refused.code, "Conflict", "{refused:?}"); + assert!(refused.message.contains("project epoch"), "{refused:?}"); + assert_eq!(commits(&fixture), before, "nothing was published"); + + fixture.cleanup(); +} diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs b/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs new file mode 100644 index 0000000000..6fdcc1bbc8 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs @@ -0,0 +1,290 @@ +//! T5.8: hidden `rm`/`mv` sources and the glob `removed_count`. +//! +//! §4.3 says a `file.mv` from a hidden path is `NotFound` and a `file.rm` of +//! a hidden path is `NotFound`. This method table spells a missing path +//! `NoSuchPath`, so those two calls report exactly what a never-existing path +//! reports; `cp`-from and `edit` consult the base view and answer `Removed`. +//! A listing counts the hidden entries among its direct results; a glob counts +//! every hidden entry the pattern would return anywhere under its query root. +//! Both totals are page-independent and `removed_count` is on every page. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::fs; +use support::{items, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, branch: &str, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text(branch)), ("path", text(path))], + ) +} + +/// The one `Removed` detail path and its frontier. +fn hidden_path(refusal: &SdkRefusal, path: &str) -> i64 { + assert_eq!(refusal.code, "Removed", "{refusal:?}"); + let Some(SdkValue::List(hidden)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + assert_eq!(hidden.len(), 1, "{refusal:?}"); + assert_eq!(hidden[0].get_text("path"), Some(path), "{refusal:?}"); + hidden[0].get_int("frontier_serial").unwrap() +} + +fn glob(fixture: &Fixture, pattern: &str, path: &str) -> SdkValue { + fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text(pattern)), + ("path", text(path)), + ], + ) +} + +fn paged_glob(fixture: &Fixture, pattern: &str, page: i64, page_size: i64) -> SdkValue { + fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text(pattern)), + ("path", text("/")), + ("page", SdkValue::Int(page)), + ("pageSize", SdkValue::Int(page_size)), + ], + ) +} + +fn matched(value: &SdkValue) -> Vec<&str> { + items(value) + .iter() + .map(|item| match item { + SdkValue::Text(text) => text.as_str(), + other => panic!("glob item is not a path: {other:?}"), + }) + .collect() +} + +#[test] +fn hidden_rm_and_mv_sources_are_absent_while_cp_and_edit_are_removed() { + let fixture = Fixture::new("t58-hidden-writes", |source| { + fs::write(source.join("a.txt"), "a\n").unwrap(); + fs::write(source.join("b.txt"), "b\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f.txt"), "f\n").unwrap(); + 4 + }); + for branch in ["rm", "mv", "cp", "edit"] { + fixture.fork(branch); + } + remove(&fixture, "/a.txt"); + remove(&fixture, "/d"); + + // §4.3's `NotFound` for a hidden `rm`/`mv`-from is this wire spelling: + // those calls read the private overlay, where a hidden path is already a + // whiteout, and answer like a never-existing path (`NoSuchPath`). + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/a.txt"))], + ) + .expect_err("hidden rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.mv", + vec![ + ("branch", text("mv")), + ("from", text("/a.txt")), + ("to", text("/moved.txt")), + ], + ) + .expect_err("hidden mv source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A hidden directory and its descendants are absent to `rm`/`mv` too. + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/d"))], + ) + .expect_err("hidden directory rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/d/f.txt"))], + ) + .expect_err("hidden child rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.mv", + vec![ + ("branch", text("mv")), + ("from", text("/d/f.txt")), + ("to", text("/moved.txt")), + ], + ) + .expect_err("hidden child mv source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // `cp`-from and `edit` consult the base tree, so a hidden source is + // `Removed` with the governing path and frontier, not `NoSuchPath`. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/a.txt")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("hidden edit"); + hidden_path(&refusal, "/a.txt"); + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/d/f.txt")), + ("oldText", text("f")), + ("newText", text("z")), + ], + ) + .expect_err("hidden child edit"); + hidden_path(&refusal, "/d/f.txt"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/a.txt")), + ("to", text("/copy.txt")), + ], + ) + .expect_err("hidden cp source"); + hidden_path(&refusal, "/a.txt"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/d")), + ("to", text("/copy")), + ], + ) + .expect_err("hidden cp directory"); + hidden_path(&refusal, "/d"); + + // A genuinely absent path is still `NoSuchPath` for `cp`/`edit`: the + // hidden and the absent cases stay distinguishable where the spec asks. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/missing")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("absent edit"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/missing")), + ("to", text("/copy")), + ], + ) + .expect_err("absent cp source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A refused call leaves the branch and the destination untouched. + assert_eq!( + read(&fixture, "cp", "/copy.txt").unwrap_err().code, + "NoSuchPath" + ); + assert_eq!( + read(&fixture, "mv", "/moved.txt").unwrap_err().code, + "NoSuchPath" + ); + assert_eq!( + read(&fixture, "rm", "/b.txt").unwrap().get_text("text"), + Some("b\n") + ); + fixture.cleanup(); +} + +#[test] +fn glob_removed_count_counts_the_hidden_results_at_any_depth_on_every_page() { + let fixture = Fixture::new("t58-glob-count", |source| { + fs::write(source.join("a.txt"), "a\n").unwrap(); + fs::write(source.join("b.txt"), "b\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f.txt"), "f\n").unwrap(); + fs::create_dir(source.join("d/sub")).unwrap(); + fs::write(source.join("d/sub/g.txt"), "g\n").unwrap(); + fs::create_dir(source.join("e")).unwrap(); + fs::write(source.join("e/h.txt"), "h\n").unwrap(); + 8 + }); + fixture.fork("work"); + remove(&fixture, "/a.txt"); + remove(&fixture, "/d"); + + // `/*` returns direct children only: the hidden file and directory are + // omitted and counted, but the hidden entries below `/d` are not results + // of this pattern and are not counted. + let top = glob(&fixture, "/*", "/"); + assert_eq!(matched(&top), vec!["/b.txt", "/e"], "{top:?}"); + assert_eq!(top.get_int("removed_count"), Some(2), "{top:?}"); + + // `/**` returns the whole queried subtree: every hidden entry the pattern + // would return counts, including `/d`'s descendants at any depth. + let all = glob(&fixture, "/**", "/"); + assert_eq!(matched(&all), vec!["/b.txt", "/e", "/e/h.txt"], "{all:?}"); + assert_eq!(all.get_int("removed_count"), Some(5), "{all:?}"); + + // A pattern that names hidden descendants counts those two, not the + // hidden directory itself (which this pattern does not match). + let below = glob(&fixture, "/d/*", "/"); + assert!(matched(&below).is_empty(), "{below:?}"); + assert_eq!(below.get_int("removed_count"), Some(2), "{below:?}"); + + // `removed_count` is present at 0 when the pattern matches no hidden entry. + let visible_only = glob(&fixture, "/b*", "/"); + assert_eq!(matched(&visible_only), vec!["/b.txt"], "{visible_only:?}"); + assert_eq!( + visible_only.get_int("removed_count"), + Some(0), + "{visible_only:?}" + ); + + // The count is page-independent: every page of the same glob reports the + // same total while its items carry only the visible matches. + let first = paged_glob(&fixture, "/**", 1, 2); + assert_eq!(matched(&first), vec!["/b.txt", "/e"], "{first:?}"); + assert_eq!(first.get_int("removed_count"), Some(5), "{first:?}"); + let second = paged_glob(&fixture, "/**", 2, 2); + assert_eq!(matched(&second), vec!["/e/h.txt"], "{second:?}"); + assert_eq!(second.get_int("removed_count"), Some(5), "{second:?}"); + fixture.cleanup(); +}