From bf6d2fc477ed564f1878d9a89af86ca5f86a9672 Mon Sep 17 00:00:00 2001 From: rain Date: Mon, 16 Nov 2026 00:37:54 +0000 Subject: [PATCH] feat(gc): PostgreSQL compaction (#49) --- .../layerfs-sdk-observe/references/tracing.md | 10 +- api/crates/layerfs-api/src/branch/mod.rs | 4 +- api/crates/layerfs-api/src/branch/remove.rs | 28 + api/crates/layerfs-api/src/calls/mod.rs | 1 + api/crates/layerfs-api/src/calls/render.rs | 97 +- api/crates/layerfs-api/src/calls/scope.rs | 5 + api/crates/layerfs-api/src/lib.rs | 2 +- api/crates/layerfs-api/src/project/delete.rs | 24 + api/crates/layerfs-api/src/project/mod.rs | 6 +- .../layerfs-api/src/project/remove_file.rs | 29 + api/crates/layerfs-api/src/store/compact.rs | 24 + api/crates/layerfs-api/src/store/mod.rs | 4 +- api/crates/layerfs-api/tests/calls_scope.rs | 10 + api/crates/layerfs-api/tests/calls_table.rs | 35 +- api/crates/layerfs-api/tests/gc_calls.rs | 324 ++++ api/crates/layerfs-api/tests/gc_trace.rs | 309 ++++ api/crates/layerfs-api/tests/logs.rs | 44 +- api/crates/layerfs-cli/tests/gc.rs | 169 ++ control/crates/layerfs-control/src/store.rs | 6 + .../crates/layerfs-store/src/calls/branch.rs | 26 + control/crates/layerfs-store/src/calls/op.rs | 10 + .../crates/layerfs-store/src/calls/project.rs | 22 + control/crates/layerfs-store/src/compact.rs | 150 ++ control/crates/layerfs-store/src/lib.rs | 2 + control/crates/layerfs-store/src/report.rs | 116 ++ .../layerfs-store/src/sqlite/transfer/step.rs | 7 + .../src/sqlite/transfer/steps.rs | 16 + control/crates/layerfs-store/src/status.rs | 55 +- .../tests/compact_report_status.rs | 212 +++ .../docs/issues/project-41/LayerFS-GC-spec.md | 37 +- .../issues/project-41/gc-t10-report-status.md | 96 + .../gc-t12-sqlite-compaction-fixtures.md | 88 + .../issues/project-41/gc-t8-sqlite-rebuild.md | 28 +- .../project-41/gc-t9-postgresql-rebuild.md | 115 ++ docs/architecture/api/calls/README.md | 10 +- docs/architecture/control/home/README.md | 8 +- .../sqlite/queries/history/gc_tuple_read.sql | 6 + .../src/backend/postgresql/compact.rs | 1413 +++++++++++++++ .../src/backend/postgresql/mod.rs | 1 + .../src/backend/records.rs | 4 + .../src/backend/sqlite/compact.rs | 384 +++- .../src/backend/sqlite/lifecycle/identity.rs | 51 + .../src/backend/sqlite/lifecycle/mod.rs | 1 + .../src/backend/sqlite/lifecycle/seal.rs | 8 + .../src/backend/sqlite/session/connection.rs | 22 + .../src/backend/sqlite/session/query.rs | 3 + .../src/backend/sqlite/session/transaction.rs | 3 + .../layerfs-persistence/src/history/mod.rs | 2 +- .../src/history/objects.rs | 71 + engine/crates/layerfs-persistence/src/lib.rs | 8 +- .../crates/layerfs-persistence/src/report.rs | 124 ++ .../tests/fixtures/sqlite_compact_golden.json | 407 +++++ .../tests/postgresql_compact.rs | 582 ++++++ .../tests/sqlite_compact.rs | 413 ++++- .../tests/sqlite_compact_crash.rs | 577 ++++++ .../tests/sqlite_compact_fixtures.rs | 1597 +++++++++++++++++ .../src/methods/branch/mod.rs | 2 + .../src/methods/branch/remove.rs | 10 + .../src/methods/dispatch.rs | 1 + .../src/methods/file/glob.rs | 2 + .../src/read/visibility.rs | 17 +- .../layerfs-store-service/src/write/files.rs | 7 + .../tests/sdk_path_hidden_semantics.rs | 290 +++ 63 files changed, 8014 insertions(+), 121 deletions(-) create mode 100644 api/crates/layerfs-api/src/branch/remove.rs create mode 100644 api/crates/layerfs-api/src/project/delete.rs create mode 100644 api/crates/layerfs-api/src/project/remove_file.rs create mode 100644 api/crates/layerfs-api/src/store/compact.rs create mode 100644 api/crates/layerfs-api/tests/gc_calls.rs create mode 100644 api/crates/layerfs-api/tests/gc_trace.rs create mode 100644 api/crates/layerfs-cli/tests/gc.rs create mode 100644 control/crates/layerfs-store/src/compact.rs create mode 100644 control/crates/layerfs-store/src/report.rs create mode 100644 control/crates/layerfs-store/tests/compact_report_status.rs create mode 100644 core/docs/issues/project-41/gc-t10-report-status.md create mode 100644 core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md create mode 100644 core/docs/issues/project-41/gc-t9-postgresql-rebuild.md create mode 100644 engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql create mode 100644 engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs create mode 100644 engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs create mode 100644 engine/crates/layerfs-persistence/src/report.rs create mode 100644 engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_compact.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs create mode 100644 engine/crates/layerfs-store-service/src/methods/branch/remove.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs diff --git a/.agents/skills/layerfs-sdk-observe/references/tracing.md b/.agents/skills/layerfs-sdk-observe/references/tracing.md index 316686abfa..5f3534511c 100644 --- a/.agents/skills/layerfs-sdk-observe/references/tracing.md +++ b/.agents/skills/layerfs-sdk-observe/references/tracing.md @@ -19,12 +19,16 @@ Python, direct `layerfs_api::call`) appends one line to and `LAYERFS_TRACE` / `LAYERFS_TRACE_STEP` do not affect it. ```json -{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","/index.ts","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} +{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","<8 bytes>","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} ``` It keeps 1.1.0's redaction rule (values cut to 200 characters; contents, -JSON, commands, messages, edit strings, passwords and URLs as byte counts) -and drops the daemon telemetry: no `sizes`, `scope` or `daemon`. `argv` is +JSON, commands, messages, edit strings, passwords and URLs as byte counts; +a path argument as the byte count of its normalized form, one leading `/` +stripped, so `/a` and `a` are both `<1 bytes>`; a glob or grep `pattern` by +its own bytes) and drops the daemon telemetry: no `sizes`, `scope` or +`daemon`. A refusal is logged as its code only: its message and `detail` are +never traced, so a `Removed` path stays out of the line. `argv` is the call as CLI words (`argv[0]`/`argv[1]` are the family and call), `outcome` is `"ok"` or the refusal code, `kind` and `commit`/`layer` come from the result, `client` is `cli`/`node`/`python`/`rust`, and `tag` is the diff --git a/api/crates/layerfs-api/src/branch/mod.rs b/api/crates/layerfs-api/src/branch/mod.rs index d4e4632dd6..bc4b6b5b7c 100644 --- a/api/crates/layerfs-api/src/branch/mod.rs +++ b/api/crates/layerfs-api/src/branch/mod.rs @@ -1,4 +1,4 @@ -//! The `branch` family: 8 calls, one file each. +//! The `branch` family: 9 calls, one file each. mod commits; mod diff; mod fork; @@ -7,6 +7,7 @@ mod get_commit; mod list; mod merge; mod publish; +mod remove; use crate::calls::Calls; @@ -21,5 +22,6 @@ pub(crate) fn calls() -> Calls { merge::CALL, publish::CALL, diff::CALL, + remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/branch/remove.rs b/api/crates/layerfs-api/src/branch/remove.rs new file mode 100644 index 0000000000..e9279cbd84 --- /dev/null +++ b/api/crates/layerfs-api/src/branch/remove.rs @@ -0,0 +1,28 @@ +//! `branch.remove`: soft-remove one Branch through its Store marker. +use crate::calls::table::Kind::{Name, Project}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::branch::{remove, BranchRemove}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "branch", + "remove", + &[a("project", Project), a("branch", Name)], + &[], + "soft-remove one Branch; its name stays taken until compaction", + run, +); + +/// Runs `branch.remove`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = BranchRemove { + project: text("project").unwrap_or_default(), + branch: text("branch").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/calls/mod.rs b/api/crates/layerfs-api/src/calls/mod.rs index 45c32f6583..ef74ed4009 100644 --- a/api/crates/layerfs-api/src/calls/mod.rs +++ b/api/crates/layerfs-api/src/calls/mod.rs @@ -18,6 +18,7 @@ pub use check::size; pub use context::Context; pub use dispatch::{call, call_scoped, home, Client}; pub use layerfs_home::json; +pub use render::REDACTED; pub use scope::Scope; pub use table::{find, table_json, Arg, Call, CallRef, Calls, Families, Family, Kind, Run}; pub use text::call_json; diff --git a/api/crates/layerfs-api/src/calls/render.rs b/api/crates/layerfs-api/src/calls/render.rs index 87d904246a..c82fc1054d 100644 --- a/api/crates/layerfs-api/src/calls/render.rs +++ b/api/crates/layerfs-api/src/calls/render.rs @@ -9,9 +9,46 @@ use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::outcome::Outcome; /// The `Text` arguments logged only as ``: exec command text, -/// Commit messages, edit strings, Store passwords and connection URLs that -/// may embed `user:pass@`. A new call with such an argument adds its name. -pub const REDACTED: [&str; 6] = ["command", "message", "old", "new", "password", "url"]; +/// Commit messages, edit strings, Store passwords, connection URLs that may +/// embed `user:pass@`, every path argument (`path`, `left_path`, +/// `right_path`, `from_path`, `to_path`, `path_prefix`) and a glob or grep +/// `pattern`. A path is counted in its normalized form ([`PATHS`]), so a +/// `project.remove_file` or `file.*` path never enters the call log (#41 +/// §4.3). A new call with such an argument adds its name. +pub const REDACTED: [&str; 13] = [ + "command", + "message", + "old", + "new", + "password", + "url", + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", +]; + +/// The arguments that name a path: they log the length of the normalized +/// path, one leading `/` stripped, so `/a` and `a` count the same and a root +/// `/` counts as 0 (#41 §4.3). A `pattern` is not a path and keeps every +/// byte it has. +const PATHS: [&str; 6] = [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", +]; + +/// A path argument's logged byte length: the §4.3 normalization (strip one +/// leading `/`) applied before counting. +fn path_len(text: &str) -> usize { + text.strip_prefix('/').unwrap_or(text).len() +} /// The most characters a logged value keeps. const CUT: usize = 200; @@ -46,6 +83,7 @@ fn word(name: &str, kind: Kind, value: &SdkValue) -> Option { (Kind::Content, SdkValue::Text(text)) => bytes(text.len()), (Kind::Content, SdkValue::Bytes(raw)) => bytes(raw.len()), (Kind::Content | Kind::Json, value) => bytes(layerfs_home::json::print(value).len()), + (Kind::Text, SdkValue::Text(text)) if PATHS.contains(&name) => bytes(path_len(text)), (Kind::Text, SdkValue::Text(text)) if REDACTED.contains(&name) => bytes(text.len()), (_, SdkValue::Text(text)) => SdkValue::Text(cut(text)), (_, SdkValue::Int(n)) => SdkValue::Text(n.to_string()), @@ -187,3 +225,56 @@ pub fn line( } SdkValue::Map(fields) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_path_argument_name_is_redacted() { + for name in [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", + ] { + assert!(REDACTED.contains(&name), "{name} is not redacted"); + } + for name in ["project", "branch", "name", "commit", "content", "changes"] { + assert!(!REDACTED.contains(&name), "{name} is redacted"); + } + } + + #[test] + fn a_path_logs_the_byte_length_of_its_normalized_form() { + // `/a` and `a` are the same path (§4.3), so they log the same count; + // the root `/` normalizes to the empty relative path. + for (text, len) in [ + ("/base.txt", 8), + ("base.txt", 8), + ("/a", 1), + ("a", 1), + ("/", 0), + ] { + let value = SdkValue::text(text); + assert_eq!( + word("path", Kind::Text, &value), + Some(SdkValue::text(format!("<{len} bytes>"))), + "{text}" + ); + } + // A glob pattern is not a path and keeps every byte it has. + assert_eq!( + word("pattern", Kind::Text, &SdkValue::text("*.txt")), + Some(SdkValue::text("<5 bytes>")) + ); + // Text that is not a path is still its own (cut) text. + assert_eq!( + word("branch", Kind::Text, &SdkValue::text("/main")), + Some(SdkValue::text("/main")) + ); + } +} diff --git a/api/crates/layerfs-api/src/calls/scope.rs b/api/crates/layerfs-api/src/calls/scope.rs index c7c81b6b2c..bdb139bf60 100644 --- a/api/crates/layerfs-api/src/calls/scope.rs +++ b/api/crates/layerfs-api/src/calls/scope.rs @@ -151,6 +151,11 @@ impl Scope { | ("file", "revision") => { return Err(out("history")); } + // GC retires or compacts a whole entity, never one folder: + // a scope refuses these instead of widening them (§10). + ("branch", "remove") | ("project", "delete" | "remove_file") | ("store", "compact") => { + return Err(out(name)); + } ("file", _) => { bound(&args)?; let paths: &[&str] = match call { diff --git a/api/crates/layerfs-api/src/lib.rs b/api/crates/layerfs-api/src/lib.rs index 0d2b610fe9..0f67fadce2 100644 --- a/api/crates/layerfs-api/src/lib.rs +++ b/api/crates/layerfs-api/src/lib.rs @@ -71,7 +71,7 @@ pub use layerfs_sandbox::{ BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi, }; -/// The ten families, in the order help lists them: 73 calls, the same +/// The ten families, in the order help lists them: 77 calls, the same /// from the `layerfs` binary, Node and Python. pub fn families() -> Families { &[ diff --git a/api/crates/layerfs-api/src/project/delete.rs b/api/crates/layerfs-api/src/project/delete.rs new file mode 100644 index 0000000000..9afba64e3f --- /dev/null +++ b/api/crates/layerfs-api/src/project/delete.rs @@ -0,0 +1,24 @@ +//! `project.delete`: retire one project in its Store and remove its home entry. +use crate::calls::table::Kind::Project; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::names::ProjectRef; +use layerfs_home::outcome::Outcome; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "project", + "delete", + &[a("project", Project)], + &[], + "retire one project in its Store and remove its home entry", + run, +); + +/// Runs `project.delete`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let project = ProjectRef::parse(args.get_text("project").unwrap_or(""))?; + let (home, events) = context.parts(); + layerfs_control::project::delete(home, events, &project) +} diff --git a/api/crates/layerfs-api/src/project/mod.rs b/api/crates/layerfs-api/src/project/mod.rs index 03b79a3597..7a65fd924f 100644 --- a/api/crates/layerfs-api/src/project/mod.rs +++ b/api/crates/layerfs-api/src/project/mod.rs @@ -1,9 +1,11 @@ -//! The `project` family: 6 calls, one file each. +//! The `project` family: 8 calls, one file each. +mod delete; mod export; mod get; mod init; mod list; mod remove; +mod remove_file; mod stats; use crate::calls::Calls; @@ -16,6 +18,8 @@ pub(crate) fn calls() -> Calls { get::CALL, stats::CALL, export::CALL, + delete::CALL, + remove_file::CALL, remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/project/remove_file.rs b/api/crates/layerfs-api/src/project/remove_file.rs new file mode 100644 index 0000000000..3c8d4f2176 --- /dev/null +++ b/api/crates/layerfs-api/src/project/remove_file.rs @@ -0,0 +1,29 @@ +//! `project.remove_file`: hide one absolute path from every node at or below +//! its marker frontier. +use crate::calls::table::Kind::{Project, Text}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::project::{remove_file, ProjectRemoveFile}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "project", + "remove_file", + &[a("project", Project), a("path", Text)], + &[], + "hide one absolute path from every node at or below its marker frontier", + run, +); + +/// Runs `project.remove_file`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = ProjectRemoveFile { + project: text("project").unwrap_or_default(), + path: text("path").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove_file(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/store/compact.rs b/api/crates/layerfs-api/src/store/compact.rs new file mode 100644 index 0000000000..dc8a044d3b --- /dev/null +++ b/api/crates/layerfs-api/src/store/compact.rs @@ -0,0 +1,24 @@ +//! `store.compact`: reclaim dead bytes from one sqlite Store. +use crate::calls::table::Kind::Name; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::names::StoreName; +use layerfs_home::outcome::Outcome; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "store", + "compact", + &[a("name", Name)], + &[], + "reclaim dead bytes from one sqlite Store", + run, +); + +/// Runs `store.compact`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let name = StoreName::parse(args.get_text("name").unwrap_or(""))?; + let (home, events) = context.parts(); + layerfs_control::store::compact(home, events, &name) +} diff --git a/api/crates/layerfs-api/src/store/mod.rs b/api/crates/layerfs-api/src/store/mod.rs index 8c0df7b57c..2142e041ea 100644 --- a/api/crates/layerfs-api/src/store/mod.rs +++ b/api/crates/layerfs-api/src/store/mod.rs @@ -1,6 +1,7 @@ -//! The `store` family: 12 calls, one file each (H1: push and pull replace +//! The `store` family: 13 calls, one file each (H1: push and pull replace //! move; attach and detach supersede #536 C4). mod attach; +mod compact; mod connect; mod create; mod detach; @@ -29,6 +30,7 @@ pub(crate) fn calls() -> Calls { push::CALL, pull::CALL, remove::CALL, + compact::CALL, disconnect::CALL, ] } diff --git a/api/crates/layerfs-api/tests/calls_scope.rs b/api/crates/layerfs-api/tests/calls_scope.rs index 11d9b26286..5b324435e0 100644 --- a/api/crates/layerfs-api/tests/calls_scope.rs +++ b/api/crates/layerfs-api/tests/calls_scope.rs @@ -115,6 +115,16 @@ fn a_scope_refuses_every_call_it_does_not_cover() { ("store.remove", &[("name", t("shop"))]), ("store.list", &[]), ("project.remove", &[("project", t("shop/api"))]), + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("shop/api"))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", t("/web/a"))], + ), + ("store.compact", &[("name", t("shop"))]), ( "project.init", &[("project", t("shop/x")), ("folder", t("/tmp"))], diff --git a/api/crates/layerfs-api/tests/calls_table.rs b/api/crates/layerfs-api/tests/calls_table.rs index 39cdc1df3e..2ba77897cd 100644 --- a/api/crates/layerfs-api/tests/calls_table.rs +++ b/api/crates/layerfs-api/tests/calls_table.rs @@ -1,4 +1,4 @@ -//! The call table and its dispatcher (#533 step 2): 73 calls in ten +//! The call table and its dispatcher (#533 step 2): 77 calls in ten //! families, arguments checked before any effect, one test per family that //! a valid call reaches its family function and a wrong one is refused. use layerfs_api::{families, json, keys, size, Call, Kind}; @@ -51,9 +51,9 @@ fn code(name: &str, args: SdkValue) -> String { } #[test] -fn the_table_has_ten_families_and_73_calls() { +fn the_table_has_ten_families_and_77_calls() { let all: Vec<&Call> = layerfs_api::calls(); - assert_eq!(all.len(), 73); + assert_eq!(all.len(), 77); // each family holds only its own calls, and has at least one for family in families() { assert!(!family.calls().is_empty(), "{}", family.name); @@ -66,7 +66,7 @@ fn the_table_has_ten_families_and_73_calls() { let mut names: Vec<_> = all.iter().map(|c| c.name()).collect(); names.sort(); names.dedup(); - assert_eq!(names.len(), 73, "call names are unique"); + assert_eq!(names.len(), 77, "call names are unique"); // an optional positional is only ever followed by optional ones (D7) for c in &all { let first_optional = c.args.iter().position(|a| a.optional); @@ -75,7 +75,7 @@ fn the_table_has_ten_families_and_73_calls() { } } let counts: Vec = families().iter().map(|f| f.calls().len()).collect(); - assert_eq!(counts, [6, 8, 3, 15, 4, 6, 12, 12, 5, 2]); + assert_eq!(counts, [8, 9, 3, 15, 4, 6, 12, 13, 5, 2]); } #[test] @@ -444,3 +444,28 @@ fn sandbox_images_are_named_by_the_image_kind() { assert!(prune.args[1].optional); assert_eq!(prune.options[0].name, "unused"); } + +#[test] +fn the_four_gc_calls_take_only_ids() { + // T11.1: the four GC calls, each one resource identifier and no option. + use layerfs_api::Kind; + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name}"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name}"); + assert!(spec.options.is_empty(), "{name} takes no option"); + } +} diff --git a/api/crates/layerfs-api/tests/gc_calls.rs b/api/crates/layerfs-api/tests/gc_calls.rs new file mode 100644 index 0000000000..c7d37f5d24 --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_calls.rs @@ -0,0 +1,324 @@ +//! T11.1: the four GC calls in the API call table — their ids-only arguments, +//! typed refusals before any effect, and the result shapes of one reachable +//! refusal per call plus the three removal results and the §11 report. +use layerfs_api::Kind; +use layerfs_bridge::control::SdkValue; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-calls-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call( + home: &Path, + name: &str, + fields: &[(&str, SdkValue)], +) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) + .map_err(|refusal| (refusal.code, refusal.message)) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> (String, String) { + call(home, name, fields).expect_err("the call is refused") +} + +/// A home with Store `shop` and project `shop/api` imported from a folder. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +#[test] +fn the_four_gc_calls_take_ids_only() { + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name} arguments"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name} required"); + assert!(spec.options.is_empty(), "{name} has no options"); + } +} + +#[test] +fn wrong_kinds_and_unknown_options_are_refused_before_any_effect() { + let home = tempdir("types"); + layerfs_api::home::open(&home).unwrap(); + let wrong: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", SdkValue::Int(1))], + ), + ("project.delete", &[("project", SdkValue::Int(1))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", SdkValue::Int(1))], + ), + ("store.compact", &[("name", SdkValue::Int(1))]), + ]; + for &(name, fields) in wrong { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.starts_with(name), "{name}: {message}"); + } + let unknown: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[ + ("project", t("shop/api")), + ("branch", t("main")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "project.delete", + &[("project", t("shop/api")), ("force", SdkValue::Bool(true))], + ), + ( + "project.remove_file", + &[ + ("project", t("shop/api")), + ("path", t("/base.txt")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "store.compact", + &[("name", t("shop")), ("force", SdkValue::Bool(true))], + ), + ]; + for &(name, fields) in unknown { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!( + message.contains("unknown argument `force`"), + "{name}: {message}" + ); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn branch_remove_refuses_reserved_and_absent_branches() { + let (home, folder) = project(); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("main"))]); + assert_eq!(code, "ReservedName", "{message}"); + // nothing ran: main is still there + ok(&home, "branch.get", &[p(), ("branch", t("main"))]); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("ghost"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + // a live Branch is hidden by the marker and reported as `Removed` + ok( + &home, + "branch.fork", + &[p(), ("name", t("work")), ("from_branch", t("main"))], + ); + let removed = ok(&home, "branch.remove", &[p(), ("branch", t("work"))]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the project is the caller's /, not the Store's inner name + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert_eq!(removed.get_text("branch"), Some("work")); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + let (code, message) = refused(&home, "branch.get", &[p(), ("branch", t("work"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_remove_file_checks_its_path_grammar_then_reports_markers() { + let (home, folder) = project(); + for bad in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "\\", + ] { + let (code, message) = refused(&home, "project.remove_file", &[p(), ("path", t(bad))]); + assert_eq!(code, "InvalidInput", "{bad:?}: {message}"); + } + // no effect: the file the bad calls named is still readable + let read = ok( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.get_text("text"), Some("hello\n")); + + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the same full project name the caller passed + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("frontier_serial").is_some()); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + // a repeat at the same frontier is UpToDate + let repeat = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(repeat.get_text("kind"), Some("UpToDate")); + assert_eq!( + repeat.get_int("frontier_serial"), + removed.get_int("frontier_serial") + ); + // the marker hides it from reads + let (code, message) = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(code, "Removed", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_delete_reports_removal_and_retires_the_entry() { + let (home, folder) = project(); + let (code, message) = refused(&home, "project.delete", &[("project", t("shop/ghost"))]); + assert_eq!(code, "NoSuchProject", "{message}"); + let removed = ok(&home, "project.delete", &[p()]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("deleted_ms").is_some_and(|ms| ms >= 0)); + assert!(removed.get_bool("home_entry_removed")); + let (code, _) = refused(&home, "project.get", &[p()]); + assert_eq!(code, "NoSuchProject"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn store_compact_refuses_an_unknown_store_and_reports_a_run() { + let home = tempdir("compact-none"); + layerfs_api::home::open(&home).unwrap(); + let (code, message) = refused(&home, "store.compact", &[("name", t("ghost"))]); + assert_eq!(code, "NoSuchStore", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + let path = report.get_text("path").expect("path"); + assert!(matches!(path, "none" | "rebuild"), "{path}"); + let generation = report.get("generation").expect("generation"); + let before = generation.get_int("before").unwrap(); + let after = generation.get_int("after").unwrap(); + if path == "none" { + assert_eq!(before, after); + } else { + assert_eq!(after, before + 1); + } + for field in [ + "dead_bytes_bound", + "dead_bytes_refined", + "logical_discarded", + "backend_reusable", + "physical_shrink", + "peak_temp_bytes", + "estimate", + "zero_reasons", + "kept_waste", + "residue", + ] { + assert!(report.get(field).is_some(), "missing {field}"); + } + let kept = report.get("kept_waste").expect("kept_waste"); + for field in [ + "dead_in_retained_packs_bytes", + "delta_bases", + "inode_blocks_bytes", + ] { + assert!(kept.get(field).is_some(), "missing kept_waste.{field}"); + } + let residue = report.get("residue").expect("residue"); + assert!(residue.get("bytes").is_some()); + assert!(residue.get("artifacts").is_some()); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn a_bare_project_name_is_refused_by_every_project_call() { + let home = tempdir("bare-names"); + layerfs_api::home::open(&home).unwrap(); + let bare: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("api"))]), + ( + "project.remove_file", + &[("project", t("api")), ("path", t("/base.txt"))], + ), + ]; + for &(name, fields) in bare { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.contains("with its Store"), "{name}: {message}"); + } + // store.compact takes a Store name; a project path is not one + let (code, message) = refused(&home, "store.compact", &[("name", t("shop/api"))]); + assert_eq!(code, "InvalidInput", "{message}"); + assert!(message.contains("store name"), "{message}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/gc_trace.rs b/api/crates/layerfs-api/tests/gc_trace.rs new file mode 100644 index 0000000000..a3cb2144c2 --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_trace.rs @@ -0,0 +1,309 @@ +//! T-G2: the call log redacts every path argument (#41 §4.3). A +//! `project.remove_file` or a hidden `file.read`/`stat`/`list`/`glob` path is +//! logged as `` of its normalized form (one leading `/` stripped), +//! the `Removed` refusal still echoes the request path, the salted trace key +//! changes with the salt, and a `store.compact` report's artifact names never +//! reach the log. +use layerfs_api::calls::REDACTED; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::path::{path_key, trace_key}; +use layerfs_history::LayerStackId; +use layerfs_home::home::paths; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard}; + +static ENV: Mutex<()> = Mutex::new(()); +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// The tests share the process environment, so each one holds this lock. +fn serial() -> MutexGuard<'static, ()> { + let guard = ENV.lock().unwrap_or_else(|poison| poison.into_inner()); + std::env::remove_var("LAYERFS_LOGS"); + std::env::remove_var("LAYERFS_LOGS_TAG"); + guard +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-trace-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkRefusal { + call(home, name, fields).expect_err("the call is refused") +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +/// A home with Store `shop` and project `shop/api` imported from a folder +/// holding `base.txt`. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +/// Every line of the call log, oldest first. +fn lines(home: &Path) -> Vec { + match std::fs::read_to_string(paths::calls_file(home)) { + Ok(text) => text + .lines() + .map(|line| layerfs_api::json::parse(line).unwrap()) + .collect(), + Err(_) => Vec::new(), + } +} + +fn last(home: &Path) -> SdkValue { + lines(home).pop().expect("a line") +} + +fn log_text(home: &Path) -> String { + std::fs::read_to_string(paths::calls_file(home)).unwrap_or_default() +} + +fn argv(line: &SdkValue) -> Vec { + match line.get("argv") { + Some(SdkValue::List(words)) => words + .iter() + .map(|word| match word { + SdkValue::Text(text) => text.clone(), + other => panic!("argv word {other:?}"), + }) + .collect(), + other => panic!("no argv: {other:?}"), + } +} + +fn bytes(n: usize) -> String { + format!("<{n} bytes>") +} + +/// The one path a `Removed` refusal details. +fn removed_path(refusal: &SdkRefusal) -> &str { + let Some(SdkValue::List(paths)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + paths + .first() + .and_then(|path| path.get_text("path")) + .expect("one path") +} + +#[test] +fn path_arguments_are_byte_counts_in_the_call_log() { + let _env = serial(); + let (home, folder) = project(); + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!( + argv(&last(&home)), + [ + "project".to_owned(), + "remove-file".into(), + "shop/api".into(), + bytes("base.txt".len()), + ] + ); + + // Hidden reads, stats, listings and globs: a path is a byte count. + let read = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.code, "Removed"); + assert_eq!( + argv(&last(&home)), + [ + "file".to_owned(), + "read".into(), + "shop/api".into(), + "main".into(), + bytes("base.txt".len()), + ] + ); + let stat = refused( + &home, + "file.stat", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(stat.code, "Removed"); + assert_eq!(argv(&last(&home))[4], bytes("base.txt".len())); + + let listed = ok( + &home, + "file.list", + &[p(), ("branch", t("main")), ("path", t("/"))], + ); + assert_eq!(listed.get_int("removed_count"), Some(1)); + assert_eq!(argv(&last(&home))[4], bytes(0)); + + let globbed = ok( + &home, + "file.glob", + &[ + p(), + ("branch", t("main")), + ("path", t("/")), + ("pattern", t("*.txt")), + ], + ); + assert_eq!(globbed.get_int("removed_count"), Some(1)); + let words = argv(&last(&home)); + assert_eq!(words[4], bytes(0)); + assert_eq!(words[5], bytes("*.txt".len())); + + let text = log_text(&home); + assert!(!text.contains("/base.txt"), "raw path in the log:\n{text}"); + assert!(!text.contains("base.txt"), "raw path in the log:\n{text}"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn a_removed_refusal_echoes_the_path_the_log_does_not() { + let _env = serial(); + let (home, folder) = project(); + ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + let refusal = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(refusal.code, "Removed"); + assert_eq!(removed_path(&refusal), "/base.txt"); + let line = last(&home); + assert_eq!(line.get_text("outcome"), Some("Removed")); + assert!(line.get("detail").is_none(), "a detail reached the log"); + let text = log_text(&home); + assert!(!text.contains("/base.txt")); + assert!(!text.contains("base.txt")); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn the_redacted_set_covers_every_path_bearing_call() { + for name in [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", + ] { + assert!(REDACTED.contains(&name), "{name} is not redacted"); + } + for (name, args) in [ + ("project.remove_file", &["path"][..]), + ("file.read", &["path"][..]), + ("file.stat", &["path"][..]), + ("file.list", &["path"][..]), + ("file.glob", &["path", "pattern"][..]), + ("file.diff", &["left_path", "right_path"][..]), + ("file.cp", &["from_path", "to_path"][..]), + ("file.mv", &["from_path", "to_path"][..]), + ("branch.diff", &["path_prefix"][..]), + ] { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + for arg in args { + assert!(spec.arg(arg).is_some(), "{name} has no {arg}"); + assert!(REDACTED.contains(arg), "{name}'s {arg} is not redacted"); + } + } + // text that is not a path stays visible + for name in ["project", "branch", "name", "commit", "content", "changes"] { + assert!(!REDACTED.contains(&name), "{name} is redacted"); + } +} + +#[test] +fn the_trace_key_is_salted_by_the_store_policy_salt() { + let _env = serial(); + let stack = LayerStackId::from_authority([7; 16]); + let path = b"a/b"; + let first = trace_key(&[1; 32], stack, path); + assert_eq!(first, trace_key(&[1; 32], stack, path), "stable salt"); + let second = trace_key(&[2; 32], stack, path); + assert_ne!(first, second, "a different salt changes the keyed hash"); + assert_eq!(first.len(), 32); + // the keyed hash is not the marker's plain BLAKE3(normalized path) + assert_ne!(first, path_key(path)); + // the stack is part of the key + let other = LayerStackId::from_authority([9; 16]); + assert_ne!(first, trace_key(&[1; 32], other, path)); +} + +#[test] +fn store_compact_artifact_names_never_reach_the_log() { + let _env = serial(); + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + assert_eq!( + argv(&last(&home)), + ["store".to_owned(), "compact".into(), "shop".into()] + ); + let text = log_text(&home); + assert!( + !text.contains(".compact"), + "an artifact name reached the log:\n{text}" + ); + let store = paths::store_file(&home, "shop"); + let name = store.file_name().unwrap().to_string_lossy().into_owned(); + assert!(!text.contains(&name), "the Store file name reached the log"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/logs.rs b/api/crates/layerfs-api/tests/logs.rs index 5fdbf68dbf..cbb0f9cc44 100644 --- a/api/crates/layerfs-api/tests/logs.rs +++ b/api/crates/layerfs-api/tests/logs.rs @@ -181,21 +181,19 @@ fn argv_has_positionals_options_flags_and_the_scope_before_it_rewrites() { ("page_size", SdkValue::Int(5)), ], ); - assert_eq!( - argv(&last(&home)), - [ - "file", - "diff", - "shop/api", - "main", - "/README.md", - "main", - "/README.md", - "--allow-missing", - "--page-size", - "5" - ] - ); + let want: Vec = vec![ + "file".into(), + "diff".into(), + "shop/api".into(), + "main".into(), + bytes("README.md".len()), + "main".into(), + bytes("README.md".len()), + "--allow-missing".into(), + "--page-size".into(), + "5".into(), + ]; + assert_eq!(argv(&last(&home)), want); // optional positionals stop at the first absent one let _ = call(&home, "backup.list", &[]); assert_eq!(argv(&last(&home)), ["backup", "list"]); @@ -266,7 +264,7 @@ fn free_text_credentials_content_and_json_are_byte_counts() { "write".into(), "shop/api".into(), "dev".into(), - "/a.ts".into(), + bytes("a.ts".len()), bytes(5321), "--message".into(), bytes(5) @@ -283,16 +281,22 @@ fn free_text_credentials_content_and_json_are_byte_counts() { ], ); assert_eq!(argv(&last(&home))[5], bytes(7)); - // text is cut at 200 characters + // a path argument is logged as its byte length, never its text let long = format!("/{}", "é".repeat(300)); let _ = call( &home, "file.stat", &[p(), ("branch", t("main")), ("path", t(&long))], ); - let path = &argv(&last(&home))[4]; - assert_eq!(path.chars().count(), 201); - assert!(path.ends_with('…')); + // a path logs the length of its normalized form: one leading `/` is + // stripped, so this path counts 600 bytes, not 601 + assert_eq!(argv(&last(&home))[4], bytes(long.len() - 1)); + // text that is not redacted is cut at 200 characters + let long = "n".repeat(300); + let _ = call(&home, "store.get", &[("name", t(&long))]); + let name = &argv(&last(&home))[2]; + assert_eq!(name.chars().count(), 201); + assert!(name.ends_with('…')); // each named text argument let _ = call( &home, diff --git a/api/crates/layerfs-cli/tests/gc.rs b/api/crates/layerfs-cli/tests/gc.rs new file mode 100644 index 0000000000..36d04aeb68 --- /dev/null +++ b/api/crates/layerfs-cli/tests/gc.rs @@ -0,0 +1,169 @@ +//! T11.2: the four GC calls on the shell. Each has one command, the call +//! that it names and the arguments the table gives it: a wrong arity or an +//! option the call does not take is exit 2 before any call, a well-formed +//! command line reaches the library, and the help is the table's. +mod common; +use common::{layerfs, run, tempdir, text}; +use std::path::Path; + +fn run_code(home: &Path, words: &[&str]) -> (Option, String, String) { + let out = run(&mut layerfs(home, words)); + (out.status.code(), text(&out.stdout), text(&out.stderr)) +} + +/// The help line of each call, as the table renders it: remove-file is the +/// shell spelling of remove_file. +const CALLS: &[(&[&str], &str, &str)] = &[ + ( + &["branch", "remove"], + "layerfs branch remove ", + "soft-remove one Branch", + ), + ( + &["project", "delete"], + "layerfs project delete ", + "retire one project", + ), + ( + &["project", "remove-file"], + "layerfs project remove-file ", + "hide one absolute path", + ), + ( + &["store", "compact"], + "layerfs store compact ", + "reclaim dead bytes", + ), +]; + +#[test] +fn help_lists_each_gc_call_with_its_usage() { + let home = tempdir("gc-help"); + let (code, out, err) = run_code(&home, &["help"]); + assert_eq!(code, Some(0), "{err}"); + assert!(err.is_empty(), "{err}"); + let line = |family: &str| { + out.lines() + .find(|line| line.split_whitespace().next() == Some(family)) + .unwrap_or_else(|| panic!("no {family} line in {out}")) + }; + assert!(line("branch").contains("remove"), "{}", line("branch")); + assert!( + line("project").contains("delete") && line("project").contains("remove-file"), + "{}", + line("project") + ); + assert!(line("store").contains("compact"), "{}", line("store")); + + for (words, usage, help) in CALLS { + let topic: Vec<&str> = std::iter::once("help") + .chain(words.iter().copied()) + .collect(); + let (code, out, err) = run_code(&home, &topic); + assert_eq!(code, Some(0), "{words:?}: {err}"); + assert!(err.is_empty(), "{words:?}: {err}"); + assert!(out.starts_with(usage), "{words:?}: {out}"); + assert!(out.contains(help), "{words:?}: {out}"); + // the same help after the call, through -h and --help + for flag in ["-h", "--help"] { + let with_flag: Vec<&str> = words.iter().copied().chain([flag]).collect(); + let (code, more, err) = run_code(&home, &with_flag); + assert_eq!(code, Some(0), "{with_flag:?}: {err}"); + assert_eq!(more, out, "{with_flag:?}"); + } + } + assert!(!home.join("store").exists(), "help made no home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn wrong_arity_and_options_are_usage_errors() { + let home = tempdir("gc-usage"); + let wrong: &[&[&str]] = &[ + &["branch", "remove"], + &["branch", "remove", "shop/api"], + &["branch", "remove", "shop/api", "work", "extra"], + &["branch", "remove", "shop/api", "work", "--force"], + &["project", "delete"], + &["project", "delete", "shop/api", "extra"], + &["project", "delete", "shop/api", "--force"], + &["project", "remove-file", "shop/api"], + &["project", "remove-file", "shop/api", "/a", "extra"], + &["project", "remove-file", "shop/api", "/a", "--force"], + &["store", "compact"], + &["store", "compact", "shop", "extra"], + &["store", "compact", "shop", "--force"], + ]; + for words in wrong { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(2), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("layerfs: "), "{words:?}: {err}"); + } + // no positional was called: a usage error makes no home + assert!(!home.join("store").exists(), "a usage error made a home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn well_formed_lines_reach_the_library() { + let home = tempdir("gc-reach"); + let well_formed: &[&[&str]] = &[ + &["branch", "remove", "shop/api", "work"], + &["project", "delete", "shop/api"], + &["project", "remove-file", "shop/api", "/a"], + &["store", "compact", "shop"], + ]; + for words in well_formed { + let (code, out, err) = run_code(&home, words); + // no Store in the home: the library refused after reading the line + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!(err.contains("\"code\":\"NoSuchStore\""), "{words:?}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_bare_project_is_refused_and_the_four_are_out_of_scope() { + let home = tempdir("gc-names"); + // the library, not the parser, refuses a project without its Store + for words in [ + &["branch", "remove", "api", "main"][..], + &["project", "delete", "api"], + &["project", "remove-file", "api", "/a"], + ] { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!( + err.contains("\"code\":\"InvalidInput\""), + "{words:?}: {err}" + ); + assert!(err.contains("with its Store"), "{words:?}: {err}"); + } + + // a folder scope guards one folder and one Branch: none of the four is a + // folder operation, so every one is refused, never widened + let scoped: &[(&[&str], &str)] = &[ + (&["branch", "remove", "shop/api", "main"], "branch.remove"), + (&["project", "delete", "shop/api"], "project.delete"), + ( + &["project", "remove-file", "shop/api", "/web/a"], + "project.remove_file", + ), + (&["store", "compact", "shop"], "store.compact"), + ]; + for (words, call) in scoped { + let mut command: Vec<&str> = words.to_vec(); + command.extend(["--scope", "/web", "--scope-project", "shop/api"]); + let (code, out, err) = run_code(&home, &command); + assert_eq!(code, Some(125), "{call}: {err}"); + assert!(out.is_empty(), "{call}: {out}"); + assert!(err.contains("out of scope: "), "{call}: {err}"); + assert!(err.contains(call), "{call}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/src/store.rs b/control/crates/layerfs-control/src/store.rs index d67e04a111..8b7dc2334f 100644 --- a/control/crates/layerfs-control/src/store.rs +++ b/control/crates/layerfs-control/src/store.rs @@ -18,6 +18,12 @@ use std::path::Path; pub use layerfs_store::close_idle; +/// `store.compact`: reclaim one sqlite Store's dead bytes, answering the +/// §11 report. +pub fn compact(home: &Path, events: &mut Events<'_>, name: &StoreName) -> Outcome { + layerfs_store::compact::compact(home, events, name.as_str()) +} + /// Who uses a Store or a project, read from the Sandbox records. pub const RELATED: Related = Related { store_idle: inventory::store_idle, diff --git a/control/crates/layerfs-store/src/calls/branch.rs b/control/crates/layerfs-store/src/calls/branch.rs index 20dc55f4e9..a37bdee913 100644 --- a/control/crates/layerfs-store/src/calls/branch.rs +++ b/control/crates/layerfs-store/src/calls/branch.rs @@ -117,6 +117,15 @@ pub struct BranchPublish { pub commit: Option, } +/// `branch.remove(project, branch)`. +#[derive(Clone, Debug, Default)] +pub struct BranchRemove { + /// The project. + pub project: String, + /// The Branch. + pub branch: String, +} + /// `branch.diff(project, from_branch, to_branch, { paths_only, page_size, /// page_pos, cursor })`. #[derive(Clone, Debug, Default)] @@ -180,6 +189,23 @@ pub fn publish(home: &Path, events: &mut Events<'_>, request: &BranchPublish) -> on_project(home, events, &request.project, StoreOp::Publish(request)) } +/// `branch.remove`: one Branch hidden by its Store marker. The result names +/// the project as the caller gave it, `/`: the Store sees +/// only the project's name within it and echoes that alone. +pub fn remove(home: &Path, events: &mut Events<'_>, request: &BranchRemove) -> Outcome { + let value = on_project( + home, + events, + &request.project, + StoreOp::BranchRemove(request), + )?; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} + /// `branch.diff`: the changes between two Branches. pub fn diff(home: &Path, events: &mut Events<'_>, request: &BranchDiff) -> Outcome { on_project( diff --git a/control/crates/layerfs-store/src/calls/op.rs b/control/crates/layerfs-store/src/calls/op.rs index 26308d38cf..4e43f6cd81 100644 --- a/control/crates/layerfs-store/src/calls/op.rs +++ b/control/crates/layerfs-store/src/calls/op.rs @@ -4,6 +4,7 @@ //! a `store.call` frame to the Store's worker). use super::branch::{ BranchCommits, BranchDiff, BranchFork, BranchGetCommit, BranchMerge, BranchPublish, + BranchRemove, }; use super::commit::CommitDiff; use super::file::{ @@ -62,6 +63,9 @@ pub enum StoreOp<'a> { Statistics, /// Entry counts at one project pin. ProjectStats(&'a super::project::ProjectStats), + /// `project.remove_file`: hide one path from every node at or below its + /// marker frontier. + RemoveFile(&'a super::project::ProjectRemoveFile), /// `project.list_branches`: the Branch `name`, or one page. ListBranches { /// The one Branch it lists. @@ -88,6 +92,8 @@ pub enum StoreOp<'a> { Merge(&'a BranchMerge), /// `branch.publish`. Publish(&'a BranchPublish), + /// `branch.remove`. + BranchRemove(&'a BranchRemove), /// `branch.diff_branches`. DiffBranches(&'a BranchDiff), /// `branch.diff_commits`. @@ -236,6 +242,7 @@ impl StoreOp<'_> { match self { Self::Statistics => "store.stats", Self::ProjectStats(_) => "project.stats", + Self::RemoveFile(_) => "project.remove_file", Self::ListBranches { .. } => "project.list_branches", Self::ListCommits { .. } => "branch.list_commits", Self::BranchCommits(_) => "branch.commits", @@ -244,6 +251,7 @@ impl StoreOp<'_> { Self::Fork(_) => "branch.fork", Self::Merge(_) => "branch.merge", Self::Publish(_) => "branch.publish", + Self::BranchRemove(_) => "branch.remove", Self::DiffBranches(_) => "branch.diff_branches", Self::DiffCommits(_) => "branch.diff_commits", Self::Read(_) => "fs.read", @@ -269,6 +277,7 @@ impl StoreOp<'_> { match self { Self::Statistics => &mut args, Self::ProjectStats(r) => args.at(&r.branch, r.commit.as_deref()), + Self::RemoveFile(r) => args.text("path", &r.path), Self::ListBranches { name, page } => args.opt("name", *name).page(*page), Self::ListCommits { branch, page } => args.text("branch", branch).page(*page), Self::BranchCommits(r) => args @@ -297,6 +306,7 @@ impl StoreOp<'_> { Self::Publish(r) => args .text("branch", &r.branch) .opt("commit", r.commit.as_deref()), + Self::BranchRemove(r) => args.text("branch", &r.branch), Self::DiffBranches(r) => args .text("from", &r.from_branch) .text("to", &r.to_branch) diff --git a/control/crates/layerfs-store/src/calls/project.rs b/control/crates/layerfs-store/src/calls/project.rs index d61b25fb0f..baaeef1ee5 100644 --- a/control/crates/layerfs-store/src/calls/project.rs +++ b/control/crates/layerfs-store/src/calls/project.rs @@ -1,5 +1,6 @@ //! Saved-tree project statistics through the ordinary Store-call route. use super::op::{on_project, StoreOp}; +use layerfs_bridge::control::SdkValue; use layerfs_home::{events::Events, outcome::Outcome}; use std::path::Path; @@ -14,6 +15,15 @@ pub struct ProjectStats { pub commit: Option, } +/// `project.remove_file(project, path)`. +#[derive(Clone, Debug, Default)] +pub struct ProjectRemoveFile { + /// `/`. + pub project: String, + /// The absolute POSIX path. + pub path: String, +} + /// Count one immutable saved tree, excluding the root directory. pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Outcome { on_project( @@ -23,3 +33,15 @@ pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Ou StoreOp::ProjectStats(request), ) } + +/// `project.remove_file`: one path hidden from every node at or below +/// its marker frontier. The result names the project as the caller gave it, +/// `/`. +pub fn remove_file(home: &Path, events: &mut Events<'_>, request: &ProjectRemoveFile) -> Outcome { + let value = on_project(home, events, &request.project, StoreOp::RemoveFile(request))?; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs new file mode 100644 index 0000000000..dab93bac77 --- /dev/null +++ b/control/crates/layerfs-store/src/compact.rs @@ -0,0 +1,150 @@ +//! `store.compact`: one sqlite Store's §7 rebuild, run in this process. +//! +//! The call takes the Store's lease, so a move or remove cannot start, and +//! closes the Store this process holds open: a call still running on it is +//! `InUse`, before any effect. It then opens the Store's own session, runs +//! the §7 (SQLite) or §8 (PostgreSQL) compaction and answers the §11 report. +//! A SQLite Store on another machine is `Unsupported` here: its rebuild runs +//! in that machine's worker. +use crate::identity; +#[cfg(feature = "postgresql")] +use crate::identity::Kind; +use crate::lease; +use crate::open::close; +use crate::open_args::OpenArgs; +use crate::place::{self, Executor}; +#[cfg(feature = "postgresql")] +use crate::postgresql; +use crate::record::{self, StoreRecord, StoreRef}; +use layerfs_bridge::control::SdkRefusal; +use layerfs_home::events::Events; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused, Outcome}; +#[cfg(feature = "postgresql")] +use layerfs_persistence::PostgresHandles; +use layerfs_persistence::{ + CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_store_service::history::{admission, lookup}; +use std::path::Path; + +/// `store.compact(name)`: reclaim one sqlite Store's dead bytes, answering +/// the §11 report. +pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { + let home = home.to_path_buf(); + let (_lease, record) = lease::claim(&home, store)?; + let store_ref = record.store_ref()?.clone(); + if matches!(store_ref, StoreRef::Postgres { .. }) { + return compact_postgresql(&record, &home, store); + } + match place::plan(&home, &store_ref)? { + Executor::Local => {} + Executor::Worker(_) => { + return Err(refused( + "Unsupported", + format!("store.compact of Store {store} runs on its own machine, not this one"), + )) + } + } + let path = paths::store_file(&home, store); + // A cached open in this process must end first; a call running on it is + // `InUse`, before any effect. + close(&path)?; + let args = OpenArgs { + home: Some(home), + store: Some(store.to_owned()), + project: None, + store_path: Some(path.display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let identity = identity::identity(&args)?; + let handles = SqliteHandles::open_writable( + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable), + &identity.binding, + identity.cursor_key, + ) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let receipt = handles + .compact(&CompactRequest { + op_id: admission::op_id(), + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + ..CompactRequest::default() + }) + .map_err(lookup::history)?; + // One serializer for the shared §11 report: the SQLite receipt projects + // into `CompactReport`, and T9's PostgreSQL receipt will do the same. + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// `store.compact` of a PostgreSQL Store in a build without the `postgresql` +/// feature is `Unsupported`. +#[cfg(not(feature = "postgresql"))] +fn compact_postgresql(_record: &StoreRecord, _home: &Path, _store: &str) -> Outcome { + Err(refused( + "Unsupported", + "this LayerFS build has no PostgreSQL engine; build the binding with --features postgresql", + )) +} + +/// The §8 PostgreSQL path: dial the Store's recorded URL, open its schema as +/// this home's authority, and run Phase M/A/B/C. The Store is usable at +/// G + 1 from the swap on; a lost reply is judged by the next call (§9). +#[cfg(feature = "postgresql")] +fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome { + let url = record::url(record)?; + let id = identity::identity(&OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + store_record: Some(record.to_record().clone()), + ..OpenArgs::default() + })?; + let Kind::Postgresql { + schema, + synchronous_commit, + } = &id.kind + else { + return Err(refused( + "Failed", + format!("Store {store} is not a PostgreSQL Store"), + )); + }; + let selection = postgresql::open::selection(&url, schema, synchronous_commit); + let handles = PostgresHandles::open_writable(&selection, &id.binding, id.cursor_key) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let receipt = handles + .compact(&CompactRequest { + op_id: admission::op_id(), + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + ..CompactRequest::default() + }) + .map_err(lookup::history)?; + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// An open refusal before the run: `Busy` is definite, everything else a +/// failure with its own text. +fn open_refusal(error: layerfs_storage::port::PersistenceError) -> SdkRefusal { + use layerfs_storage::port::PersistenceError; + match error { + PersistenceError::Busy => refused("Busy", "the Store is held by another writer"), + PersistenceError::Refused { status } => refused( + "Busy", + format!("the Store refused the compaction: {status}"), + ), + other => failed(other), + } +} diff --git a/control/crates/layerfs-store/src/lib.rs b/control/crates/layerfs-store/src/lib.rs index 97fdd79303..449f106438 100644 --- a/control/crates/layerfs-store/src/lib.rs +++ b/control/crates/layerfs-store/src/lib.rs @@ -14,6 +14,7 @@ pub mod backup; pub mod call; #[doc(hidden)] pub mod calls; +pub mod compact; pub mod diagnostic; #[doc(hidden)] pub mod identity; @@ -26,6 +27,7 @@ pub mod open_args; #[doc(hidden)] pub mod place; pub mod postgresql; +pub mod report; #[doc(hidden)] pub mod project; #[doc(hidden)] diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs new file mode 100644 index 0000000000..a02dc621e6 --- /dev/null +++ b/control/crates/layerfs-store/src/report.rs @@ -0,0 +1,116 @@ +//! The wire rendering of the §11 compaction report and the `store.status` +//! compaction projection (T10). +//! +//! One serializer renders the shared [`CompactReport`], so the SQLite path and +//! T9's PostgreSQL path cannot disagree on a key: every key is snake_case (§10; +//! §11 names are logical), a zero field is named by its wire key, and +//! `residue.reason` appears only when something remains. +use layerfs_bridge::control::SdkValue; +use layerfs_persistence::{CompactReport, CompactStatus}; + +/// The §11 report as `store.compact` returns it: +/// `{kind: "Compacted", path, generation {before, after}, ...}`. +#[must_use] +pub fn compact_report(report: &CompactReport) -> SdkValue { + let int = |value: u64| SdkValue::Int(i64::try_from(value).unwrap_or(i64::MAX)); + let references = report + .kept_waste + .delta_base_references + .iter() + .map(|id| SdkValue::text(layerfs_bridge::sdk_values::hex(id))) + .collect(); + let mut residue = vec![ + ("bytes".to_owned(), int(report.residue.bytes)), + ( + "artifacts".to_owned(), + SdkValue::List( + report + .residue + .artifacts + .iter() + .map(|artifact| SdkValue::text(artifact)) + .collect(), + ), + ), + ]; + if let Some(reason) = &report.residue.reason { + residue.push(("reason".to_owned(), SdkValue::text(reason))); + } + let zero_reasons = report + .zero_reasons + .iter() + .map(|zero| { + SdkValue::map([ + ("field", SdkValue::text(zero.wire_field())), + ("reason", SdkValue::text(zero.reason)), + ]) + }) + .collect(); + SdkValue::Map( + [ + ("kind", SdkValue::text("Compacted")), + ("path", SdkValue::text(report.path.as_str())), + ( + "generation", + SdkValue::map([ + ("before", int(report.generation.before)), + ("after", int(report.generation.after)), + ]), + ), + ("dead_bytes_bound", int(report.dead_bytes_bound)), + ("dead_bytes_refined", int(report.dead_bytes_refined)), + ("logical_discarded", int(report.logical_discarded)), + ("backend_reusable", int(report.backend_reusable)), + ("physical_shrink", int(report.physical_shrink)), + ( + "kept_waste", + SdkValue::map([ + ( + "dead_in_retained_packs_bytes", + int(report.kept_waste.dead_in_retained_packs_bytes), + ), + ( + "delta_bases", + SdkValue::map([ + ("bytes", int(report.kept_waste.delta_bases_bytes)), + ("references", SdkValue::List(references)), + ]), + ), + ("inode_blocks_bytes", int(report.kept_waste.inode_blocks_bytes)), + ]), + ), + ("peak_temp_bytes", int(report.peak_temp_bytes)), + ("residue", SdkValue::Map(residue.into_iter().collect())), + ("estimate", SdkValue::Bool(report.estimate)), + ("zero_reasons", SdkValue::List(zero_reasons)), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The `store.status` projection of a compaction: `{id, name, phase, started, +/// uncertain, state, phases}`. The caller splits the op fields into its `op` +/// and leaves `state`/`phases` beside them (§11). +#[must_use] +pub fn compact_status(status: &CompactStatus) -> SdkValue { + SdkValue::map([ + ("id", SdkValue::text(&status.id)), + ("name", SdkValue::text(status.name)), + ("phase", SdkValue::text(&status.phase)), + ("started", SdkValue::Int(status.started)), + ("uncertain", SdkValue::Bool(status.uncertain)), + ("state", SdkValue::text(status.state)), + ( + "phases", + SdkValue::List( + status + .phases + .iter() + .map(|phase| SdkValue::text(*phase)) + .collect(), + ), + ), + ]) +} diff --git a/control/crates/layerfs-store/src/sqlite/transfer/step.rs b/control/crates/layerfs-store/src/sqlite/transfer/step.rs index 50dca83651..98dc7021d0 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/step.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/step.rs @@ -12,6 +12,11 @@ type Result = std::result::Result; pub enum StoreStep { /// Whether the file is there, its size and the free bytes. Stat, + /// The compaction operation recorded beside the Store, for `store.status`. + /// + /// It reads only `.compact-op`; never the Store database, an + /// artifact or a directory listing. + CompactOp, /// The idle Store's bytes, sha256 and heads. Snapshot, /// A consistent snapshot of the live Store, staged beside it. @@ -68,6 +73,7 @@ impl StoreStep { pub fn name(&self) -> &'static str { match self { Self::Stat => "stat", + Self::CompactOp => "compact-op", Self::Snapshot => "snapshot", Self::Stage => "stage", Self::Unstage => "unstage", @@ -161,6 +167,7 @@ impl StepFrame { layerfs_home::names::StoreName::parse(store)?; let step = match op { "stat" => StoreStep::Stat, + "compact-op" => StoreStep::CompactOp, "snapshot" => StoreStep::Snapshot, "stage" => StoreStep::Stage, "unstage" => StoreStep::Unstage, diff --git a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs index ef250836ef..fdca5f159a 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs @@ -25,6 +25,16 @@ fn with_suffix(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(name) } +/// Whether the process a compaction record names still runs on this machine. +/// The kernel start time and boot id, when recorded, keep a reused id from +/// passing as the owner. +fn owner_alive(op: &layerfs_persistence::CompactOpStatus) -> bool { + let Some(pid) = op.owner_pid else { + return false; + }; + layerfs_home::process::alive_as(pid, op.owner_pid_start, op.owner_pid_boot) +} + fn length(path: &Path) -> Result { match std::fs::metadata(path) { Ok(meta) => Ok(meta.len()), @@ -124,6 +134,12 @@ pub fn run_observed(root: &Path, frame: &StepFrame, progress: &mut dyn FnMut(u64 ), ("incoming_bytes", SdkValue::Int(length(&incoming)? as i64)), ])), + // the compaction record beside the Store only: `store.status` shows + // its op and phase without opening the Store database or an artifact + StoreStep::CompactOp => match layerfs_persistence::compact_op_status(&file) { + None => Ok(SdkValue::Null), + Some(op) => Ok(crate::report::compact_status(&op.status(owner_alive(&op)))), + }, // snapshot: the idle Store's bytes, sha256 and history heads. A Store // nobody holds has no write-ahead log left unless a process crashed; // if a crashed opener left one, checkpoint and close it first. diff --git a/control/crates/layerfs-store/src/status.rs b/control/crates/layerfs-store/src/status.rs index e62157e506..5656f73eb7 100644 --- a/control/crates/layerfs-store/src/status.rs +++ b/control/crates/layerfs-store/src/status.rs @@ -8,31 +8,65 @@ use layerfs_bridge::control::SdkValue; use layerfs_home::outcome::Outcome; use std::path::Path; -/// `store.status`: who holds it, the operation its record holds (`op`, as -/// H6 shows it), its size. It starts no machine: a Store whose machine is -/// not running is `down`, with the reason (O34). It changes nothing (#536 -/// §12 item 3): no lock, and a stale lease is ignored, not removed. +/// `store.status`: who holds it, the operation on it (`op`, as H6 shows it), +/// its size. It starts no machine: a Store whose machine is not running is +/// `down`, with the reason (O34). It changes nothing (#536 §12 item 3): no +/// lock, and a stale lease is ignored, not removed. +/// +/// While a compaction runs, or after a dead owner left one, `op` is that run's +/// projection and `state` is `busy`/`unresolved` (§11); `phases` names the +/// backend's ordered phases. The compaction record is read from +/// `.compact-op` alone: the Store database and its artifacts are never +/// opened for it. pub fn status(home: &Path, name: &str) -> Outcome { let record = record_of(home, name)?; let (holders, _) = lease::read_holders(home, name)?; let mut down = None; let mut more = Vec::new(); let store_ref = record.store_ref()?; + let mut compaction = SdkValue::Null; if let StoreRef::Postgres { .. } = store_ref { more.push(("server", crate::postgresql::connect::status(&record))); + // A PostgreSQL compaction's op lives in the Store's `gc_op`; T9 reads + // it here with its rebuild. This path reads no file. } else { more.push(("machine", SdkValue::text(store_ref.machine()?))); - // the file's size where it lives, starting nothing (P4); a machine - // that is down says why - let file = place::plan(home, store_ref) - .and_then(|mut executor| step_on(home, &mut executor, name, StoreStep::Stat)); + // the file's size where it lives, starting nothing (P4), and the + // compaction record beside it; a machine that is down says why + let mut placed = place::plan(home, store_ref); + let (file, op) = match placed.as_mut() { + Ok(executor) => ( + step_on(home, executor, name, StoreStep::Stat), + step_on(home, executor, name, StoreStep::CompactOp).unwrap_or_default(), + ), + Err(why) => (Err(why.clone()), SdkValue::Null), + }; + compaction = op; let file = file.unwrap_or_else(|why| { down = Some(why.message.clone()); SdkValue::map([("unavailable", SdkValue::Text(why.message))]) }); more.push(("file", file)); } - let state = record.state(down.is_some(), &holders).as_str(); + // A compaction's record is the operation the Store is running, even when + // some other process holds its lease; otherwise the record's own `op` + // (a push, pull or remove) is what `store.status` shows. + let op = if compaction.get("id").is_some() { + let field = |field: &str| compaction.get(field).cloned().unwrap_or_default(); + SdkValue::map([ + ("id", field("id")), + ("name", field("name")), + ("phase", field("phase")), + ("started", field("started")), + ("uncertain", field("uncertain")), + ]) + } else { + record.op().map(|op| op.to_sdk()).unwrap_or_default() + }; + let phases = compaction.get("phases").cloned().unwrap_or_default(); + let state = compaction + .get_text("state") + .unwrap_or_else(|| record.state(down.is_some(), &holders).as_str()); let mut fields = vec![ ("name", SdkValue::text(name)), ("kind", SdkValue::text(store_ref.kind().as_str())), @@ -46,7 +80,8 @@ pub fn status(home: &Path, name: &str) -> Outcome { .collect(), ), ), - ("op", record.op().map(|op| op.to_sdk()).unwrap_or_default()), + ("op", op), + ("phases", phases), ("down_reason", down.map(SdkValue::Text).unwrap_or_default()), ]; fields.extend(more); diff --git a/control/crates/layerfs-store/tests/compact_report_status.rs b/control/crates/layerfs-store/tests/compact_report_status.rs new file mode 100644 index 0000000000..88aa9e03da --- /dev/null +++ b/control/crates/layerfs-store/tests/compact_report_status.rs @@ -0,0 +1,212 @@ +//! T10: the §11 report wire shape and the `store.status` compaction op +//! projection. The report renders snake_case from the shared `CompactReport`; +//! status reads only the `.compact-op` sidecar and never opens the +//! Store database. +use layerfs_bridge::control::SdkValue; +use layerfs_home::home::{paths, private_dir, write_entity}; +use layerfs_home::names::StoreName; +use layerfs_persistence::{ + CompactKeptWaste, CompactPath, CompactReceipt, CompactReport, CompactResidue, CompactZeroReason, +}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t10-report-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + let SdkValue::List(items) = value else { + return Vec::new(); + }; + items + .iter() + .filter_map(|item| match item { + SdkValue::Text(text) => Some(text.clone()), + _ => None, + }) + .collect() +} + +/// A home holding one empty sqlite Store `s` on native. +fn store_home() -> PathBuf { + let home = tempdir("home"); + let record = SdkValue::map([ + ("name", SdkValue::text("s")), + ("kind", SdkValue::text("sqlite")), + ("machine", SdkValue::text("native")), + ]); + write_entity(&paths::store_record(&home, "s"), &record).unwrap(); + private_dir(&paths::store_dir(&home, "s")).unwrap(); + std::fs::write(paths::store_file(&home, "s"), b"").unwrap(); + home +} + +/// A complete `.compact-op` record, as the persistence layer writes it. +fn sidecar(home: &Path, phase: &str, owner: u32) -> PathBuf { + let mut path = paths::store_file(home, "s").into_os_string(); + path.push(".compact-op"); + let path = PathBuf::from(path); + let zeros = "00".repeat(32); + std::fs::write( + &path, + format!( + "version=1\nop_id=00112233445566778899aabbccddeeff\ngeneration=3\n\ +phase={phase}\nstarted_ms=1234\nowner_pid={owner}\nowner_pid_start=\nowner_pid_boot=\n\ +catalog_id=0102030405060708090a0b0c0d0e0f10\nnext_history_serial=5\nnext_pack_id=6\n\ +next_ordinal=7\nmark_serial=4\nmark_pack_id=6\nmark_digest={zeros}\nmark_totals=1,2,3,4\n\ +mark_stamp=9\nhistory_rows=10\nlive_heads_digest={zeros}\n" + ), + ) + .unwrap(); + path +} + +fn receipt() -> CompactReceipt { + CompactReceipt { + path: CompactPath::Rebuild, + generation_before: 3, + generation_after: 4, + dead_bytes_bound: 100, + dead_bytes_refined: 80, + logical_discarded: 40, + backend_reusable: 4096, + physical_shrink: 8192, + kept_waste: CompactKeptWaste { + dead_in_retained_packs_bytes: 5, + delta_bases_bytes: 6, + delta_base_references: vec![[7; 32]], + inode_blocks_bytes: 8, + }, + peak_temp_bytes: 900, + residue: CompactResidue { + bytes: 0, + artifacts: Vec::new(), + reason: None, + }, + estimate: true, + zero_reasons: vec![ + CompactZeroReason { + field: "deadBytesBound", + reason: "no-waste", + }, + CompactZeroReason { + field: "logicalDiscarded", + reason: "no-history", + }, + ], + } +} + +#[test] +fn the_report_wire_is_the_snake_case_contract_shape() { + let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt())); + assert_eq!(text(wire.get("kind").unwrap()), "Compacted"); + assert_eq!(text(wire.get("path").unwrap()), "rebuild"); + let generation = wire.get("generation").unwrap(); + assert_eq!(generation.get_int("before"), Some(3)); + assert_eq!(generation.get_int("after"), Some(4)); + assert_eq!(wire.get_int("dead_bytes_bound"), Some(100)); + assert_eq!(wire.get_int("dead_bytes_refined"), Some(80)); + assert_eq!(wire.get_int("logical_discarded"), Some(40)); + assert_eq!(wire.get_int("backend_reusable"), Some(4096)); + assert_eq!(wire.get_int("physical_shrink"), Some(8192)); + assert_eq!(wire.get_int("peak_temp_bytes"), Some(900)); + assert_eq!(wire.get("estimate"), Some(&SdkValue::Bool(true))); + let waste = wire.get("kept_waste").unwrap(); + assert_eq!(waste.get_int("dead_in_retained_packs_bytes"), Some(5)); + assert_eq!(waste.get_int("inode_blocks_bytes"), Some(8)); + let bases = waste.get("delta_bases").unwrap(); + assert_eq!(bases.get_int("bytes"), Some(6)); + assert_eq!( + texts(bases.get("references").unwrap()), + vec!["0707070707070707070707070707070707070707070707070707070707070707"] + ); + let residue = wire.get("residue").unwrap(); + assert_eq!(residue.get_int("bytes"), Some(0)); + assert!(texts(residue.get("artifacts").unwrap()).is_empty()); + assert!(residue.get("reason").is_none()); + // One reason per zero field, named by its snake_case wire key (§10). + let SdkValue::List(zero) = wire.get("zero_reasons").unwrap() else { + panic!("zero_reasons is a list"); + }; + let fields: Vec<&str> = zero + .iter() + .map(|zero| text(zero.get("field").unwrap())) + .collect(); + assert_eq!(fields, ["dead_bytes_bound", "logical_discarded"]); + assert_eq!( + text(zero[0].get("reason").unwrap()), + "no-waste", + "the closed vocabulary passes through" + ); + assert!(wire.get("zeroReason").is_none(), "no camelCase key"); +} + +#[test] +fn status_projects_the_sidecar_phase_and_state() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + + // No op: the existing idle shape, no phases. + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + + // A live owner is busy and certain; `build` is the sidecar's phase. + let op_path = sidecar(&home, "build", std::process::id()); + let busy = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + assert_eq!(text(op.get("name").unwrap()), "store.compact"); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get_int("started"), Some(1234)); + // The record remains unsettled while it runs; `state` is what tells a + // live owner from a leftover. + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + texts(busy.get("phases").unwrap()), + ["mark", "build", "verify", "install"] + ); + + // A gone owner leaves an uncertain op: unresolved, same phase, same id. + sidecar(&home, "build", 99_999_999); + let leftover = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + + // A settled run removes the record: the idle shape is back. + std::fs::remove_file(op_path).unwrap(); + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/core/docs/issues/project-41/LayerFS-GC-spec.md b/core/docs/issues/project-41/LayerFS-GC-spec.md index 3cd2a9a942..1a8baa36cd 100644 --- a/core/docs/issues/project-41/LayerFS-GC-spec.md +++ b/core/docs/issues/project-41/LayerFS-GC-spec.md @@ -161,10 +161,13 @@ and returns `home_entry_removed: true`. The durable delete intent `{stack_id, deleted_ms, attempt_ms}` lives outside the removed entry (or the entry is **not** renamed; one sidecar file at `/store//delete/.json` with -`{store, project, stack_id, deleted_ms, attempt_ms, state}` is written +`{store, project, stack_id, deleted_ms, attempt_ms, state, op_id}` is written (temp + fsync + rename + dir fsync) after the identity save and before the marker, with `pending|marked|done` updated atomically and unlinked after the -entry removal; the retry +entry removal. `op_id` is additive (an older sidecar without it is still +valid): it records the project op the admit/mark attempt ran under, so the +retry can settle a lost admit/mark reply instead of leaving it uncertain +(§5.1, §10). The retry re-reads the Store by the recorded stack id and compares it with the home record, discarding a mismatch without touching a re-created project's entry; a retired row reuses `deleted_ms` and finishes the entry; a live row with no @@ -289,7 +292,8 @@ caller's record by stack id and allow-lists its retired stack. same position: the marker hides the path and a mount of the Branch that carries it whiteouts, so publish needs no whiteout of its own. A merge that cannot materialize the whiteout is refused `Removed` with - `detail {paths: [...]}`. + `detail {paths: [{path, frontier_serial}]}`, listing every unmaterializable + path as a full element, deduplicated and sorted by normalized path bytes. - A path explicitly re-created after the frontier is not a whiteout: the marker hides only content inherited from nodes at or below the frontier. In a merge of a head at or below the frontier with a head whose path was @@ -317,9 +321,11 @@ caller's record by stack id and allow-lists its retired stack. `NotFound` once compaction collects it, never different bytes. - The result is `{kind: "Removed", project, path, frontier_serial, removed_ms}` or `{kind: "UpToDate", project, path, frontier_serial, - removed_ms}` (the stored values). Every refusal carries one detail shape: - `{paths: [{path, frontier_serial}]}` (one element for read/stat; the - governing maximum frontier and the request path). `file.read`/`file.stat` + removed_ms}` (the stored values). A `Removed` refusal always carries the one + detail shape `{paths: [{path, frontier_serial}]}`: one element for + read/stat (the request path with its governing maximum frontier); a merge + lists every unmaterializable path, sorted by normalized path bytes. + `file.read`/`file.stat` refuse `Removed` with it. `file.list`/`file.glob` omit hidden entries and report one top-level `removed_count` over the queried subtree (a hidden target path is `Removed`; a marked-but-absent path is not counted). @@ -336,11 +342,11 @@ caller's record by stack id and allow-lists its retired stack. hidden at-or-below version (never skips past it), returning only visible versions. `file.blame` stops at the hidden version and attributes the remaining lines at the tip. `file.grep`/`file.blame` omit hidden - hits. `project.export` omits it; a cp-from source is `Removed`; `file.edit` - is `Removed`; `file.mv` from a hidden path is `NotFound` and to a hidden - path is a visible re-creation; `mkdir`/`cp`/`apply` follow the same - construction rule; `file.rm` of a hidden path is `NotFound`; a - post-frontier write is the visible re-creation. + hits. `project.export` omits it; `file.cp` from a hidden source and + `file.edit` are `Removed`; `file.mv`/`file.rm` from a hidden path return the + `NotFound`/`NoSuchPath` wire code, not `Removed`, and `file.mv` to a hidden + path is a visible re-creation; `mkdir`/`apply` follow the same + construction rule; a post-frontier write is the visible re-creation. - The call accepts an absolute POSIX path and converts it to the Store's normalized relative components (strip one leading `/`; `/a` and `a` are the same). It refuses `InvalidInput` for empty, `/`, `//a`, `a//b`, `a/`, `./a`, @@ -973,8 +979,9 @@ reported as `markBytes`; the Phase A table copy itself moves no pack bytes. | Code | Status | |---|---| | `MountsActive`, `Busy`, `Unresolved`, `NotFound`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NoSuchStore`, `Unsupported`, `ReservedName`, `Conflict`, `NameTaken`, `Capacity`, `Exists`, `InvalidInput` | existing (`Unresolved` now used by the four calls) | +| `Unknown` | service/control-plane outcome, not a `HistoryError` code (§4, §4.2): a lost reply or an undeterminable Store listing; detail `{entity: {kind, name}, phase, op}`, with `phase`/`op` `null` when no durable op is known | | `StaleStore` | new; one `HistoryError` variant with an explicit mapping arm, because the `_ => "Failed"` wildcard (`engine/crates/layerfs-store-service/src/history/lookup.rs:54,64`) would hide it | -| `Removed` | new; reading a path removed by `remove_file`; an explicit `HistoryError::Removed` arm in `lookup.rs` before the `_ => "Failed"` wildcard; detail `{paths: [{path, frontier_serial}]}` | +| `Removed` | new; reading a path removed by `remove_file`; an explicit `HistoryError::Removed` arm in `lookup.rs` before the `_ => "Failed"` wildcard; detail always `{paths: [{path, frontier_serial}]}` — a merge lists every unmaterializable path, sorted by path bytes; `file.mv`/`file.rm` from a hidden path are the `NotFound`/`NoSuchPath` wire code, while `file.cp`/`file.edit` return `Removed` | - `project.init` whose name is held returns `NameTaken` (today it returns @@ -1059,8 +1066,10 @@ inodeBlocksBytes}`; `residue` is `{bytes, artifacts: [string], reason?}`; top-level `estimate: bool`; a capped field is `{value, cap}`; `zeroReason` is per zero field with the closed vocabulary `empty-store`, `none-path`, `no-history`, `no-waste`, `postgresql-in-place`; `markBytes` is a #52 metric, -not a result field. The SDK/CLI render every key -snake_case. On the `none` path `deadBytesRefined` equals `deadBytesBound` (no +not a result field. The names in this section (e.g. `deadBytesBound`, +`keptWaste.deadInRetainedPacksBytes`) are logical Rust names only; the wire, +SDK and CLI keys are snake_case (`dead_bytes_bound`, +`kept_waste.dead_in_retained_packs_bytes`), per §10. On the `none` path `deadBytesRefined` equals `deadBytesBound` (no payload sweep runs below the threshold), and the report says so. `physicalShrink` reads the old size before the rename; `backendReusable` reads the SQLite free-page count × page size before it; `logicalDiscarded` is the summed diff --git a/core/docs/issues/project-41/gc-t10-report-status.md b/core/docs/issues/project-41/gc-t10-report-status.md new file mode 100644 index 0000000000..48df05d38d --- /dev/null +++ b/core/docs/issues/project-41/gc-t10-report-status.md @@ -0,0 +1,96 @@ +# T10: the §11 report and `store.status` + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §10/§11. The spec wins; `§N` cites it. No contract is added here. + +## Placement + +- `engine/crates/layerfs-persistence/src/report.rs` — the backend-neutral + `CompactReport` (with `CompactGeneration`, `CompactKeptWaste`, + `CompactResidue`, `CompactZeroReason`), the `CompactReportSource` trait that + projects a backend receipt into it, and `CompactStatus`, the rendered + `store.status` projection of a run. The sidecar reader itself is T12.2's + `compact_op_status`/`CompactOpStatus`; T10 extends that projection with the + run's `started_ms` and owner (`pid`, kernel start, boot). +- `control/crates/layerfs-store/src/report.rs` — the one serializer: + `compact_report` and `compact_status`. Every key is snake_case (§10; §11 + names are logical) and `CompactZeroReason::wire_field` names each zero field + by its rendered key. +- `CompactReceipt` is unchanged: `From<&CompactReceipt>`/`From` + and the trait project it. The control renderer no longer builds a map itself. +- `layerfs_store::sqlite::transfer::step::StoreStep::CompactOp` reads only + `.compact-op`; it travels the same `store.file` frame to a worker, so + a remote SQLite Store answers it without opening the database, scanning the + directory or touching an artifact. + +## Report keys + +| §11 name | rendered key | +|---|---| +| `path` | `path` (`none`/`rebuild`) | +| `generation` | `generation {before, after}` | +| `deadBytesBound` / `deadBytesRefined` | `dead_bytes_bound` / `dead_bytes_refined` | +| `logicalDiscarded` | `logical_discarded` | +| `backendReusable` | `backend_reusable` | +| `physicalShrink` | `physical_shrink` | +| `keptWaste` | `kept_waste {dead_in_retained_packs_bytes, delta_bases {bytes, references}, inode_blocks_bytes}` | +| `peakTempBytes` | `peak_temp_bytes` | +| `residue` | `residue {bytes, artifacts, reason?}` | +| `estimate` | `estimate` | +| `zeroReason` per zero field | `zero_reasons [{field, reason}]` | + +`kind` is `Compacted`. `residue.bytes` is what is **left**, not what was +removed; `clear_artifacts` now adds a size only when a removal left a file, so +a successful settle reports an empty residue of 0 bytes (T10 fixes the T8 +accounting). + +## `store.status` + +While a compaction runs, or after a dead owner left one, the call adds: + +- `op` — `{id, name, phase, started, uncertain}` from `.compact-op` + (SQLite) / the Store's `gc_op` (PostgreSQL, T9); +- `state` — `busy` while the owner lives, `unresolved` once it is gone + (otherwise the existing `idle`/`in_use`/`busy`/`down` state). `uncertain` + is the engine projection's own fact and is true for **every** record that + remains, live or leftover; `state` is what separates the two. Equating + `uncertain == true` with `unresolved` is wrong. +- `phases` — the backend's ordered phases: `mark`, `build`, `verify`, + `install` on SQLite (T9 adds `mark`, `build`, `swap`, `cleanup`). + +No compaction record means `op: null` and `phases: null`, and a transfer +`op` still renders exactly as before. The sidecar now records the owner +`pid`, kernel start time and boot id; the status step compares them with +`layerfs_home::process::alive_as`, as leases do, so a reused pid cannot pass +as a live owner. The run writes `mark`, then `build`, `verify` and `install` +as it enters each phase, so the phase is visible while it runs (AC-51). + +## PostgreSQL hook (T9) + +`CompactReportSource` is the hook: SQLite's `CompactReceipt` implements it, +and T9 implements it for its PostgreSQL receipt. The report struct names no +backend-only field, so PostgreSQL reports its zeros through the same keys +(`backend_reusable = 0`, `dead_bytes_refined == dead_bytes_bound`) without a +wire change. `store.status` reads the Store-side `gc_op` at the same point in +`status.rs`; nothing is read from a file there. + +## Tests + +- `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: + `the_report_projects_every_section_11_field`, + `the_empty_store_report_explains_its_zeros`, + `a_leftover_op_reports_its_phase_and_state` (a real `AfterBuild` failpoint: + phase `build`; `status(true)` is `busy`, `status(false)` is `unresolved`, + both uncertain; ordered phases, then the next run settles the artifact and + the record). +- `control/crates/layerfs-store/tests/compact_report_status.rs`: the report + wire is the snake_case contract shape (including `zero_reasons` and the + `delta_bases.references` hex), and `store.status` projects an idle Store, + a live owner's record and a leftover one (`op`, `state`, `phases`). + +## Gaps + +- T9's PostgreSQL rebuild and its `gc_op` status read are not implemented; + the trait and the status comment are the only PostgreSQL pieces. +- The `{value, cap}` capped-field shape is not produced: the SQLite receipt + has no cap today. diff --git a/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md b/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md new file mode 100644 index 0000000000..c96960b7a7 --- /dev/null +++ b/core/docs/issues/project-41/gc-t12-sqlite-compaction-fixtures.md @@ -0,0 +1,88 @@ +# T12.1: SQLite compaction fixtures and the golden receipt + +> Acceptance fixture for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §11 (report), §12 (acceptance) and §6.4 (live-heads digest). It adds no +> contract: the spec and the matrix win; `§N` and `AC-nn` cite them. + +## What the fixture is + +`engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs` builds one +Store through the public APIs only (`Fixture::put`/`record`/`inode`, the +`HistoryCatalog` methods, `SqliteHandles::compact`). It is the first fixture +that exercises the §7 rebuild with every §11 report component non-trivial: + +| Fixture part | What it pins | +|---|---| +| Two Stacks (one live, one `mark_stack_deleted`) | retained history and a project collected whole | +| A published Branch retired but still named by its retained Layers | `branch_retired` marker survives; name stays taken | +| An unmerged Branch retired and named by nothing | retired Branch row and marker collected | +| Commits with known `statistics.logical` (120/240/360/70/40) | `logicalDiscarded` is the exact history `logical` delta (160) | +| 12 unreferenced whole files, one per pack | wholly dead packs and `deadBytesBound` | +| A depth-2 PREFIX chain `base2 <- base1 <- target` with only `target` a retained root | `keptWaste.deltaBases` references both bases; pack ids survive (AC-17) | +| One retained pack with a reached group and an unreached group | `keptWaste.deadInRetainedPacksBytes`; a 2.0 rebuild keeps the group | +| A pooled inode leaf inside every tree | `keptWaste.inodeBlocksBytes` (all `inode_value_block` rows are kept) | +| Two `file_digest` rows, one root retained and one collected | AC-18 | +| A retired Branch with no head and a fixed-id digest Store | §6.4 golden vector (AC-56, SQLite half) | + +The transcript covers three outcomes: + +- `emptyNone`: an empty Store is `none` with `zeroReason: empty-store`, no + generation change, no artifact. +- `noDeadNone`: a Store whose every pack holds a live object is `none` with + `zeroReason: no-waste`; `deadBytesRefined == deadBytesBound` and the digest + and generation are unchanged. +- `rebuild`: `G -> G+1`; live objects (including the delta target through its + chain) read, dead objects and packs disappear, the retained pack keeps its + unreached group, collected history rows and their markers disappear, a + retained `branch_retired` marker stays, the full doctor passes and the + `live_heads_digest` is unchanged. + +## Defect found by the fixture + +The rebuild report's `residue.bytes` counted the bytes of every artifact the +settle *removed*, not only the ones that remain, so a fully successful rebuild +reported a nonzero residue with `artifacts: []` and no `reason` — contrary to +§11 ("a nonzero residue is explained") and the T8 note ("residue reports +anything that could not be removed"). `clear_artifacts` now adds a file's size +only when its removal failed; the fixture asserts `residue.bytes == 0` and the +golden pins it. `sqlite_compact` still passes. + +## Golden transcript + +`engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json` +is the byte-for-byte golden (`Json::render`, every object sorted by key). It +contains the three receipts, the mark tuple (digest, object count, packs), the +history markers and the fixture identities. A change to any value fails the +test; update it deliberately: + +``` +UPDATE_GOLDEN=1 cargo test -p layerfs-persistence --test sqlite_compact_fixtures +cargo test -p layerfs-persistence --test sqlite_compact_fixtures +``` + +The second run recompiles the embedded golden and checks it. + +## Matrix rows covered (SQLite half) + +| Case | Evidence | +|---|---| +| AC-04 | deterministic fixture: no live object reaches a collected row; closure compared before/after | +| AC-05 | full doctor (includes `diagnostic/scan.rs`) on the compacted Store | +| AC-17 | depth-2 chain copied; `pack_id` preserved on the bases and the target | +| AC-18 | collected digest row gone, retained one kept | +| AC-25 | retained `branch_retired` survives; collected `branch_retired`/`layer_stack_retired` go | +| AC-38 | the successful artifact set is removed and no stray name remains | +| AC-40 | SQLite `live_heads_digest` is identical before and after; a retired head moves it | +| AC-47 | `none` leaves no `.compact`; a successful settle leaves no artifact | +| AC-48 | empty Store (0/0) is `none` with `zeroReason`; the fixture is a `rebuild` arm | +| AC-52 | every §11 field on both paths, with units and sampling point, pinned by the golden | +| AC-56 | fixed-id digest vector with out-of-order ids, a retired Branch and stack, and an absent head (SQLite half) | +| AC-58 | `none` leaves no `.compact`/`.compact-visited`; `rebuild` settles to no artifact | + +Not covered here, by design (T12.2/T12.3): crash failpoints and `kill -9` +(AC-01, AC-03, AC-33, AC-46, AC-47 crash arm, AC-58 mid-run arm, AC-59), the +9%/10%/30% boundary arms and property tests (AC-04 at E3, AC-48 boundary, +AC-57), the PostgreSQL halves (AC-02, AC-28..AC-44, AC-60), admission races +(AC-07, AC-08, AC-49), and the report/`store.status` op projection (AC-51). +The PostgreSQL digest agreement (AC-40) and the shared literal vector (AC-56) +are proven on SQLite only here. diff --git a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md index 3ce29deab2..cc8461befe 100644 --- a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md +++ b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md @@ -113,15 +113,25 @@ the guard refuses direct Store access; it is unrelated to this change. ## Gaps (for T10/T12) -- The `(dev, ino)` `StaleStore` fence (§10) is not implemented in the SQLite - session; install excludes admitted writers by the held lane, not by a - per-transaction inode check. -- `Capacity` mapping is implemented (`SQLITE_FULL`/`TOOBIG`/`NOMEM`, `ENOSPC`, - `EDQUOT`, `EFBIG`) but no capacity/failpoint test asserts it. -- The 9%/10%/30% boundary and the `GroupRows`/`GroupRowsIndexed` and - acquisition layouts are implemented (unit tables conditional, acquisition - rows cleared, `sqlite_sequence` preserved) but exercised only on the - Monolithic layout. The `postgresql` feature is not built here. +- ~~The `(dev, ino)` `StaleStore` fence (§10) is not implemented in the SQLite + session.~~ Implemented in T8.1: `Session` records `(dev, ino)` at open and + refuses the next explicit transaction, read-only snapshot or immutable read + with `StaleStore`; the install fence compares the same pair with + `SealedFacts` before the rename. +- ~~`Capacity` mapping is implemented but no capacity/failpoint test asserts + it.~~ T8.1 adds `CompactFailpoint::CapacityAtBuild` (ENOSPC after the staged + file exists) and unit tests for the `ENOSPC`/`EDQUOT`/`EFBIG` mapping. +- ~~The 9%/10%/30% boundary and the `GroupRows`/`GroupRowsIndexed` and + acquisition layouts are exercised only on the Monolithic layout.~~ T8.1 + exercises 9% (`none`), 10% (rebuild at the boundary) and 30% (rebuild), both + grouped layouts on the `none` and rebuild paths, and the acquisition layout + on open/`none`/rebuild with its working rows cleared. The `postgresql` + feature is not built here. +- A definite pre-install failure now releases its `gc_intent` while the Store + is still open (T8.1): the admission row cannot be cleared after the seal, + and a live intent that outlived its op record refused every later GC call + `BusyOp`. The op record is kept only when the release itself fails, as the + next run's recovery breadcrumb (§9). - Peak temp is the new file plus the visited scratch; journal/WAL sizes after the checkpoint are zero and are not added. - The mark runs before the seal under the same lane (the spec's step 1.5 runs diff --git a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md new file mode 100644 index 0000000000..1d9265c448 --- /dev/null +++ b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md @@ -0,0 +1,115 @@ +# T9: PostgreSQL `store.compact` rebuild + +> Implementation note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §8 (PostgreSQL rebuild), §9 (failure and leftovers) and §11 (report). It adds +> no contract: the spec wins and `§N` cites it. + +## Placement + +`PostgresHandles::compact` is the Store-side primitive +(`engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs`); the +`store.compact` control call dials the Store's recorded URL, opens its schema, +and runs it (`control/crates/layerfs-store/src/compact.rs`). Unlike SQLite the +session is kept; the caller reopens when it needs the new generation pinned +(the diagnostic guard rejects a stale incarnation by design, §10). + +The mark reuses T7.2: `HistoryProvider::mark_gc` reads the §8 tuple as the +mark's first statement and then runs the shared §6.1/§6.2 closure in one write +transaction. The new SQLite-portable statement `gc_tuple_read` is registered +with the other history statements, so both engines share it. + +## Run (PostgreSQL) + +1. **Admission.** `settle_leftover` judges a recorded `gc_op` first (§9): `G` + with a live `idle` owner is terminated with the two-argument + `pg_terminate_backend(pid, backend_start)` and adopted; a live working owner + is `Busy`; `G + 1` resumes Phase C. Then one Store-scoped `gc_intent` + (`compaction`, exclusive) is registered: its session advisory key is held + through Phase C, and a failure releases it. +2. **Phase M — mark.** The first statement reads `(catalog_incarnation, + next_history_serial, next_pack_id, next_ordinal, store_totals.*, + MAX(stamp), history rows)`; the T7.2 closure writes `kept_node` and returns + the object/pack set and live-heads digest. The set is loaded into session + temporary tables (`gc_visited`, `gc_live_pack`). + `deadBytesBound` = stored bytes minus the retained packs' bytes. + `gc_op` (phase `mark`, deterministic scratch name `lfs_gc_`) and the + persisted `gc_dead` membership (dead Commit/Layer/Branch/Stack ids, + `content_signature` slots, `file_digest` roots) are written next; the + `dead_logical` sum of dead history rows is recorded. A `none` decision + truncates `gc_dead`, clears `gc_op`, releases the intent and returns. +3. **Phase A — build** (session advisory key, no table lock). `CREATE SCHEMA` + `lfs_gc_`; `CREATE TABLE ... (LIKE schema.pack INCLUDING STORAGE + INCLUDING COMPRESSION INCLUDING CONSTRAINTS)` plus the explicit primary key + (the copy does not carry it), the same for `object_location`; owner, + tablespace and reloptions are copied from `pg_class`. Retained rows are + copied in pack-id batches, each its own transaction and idempotent by + deleting its own range first; `object_location` uses an `EXISTS` semi-join. + The prefix index, foreign key and `ANALYZE` follow; counts are checked + against the mark. +4. **Phase B — swap** (one short transaction). Every Store base table is + locked `ACCESS EXCLUSIVE NOWAIT`; `55P03` rolls back, drops the scratch + schema in a new transaction and is `Busy`. The mark tuple and live-heads + digest are re-read under the lock; any change is `Conflict` with no writes. + Owner/ACL are applied, the two tables swapped, the `pack_immutable` trigger + recreated, counts and the four totals checked, `dead_canonical/stored/ + metadata` persisted, `catalog_incarnation = G + 1`, the effective + `synchronous_commit` (Store and server, never lowered) set, the standby + predicate re-checked against `synchronous_standby_names` and `pg_stat_replication`, + the scratch schema dropped, and `COMMIT`. +5. **Phase C — cleanup** (batched, after the swap). Only `gc_dead` members are + deleted, child first; `binding_hint_block` is cleared; the acquisition + working tables are emptied. `store_totals` is reduced once by the persisted + `dead_*` totals in the settle transaction and checked against the retained + row sums; `gc_dead`/`gc_op` are cleared and the intent released. A row + created after the mark is not a member, so it survives. + +Report (§11): `path`, `generation {before, after}`, `deadBytesBound` = +`deadBytesRefined`, `logicalDiscarded`, `backendReusable = 0` +(`postgresql-in-place`), `physicalShrink` from `pg_total_relation_size`, +`keptWaste`, `peakTempBytes` (scratch schema), empty `residue`, `estimate` and +the zero reasons; PostgreSQL reports 0 for rebuilt-table reusable space. + +## Tests + +`engine/crates/layerfs-persistence/tests/postgresql_compact.rs` (real +PostgreSQL; `LAYERFS_TEST_POSTGRES_URL`): + +| Test | Covers | +|---|---| +| `an_empty_store_takes_the_none_path` | empty Store → `none`, `empty-store`, no artifact, cleared `gc_op`/`gc_dead`/intent | +| `a_live_store_takes_the_none_path` | live Store → `none`, bound = refined, no shrink | +| `a_wholly_dead_pack_is_rebuilt_away_and_totals_settle` | rebuild, dead pack collected, retained object kept, `G → G+1`, totals invariant, scratch dropped | +| `retained_roots_survive_a_rebuild_that_collects_a_deleted_project` | retained roots survive; a deleted project's history, markers and objects are collected; `logicalDiscarded > 0` | +| `a_lost_swap_reply_is_resumed_by_the_next_compact` | failpoint after the swap: the op stays at phase `swap`, `G+1`; the next compact finishes Phase C and reports the installed outcome | +| `a_stale_owner_backend_is_terminated_before_the_next_round` | a failpoint before Phase A leaves an idle owner; the next session terminates it with `pg_terminate_backend(pid, backend_start)` and adopts the op | +| `the_same_session_resumes_an_uncertain_round` | a failpoint before Phase A leaves the caller's own idle session as the recorded owner; the retry adopts its own op instead of terminating itself | +| `an_exclusive_table_lock_makes_the_swap_busy` | a conflicting `ACCESS EXCLUSIVE` lock at the swap is `Busy` (55P03), and the scratch schema is dropped | + +Unit tests in `compact.rs` pin the standby predicate (`rank < remote_write` or +no names → no wait; names → a streaming sync/quorum standby is required) and +the `synchronous_commit` strength order. + +Result at T9: `cargo test -p layerfs-persistence --features postgresql --test +postgresql_compact` — 8 passed on PostgreSQL 17.12 (Debian, server on +`127.0.0.1:55432`); `--lib` 4 passed; the SQLite suite +(`--test sqlite_compact`) passed (17 when T9 branched, 20 on the rebased main with the T12 fixtures); `sqlite_compact_crash` passed 13. + +## Gaps (not contract changes) + +- **AC-41/AC-42/AC-43/AC-44/AC-60 remain `open`** in the acceptance matrix: + the tests are E1/E2 evidence, not the full #52 matrix. Not covered here: + a reserved-pack publish racing Phase A/C, a mid-cleanup crash failpoint, a + real synchronous standby, and a truly lost `COMMIT` reply from a killed + backend. The swap-phase failpoint proves the resume path, not the server's + reply loss. +- **ACL replay** is best-effort: the owner and a default table ACL are copied; + non-default column/function ACLs are outside this note. +- **Busy holder detail** is the code only (`{entity, phase, op}` and PG + `pid/client_addr/backend_start` are not populated yet); the runbook's lock + query is the manual path. +- The Phase A batch size is a fixed 512 packs, not a 1 GB byte budget; the + scratch copy is `INSERT ... SELECT`, so no pack bytes cross the client. +- `physicalShrink`/`peakTempBytes` are sampled, and a resumed cleanup reports + them as 0 (the op does not persist them). +- `store.status` reading the Store-side `gc_op` is T10/T11; this note only + writes the row/phases it needs. diff --git a/docs/architecture/api/calls/README.md b/docs/architecture/api/calls/README.md index dc953392b0..d17fa5cc97 100644 --- a/docs/architecture/api/calls/README.md +++ b/docs/architecture/api/calls/README.md @@ -118,8 +118,14 @@ with one line in `/trace/calls.jsonl`, except the `logs` calls and with | `tag` | a top-level `log_tag` argument (taken out before the check; no entry declares it), else `LAYERFS_LOGS_TAG`; at most 128 bytes | Values are cut to 200 characters. Content, JSON, and the `Text` arguments -`command`, `message`, `old`, `new`, `password` and `url` are logged only as -`` (`REDACTED`). The line is written after the outcome and never +`command`, `message`, `old`, `new`, `password`, `url` and every path argument +(`path`, `left_path`, `right_path`, `from_path`, `to_path`, `path_prefix`) +are logged only as `` (`REDACTED`), so a `project.remove_file` or +`file.*` path never enters the log (#41 §4.3). A path counts its normalized +form, one leading `/` stripped (`/a` and `a` are both `<1 bytes>`); a glob or +grep `pattern` is not a path and counts its own bytes. A refusal contributes +only its code: its message and `detail` (which may echo the path, as `Removed` +does) are never logged. The line is written after the outcome and never changes it. Reading and clearing it are the [`logs`](../sdk/logs.md) calls. ## Invariants diff --git a/docs/architecture/control/home/README.md b/docs/architecture/control/home/README.md index 9ed4321e45..8c7ed8b17e 100644 --- a/docs/architecture/control/home/README.md +++ b/docs/architecture/control/home/README.md @@ -155,8 +155,12 @@ already holds as it is. | `tag` | per-call `log_tag`, else `LAYERFS_LOGS_TAG`; at most 128 bytes, no control characters, else dropped | Values are cut to 200 characters. Content and Json arguments and the Text -arguments `command`, `message`, `old`, `new`, `password` and `url` are -logged as ``. +arguments `command`, `message`, `old`, `new`, `password`, `url` and every +path argument (`path`, `left_path`, `right_path`, `from_path`, `to_path`, +`path_prefix`) are logged as `` of the normalized form, one leading +`/` stripped (`/a` and `a` are both `<1 bytes>`); a glob or grep `pattern` +counts its own bytes. A refusal contributes only its code: its message and +`detail` are never logged. No line is written for a running or killed call, worker requests, a call's internal parts, commands outside the call table (`help`, diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql new file mode 100644 index 0000000000..5c235efcef --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_tuple_read.sql @@ -0,0 +1,6 @@ +SELECT h.catalog_incarnation,h.next_history_serial,p.next_pack_id,p.next_ordinal, + t.logical,t.canonical,t.stored,t.metadata, + (SELECT COALESCE(MAX(stamp),0) FROM content_signature), + (SELECT COUNT(*) FROM "commit")+(SELECT COUNT(*) FROM layer) + FROM history_meta h, store_policy p, store_totals t + WHERE h.id=1 AND p.id=1 AND t.id=1 diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs new file mode 100644 index 0000000000..a359f6cd6f --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs @@ -0,0 +1,1413 @@ +//! §8 PostgreSQL Store compaction: Phase M mark, Phase A build, Phase B swap +//! and Phase C cleanup. +//! +//! One call takes the T6 Store-scoped exclusive intent (a session advisory +//! key held through cleanup), records the §8 mark tuple as its first +//! statement, runs the T7.2 mark, and decides from the wholly dead pack +//! bytes. A `rebuild` copies only retained `pack`/`object_location` rows into +//! a scratch schema, swaps the two tables in one short transaction at +//! `G -> G + 1`, and deletes only the ids the mark persisted in `gc_dead`. +//! The op row in `gc_op` names the scratch schema before it is created, so a +//! crash (or a lost COMMIT reply) is judged by the next `store.compact`: +//! generation `G` removes the artifact, `G + 1` resumes cleanup. + +use super::session::{ + connection::Session, + transaction::{Mode, Transaction}, +}; +use crate::backend::records::Param; +use crate::backend::sqlite::compact::{ + CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, + CompactResidue, CompactZeroReason, +}; +use crate::history::GcTuple; +use crate::{MarkResult, PostgresHandles}; +use layerfs_history::error::{HistoryError, HistoryResult}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::HistoryCatalog; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// The §3 rebuild threshold: rebuild when dead * 10 >= stored. +const THRESHOLD: u64 = 10; +/// `lfs_gc_`: the deterministic scratch schema name. +const SCRATCH_PREFIX: &str = "lfs_gc_"; + +/// The three wholly dead domains the swap records before COMMIT. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct DeadTotals { + canonical: i64, + stored: i64, + metadata: i64, + logical: i64, +} + +/// One compaction op row, as the next call judges it. +#[derive(Clone, Debug)] +struct OpRow { + generation: i64, + artifact: Option, + dead: DeadTotals, + totals_applied: bool, + pid: Option, + backend_start: Option, +} + +/// What Phase A leaves for Phase B. +struct Built { + scratch: String, + scratch_bytes: u64, + dead: DeadTotals, +} + +/// What the swap records for the report and Phase C. +struct Swapped { + physical_before: u64, + physical_after: u64, + peak_temp: u64, + dead: DeadTotals, +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn scratch_name(op_id: &[u8; 16]) -> String { + format!("{SCRATCH_PREFIX}{}", hex(op_id)) +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// One write transaction on the Store session; scratch/DDL work never waits +/// for the publication lane (the session advisory key is the exclusion). +fn stage( + session: &Arc, + body: impl FnOnce(&Transaction<'_>) -> HistoryResult, +) -> HistoryResult { + session.run_as(Mode::Stage, body) +} + +fn scalar(tx: &Transaction<'_>, sql: &str, params: Vec) -> HistoryResult { + let rows = tx.query(sql, params)?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction scalar")); + } + rows[0].get(0).map_err(Into::into) +} + +fn scalar_session(session: &Arc, sql: &str, params: Vec) -> HistoryResult { + stage(session, |tx| scalar(tx, sql, params)) +} + +fn read_generation(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!("SELECT catalog_incarnation FROM {schema}.history_meta WHERE id=1"), + Vec::new(), + ) +} + +fn read_op(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT generation,artifact,dead_canonical,dead_stored,dead_metadata,dead_logical,totals_applied,pid,backend_start FROM {schema}.gc_op WHERE id=1" + ), + Vec::new(), + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + Ok(Some(OpRow { + generation: row.get(0)?, + artifact: row.get(1)?, + dead: DeadTotals { + canonical: row.get(2)?, + stored: row.get(3)?, + metadata: row.get(4)?, + logical: row.get(5)?, + }, + totals_applied: row.get::(6)? != 0, + pid: row.get(7)?, + backend_start: row.get(8)?, + })) + }) +} + +fn set_phase(session: &Arc, schema: &str, phase: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.query( + &format!("UPDATE {schema}.gc_op SET phase=$1 WHERE id=1"), + vec![Param::Text(phase.to_owned())], + )?; + Ok(()) + }) +} + +fn stop( + request: &CompactRequest, + point: CompactFailpoint, + session: &Arc, + schema: &str, + phase: &str, +) -> HistoryResult<()> { + if request.failpoint != Some(point) { + return Ok(()); + } + set_phase(session, schema, phase)?; + Err(HistoryError::UnknownOutcome) +} + +/// The set of live object ids and live pack ids, as session temporary tables +/// the Phase A copy and the dead-set query read. +fn load_visited(session: &Arc, schema: &str, mark: &MarkResult) -> HistoryResult<()> { + let objects: Vec> = mark + .objects() + .objects() + .iter() + .map(|id| id.as_bytes().to_vec()) + .collect(); + let packs: Vec = mark.objects().packs().to_vec(); + stage(session, |tx| { + tx.batch( + "DROP TABLE IF EXISTS pg_temp.gc_visited;\ + DROP TABLE IF EXISTS pg_temp.gc_live_pack;\ + CREATE TEMP TABLE pg_temp.gc_visited(object_id bytea PRIMARY KEY) ON COMMIT PRESERVE ROWS;\ + CREATE TEMP TABLE pg_temp.gc_live_pack(pack_id bigint PRIMARY KEY) ON COMMIT PRESERVE ROWS;", + )?; + let bytes = objects.iter().map(Vec::len).sum::() as u64; + tx.typed( + "INSERT INTO pg_temp.gc_visited(object_id) SELECT unnest($1::bytea[])", + &[&objects], + bytes, + )?; + tx.typed( + "INSERT INTO pg_temp.gc_live_pack(pack_id) SELECT unnest($1::bigint[])", + &[&packs], + 8 * packs.len() as u64, + )?; + let _ = schema; + Ok(()) + }) +} + +/// Phase M: write `gc_op` (phase mark) and the persisted `gc_dead` set in +/// batches. The set is a membership, not a predicate, so a row created after +/// the mark is never collected. +fn persist_mark( + session: &Arc, + schema: &str, + mark: &MarkResult, + tuple: &GcTuple, + scratch: &str, + request: &CompactRequest, +) -> HistoryResult { + let digest = mark.live_heads_digest().to_vec(); + stage(session, |tx| { + tx.query(&format!("DELETE FROM {schema}.gc_dead"), Vec::new())?; + tx.batch(&format!( + "INSERT INTO {schema}.gc_dead(kind,id) \ + SELECT 'commit',c.commit_id FROM {schema}.\"commit\" c \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='commit' AND k.node_id=c.commit_id) \ + UNION ALL \ + SELECT 'layer',l.layer_id FROM {schema}.layer l \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='layer' AND k.node_id=l.layer_id) \ + UNION ALL \ + SELECT 'branch',b.branch_id FROM {schema}.branch b \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='branch' AND k.node_id=b.branch_id) \ + UNION ALL \ + SELECT 'stack',s.layer_stack_id FROM {schema}.layer_stack s \ + WHERE NOT EXISTS (SELECT 1 FROM {schema}.kept_node k WHERE k.kind='stack' AND k.node_id=s.layer_stack_id) \ + UNION ALL \ + SELECT 'signature',int8send(cs.slot) FROM {schema}.content_signature cs \ + WHERE NOT EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=cs.object_id) \ + UNION ALL \ + SELECT 'digest',fd.digest FROM {schema}.file_digest fd \ + WHERE NOT EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=fd.root)" + ))?; + let logical = scalar( + tx, + &format!( + "SELECT COALESCE((SELECT SUM(c.logical)::bigint FROM {schema}.\"commit\" c \ + WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='commit' AND d.id=c.commit_id)),0) \ + + COALESCE((SELECT SUM(l.logical)::bigint FROM {schema}.layer l \ + WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='layer' AND d.id=l.layer_id)),0)" + ), + Vec::new(), + )?; + let started = now_ms(); + tx.query( + &format!( + "INSERT INTO {schema}.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,mark_totals,mark_stamp,started_ms,dead_canonical,dead_stored,dead_metadata,dead_logical,totals_applied,artifact,pid,backend_start,incarnation) \ + VALUES (1,$1,'mark',$2,$3,$4,$5,int8send($6)||int8send($7)||int8send($8)||int8send($9),$10,$11,0,0,0,$12,FALSE,$2,pg_backend_pid(),(SELECT backend_start::text FROM pg_stat_activity WHERE pid=pg_backend_pid()),$1) \ + ON CONFLICT (id) DO UPDATE SET generation=EXCLUDED.generation,phase=EXCLUDED.phase,scratch_schema=EXCLUDED.scratch_schema,mark_serial=EXCLUDED.mark_serial,mark_pack_id=EXCLUDED.mark_pack_id,mark_digest=EXCLUDED.mark_digest,mark_totals=EXCLUDED.mark_totals,mark_stamp=EXCLUDED.mark_stamp,started_ms=EXCLUDED.started_ms,dead_canonical=0,dead_stored=0,dead_metadata=0,dead_logical=EXCLUDED.dead_logical,totals_applied=FALSE,artifact=EXCLUDED.artifact,pid=EXCLUDED.pid,backend_start=EXCLUDED.backend_start,incarnation=EXCLUDED.incarnation" + ), + vec![ + Param::I64(tuple.generation), + Param::Text(scratch.to_owned()), + Param::I64(tuple.next_history_serial), + Param::I64(tuple.next_pack_id), + Param::Bytes(digest), + Param::I64(tuple.logical), + Param::I64(tuple.canonical), + Param::I64(tuple.stored), + Param::I64(tuple.metadata), + Param::I64(tuple.stamp), + Param::I64(started), + Param::I64(logical), + ], + )?; + let _ = request; + Ok(DeadTotals { + logical, + ..DeadTotals::default() + }) + }) +} + +fn clear_round(session: &Arc, schema: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_dead; DELETE FROM {schema}.gc_op;" + ))?; + Ok(()) + }) +} + +/// The stored bytes of every pack the mark retained, in the current Store. +fn live_stored(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + ) +} + +fn total_stored(session: &Arc, schema: &str) -> HistoryResult { + scalar_session( + session, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack"), + Vec::new(), + ) +} + +fn retained_totals(session: &Arc, schema: &str) -> HistoryResult<(i64, i64, i64)> { + stage(session, |tx| { + let canonical = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(l.canonical_length)::bigint,0) FROM {schema}.object_location l \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=l.object_id)" + ), + Vec::new(), + )?; + let stored = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + )?; + let metadata = scalar( + tx, + &format!( + "SELECT COALESCE(SUM(p.length)::bigint,0) FROM {schema}.pack p \ + WHERE p.domain=0 AND EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + Vec::new(), + )?; + Ok((canonical, stored, metadata)) + }) +} + +fn relation_bytes(session: &Arc, schema: &str, names: &[&str]) -> HistoryResult { + let list = names + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(","); + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT COALESCE(SUM(pg_total_relation_size(c.oid))::bigint,0) FROM pg_class c \ + JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname IN ({list})" + ), + vec![Param::Text(schema.to_owned())], + )?; + let value: i64 = rows[0].get(0)?; + Ok(u64::try_from(value).unwrap_or(0)) + }) +} + +fn table_storage( + tx: &Transaction<'_>, + schema: &str, + table: &str, +) -> HistoryResult<(String, Vec)> { + { + let rows = tx.query( + "SELECT COALESCE((SELECT t.spcname FROM pg_tablespace t WHERE t.oid=c.reltablespace),''), \ + COALESCE(array_to_string(c.reloptions,','),'') \ + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname=$2", + vec![Param::Text(schema.to_owned()), Param::Text(table.to_owned())], + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction source table")); + } + let tablespace: String = rows[0].get(0)?; + let reloptions: String = rows[0].get(1)?; + let options = reloptions + .split(',') + .filter(|option| !option.is_empty()) + .map(str::to_owned) + .collect(); + Ok((tablespace, options)) + } +} + +fn create_scratch(session: &Arc, schema: &str, scratch: &str) -> HistoryResult<()> { + let exists = scalar_session( + session, + "SELECT count(*) FROM pg_namespace WHERE nspname=$1", + vec![Param::Text(scratch.to_owned())], + )?; + if exists != 0 { + return Err(HistoryError::Exists("compaction scratch schema")); + } + stage(session, |tx| { + tx.batch(&format!("CREATE SCHEMA {scratch}"))?; + for table in ["pack", "object_location"] { + let (tablespace, options) = table_storage(tx, schema, table)?; + let mut sql = format!( + "CREATE TABLE {scratch}.{table} (LIKE {schema}.{table} INCLUDING STORAGE INCLUDING COMPRESSION INCLUDING CONSTRAINTS)" + ); + if !options.is_empty() { + sql.push_str(&format!(" WITH ({})", options.join(", "))); + } + if !tablespace.is_empty() { + sql.push_str(&format!(" TABLESPACE {}", quote_ident(&tablespace))); + } + tx.batch(&sql)?; + } + tx.batch(&format!( + "ALTER TABLE {scratch}.pack ADD PRIMARY KEY (pack_id)" + ))?; + Ok(()) + }) +} + +/// Phase A: copy retained rows in pack-id batches, each its own transaction; +/// a retried batch first deletes its own range, so the copy is idempotent. +fn build( + session: &Arc, + schema: &str, + scratch: &str, + mark: &MarkResult, + tuple: &GcTuple, +) -> HistoryResult { + create_scratch(session, schema, scratch)?; + let packs: Vec = mark.objects().packs().to_vec(); + let chunk = 512usize; + for batch in packs.chunks(chunk) { + let lo = batch[0]; + let hi = batch[batch.len() - 1]; + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {scratch}.pack WHERE pack_id BETWEEN {lo} AND {hi};\ + DELETE FROM {scratch}.object_location WHERE pack_id BETWEEN {lo} AND {hi};" + ))?; + tx.query( + &format!( + "INSERT INTO {scratch}.pack SELECT p.* FROM {schema}.pack p \ + WHERE p.pack_id BETWEEN $1 AND $2 \ + AND EXISTS (SELECT 1 FROM pg_temp.gc_live_pack v WHERE v.pack_id=p.pack_id)" + ), + vec![Param::I64(lo), Param::I64(hi)], + )?; + tx.query( + &format!( + "INSERT INTO {scratch}.object_location SELECT l.* FROM {schema}.object_location l \ + WHERE l.pack_id BETWEEN $1 AND $2 \ + AND EXISTS (SELECT 1 FROM pg_temp.gc_visited v WHERE v.object_id=l.object_id)" + ), + vec![Param::I64(lo), Param::I64(hi)], + )?; + Ok(()) + })?; + } + stage(session, |tx| { + tx.batch(&format!( + "CREATE INDEX object_location_prefix ON {scratch}.object_location \ + ((('x' || encode(substring(object_id FROM 1 FOR 8), 'hex'))::bit(64)::bigint));\ + ALTER TABLE {scratch}.object_location \ + ADD FOREIGN KEY (pack_id) REFERENCES {scratch}.pack(pack_id);\ + ANALYZE {scratch}.pack; ANALYZE {scratch}.object_location;" + ))?; + Ok(()) + })?; + let (objects, distinct, pack_rows) = stage(session, |tx| { + let objects = scalar( + tx, + &format!("SELECT count(*) FROM {scratch}.object_location"), + Vec::new(), + )?; + let distinct = scalar( + tx, + &format!("SELECT count(DISTINCT object_id) FROM {scratch}.object_location"), + Vec::new(), + )?; + let pack_rows = scalar( + tx, + &format!("SELECT count(*) FROM {scratch}.pack"), + Vec::new(), + )?; + Ok((objects, distinct, pack_rows)) + })?; + if distinct != mark.objects().len() as i64 || objects != distinct { + return Err(HistoryError::Integrity("compaction object count")); + } + if pack_rows != mark.objects().packs().len() as i64 { + return Err(HistoryError::Integrity("compaction pack count")); + } + let (canonical, stored, metadata) = retained_totals(session, schema)?; + let dead = DeadTotals { + canonical: tuple.canonical - canonical, + stored: tuple.stored - stored, + metadata: tuple.metadata - metadata, + logical: 0, + }; + let scratch_bytes = relation_bytes(session, scratch, &["pack", "object_location"])?; + Ok(Built { + scratch: scratch.to_owned(), + scratch_bytes, + dead, + }) +} + +/// The §6.4 live-heads digest, computed inside one transaction (the mark's +/// SQL is the same vector). +fn digest(tx: &Transaction<'_>) -> HistoryResult<[u8; 32]> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let stacks = tx.query( + "SELECT s.layer_stack_id, s.head_layer_id FROM layer_stack s \ + WHERE NOT EXISTS (SELECT 1 FROM layer_stack_retired r WHERE r.layer_stack_id=s.layer_stack_id) \ + ORDER BY s.layer_stack_id", + Vec::new(), + )?; + for row in &stacks { + let id: Vec = row.get(0)?; + let head: Vec = row.get(1)?; + if id.len() != 17 || head.len() != 33 { + return Err(HistoryError::Integrity("live-heads stack width")); + } + hasher.update(&id); + hasher.update(&head); + } + let branches = tx.query( + "SELECT b.branch_id, b.head_commit_id, b.base_layer_id FROM branch b \ + WHERE NOT EXISTS (SELECT 1 FROM branch_retired r WHERE r.branch_id=b.branch_id) \ + AND NOT EXISTS (SELECT 1 FROM layer_stack_retired s WHERE s.layer_stack_id=b.layer_stack_id) \ + ORDER BY b.branch_id", + Vec::new(), + )?; + for row in &branches { + let id: Vec = row.get(0)?; + let head: Option> = row.get(1)?; + let base: Vec = row.get(2)?; + if id.len() != 17 || base.len() != 33 { + return Err(HistoryError::Integrity("live-heads branch width")); + } + hasher.update(&id); + match head { + Some(head) if head.len() == 33 => hasher.update(&head), + Some(_) => return Err(HistoryError::Integrity("live-heads commit width")), + None => hasher.update(&[0u8; 33]), + }; + hasher.update(&base); + } + Ok(*hasher.finalize().as_bytes()) +} + +/// The strength order of §8 step 10: off < local < remote_write < on < +/// remote_apply. +fn sync_rank(setting: &str) -> u8 { + match setting { + "local" => 1, + "remote_write" => 2, + "on" => 3, + "remote_apply" => 4, + _ => 0, + } +} + +fn sync_name(rank: u8) -> &'static str { + match rank { + 1 => "local", + 2 => "remote_write", + 3 => "on", + 4 => "remote_apply", + _ => "off", + } +} + +/// §8 step 10: the swap waits iff the effective level is at least +/// `remote_write` and `synchronous_standby_names` is non-empty; then a named +/// standby must be streaming with `sync_state` sync or quorum. +fn standby_satisfied(rank: u8, names: &str, streaming: i64) -> bool { + if rank < 2 || names.trim().is_empty() { + return true; + } + streaming > 0 +} + +/// The effective `synchronous_commit` and whether the standby predicate +/// applies, read from the server and the Store's frozen setting. +fn effective_sync_rank(session: &Arc) -> HistoryResult { + stage(session, |tx| { + let rows = tx.query("SELECT current_setting('synchronous_commit')", Vec::new())?; + let server: String = rows[0].get(0)?; + let stored = session.settings.synchronous_commit.setting(); + Ok(sync_rank(&server).max(sync_rank(stored))) + }) +} + +/// §8 step 10: the standby predicate checked before Phase M. +fn standby_precheck(session: &Arc, rank: u8) -> HistoryResult<()> { + stage(session, |tx| { + let rows = tx.query( + "SELECT current_setting('synchronous_standby_names')", + Vec::new(), + )?; + let names: String = rows[0].get(0)?; + standby_check(tx, rank, &names) + }) +} + +fn standby_check(tx: &Transaction<'_>, rank: u8, names: &str) -> HistoryResult<()> { + if rank < 2 || names.trim().is_empty() { + return Ok(()); + } + let streaming = scalar( + tx, + "SELECT count(*) FROM pg_stat_replication WHERE state='streaming' AND sync_state IN ('sync','quorum')", + Vec::new(), + )?; + if !standby_satisfied(rank, names, streaming) { + return Err(HistoryError::Busy); + } + Ok(()) +} + +/// The recorded owner backend's state when it is still alive: `idle` means it +/// is not executing the op, so §9's dead-owner rule applies; any other state +/// is a live owner and stays `Busy`. +fn current_backend_pid(session: &Arc) -> HistoryResult { + scalar_session(session, "SELECT pg_backend_pid()", Vec::new()) +} + +fn owner_state(session: &Arc, op: &OpRow) -> HistoryResult> { + let (Some(pid), Some(start)) = (op.pid, op.backend_start.as_deref()) else { + return Ok(None); + }; + stage(session, |tx| { + let rows = tx.query( + "SELECT state::text FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2", + vec![Param::I64(pid), Param::Text(start.to_owned())], + )?; + rows.first() + .map(|row| row.get(0)) + .transpose() + .map_err(Into::into) + }) +} + +/// §12 PG privileges and reply: the two-argument `pg_terminate_backend(pid, +/// backend_start)` proof, scoped to this database and never the caller. Used +/// only for a recorded owner the server reports `idle` (it cannot finish the +/// op); a live owner is `Busy`, never killed. +fn terminate_owner(session: &Arc, op: &OpRow) -> HistoryResult { + let (Some(pid), Some(start)) = (op.pid, op.backend_start.as_deref()) else { + return Ok(false); + }; + stage(session, |tx| { + let rows = tx.query( + "SELECT pg_terminate_backend(a.pid, 30000) FROM pg_stat_activity a \ + WHERE a.pid=$1 AND a.backend_start::text=$2 AND a.datname=current_database() \ + AND a.pid<>pg_backend_pid()", + vec![Param::I64(pid), Param::Text(start.to_owned())], + )?; + let terminated: Option = rows.first().map(|row| row.get(0)).transpose()?; + Ok(terminated.unwrap_or(0) != 0) + }) +} + +fn drop_scratch(session: &Arc, scratch: &str) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!("DROP SCHEMA IF EXISTS {scratch} CASCADE"))?; + Ok(()) + }) +} + +fn compaction_intent(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + &format!( + "SELECT op_id FROM {schema}.gc_intent WHERE kind='compaction' ORDER BY updated_ms LIMIT 1" + ), + Vec::new(), + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let bytes: Vec = row.get(0)?; + let op: [u8; 16] = bytes + .as_slice() + .try_into() + .map_err(|_| HistoryError::Integrity("compaction intent op"))?; + Ok(Some(op)) + }) +} + +/// §9: judge a recorded leftover. `G` removes the artifact and clears the +/// op; `G + 1` finishes Phase C and answers the installed outcome. `None` +/// means there was nothing to resume. +fn settle_leftover( + handles: &PostgresHandles, + session: &Arc, + schema: &str, +) -> HistoryResult> { + let Some(op) = read_op(session, schema)? else { + return Ok(None); + }; + let current = read_generation(session, schema)?; + if current == op.generation { + // The caller's own session is the recorded owner when it retries its + // own uncertain round: adopt it without terminating itself. + if op.pid != Some(current_backend_pid(session)?) { + if let Some(state) = owner_state(session, &op)? { + if state != "idle" { + return Err(HistoryError::Busy); + } + // The backend is alive but not executing the op: terminate it + // with the (pid, backend_start) pair (§12), then adopt it as a + // dead owner. + if !terminate_owner(session, &op)? { + return Err(HistoryError::Busy); + } + } + } + if let Some(artifact) = &op.artifact { + drop_scratch(session, artifact)?; + } + clear_round(session, schema)?; + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_intent WHERE kind='compaction'" + ))?; + Ok(()) + })?; + return Ok(None); + } + if current != op.generation + 1 { + return Err(HistoryError::Integrity("compaction op generation")); + } + let op_id = compaction_intent(session, schema)?; + cleanup(handles, session, schema, op.dead, op.totals_applied, op_id)?; + Ok(Some(report( + CompactPath::Rebuild, + u64::try_from(op.generation).unwrap_or(0), + u64::try_from(current).unwrap_or(0), + 0, + 0, + u64::try_from(op.dead.logical).unwrap_or(0), + 0, + 0, + CompactKeptWaste::default(), + 0, + CompactResidue::default(), + 0, + 0, + ))) +} + +/// Phase B: lock every base table, re-check the mark tuple, swap the two +/// payload tables, bump the generation and drop the scratch schema in one +/// short transaction. `55P03` is `Busy`; any tuple change is `Conflict` with +/// no writes. +fn swap( + _handles: &PostgresHandles, + session: &Arc, + schema: &str, + tuple: &GcTuple, + mark: &MarkResult, + built: &Built, +) -> HistoryResult { + let physical_before = relation_bytes(session, schema, &["pack", "object_location"])?; + let rank = effective_sync_rank(session)?; + let tables = tables_of(session, schema)?; + let locked = tables + .iter() + .map(|table| format!("{schema}.{}", quote_ident(table))) + .collect::>() + .join(", "); + let scratch = built.scratch.clone(); + let result = stage(session, |tx| -> HistoryResult { + tx.batch(&format!( + "SET LOCAL synchronous_commit TO {}", + sync_name(rank) + ))?; + tx.batch(&format!( + "LOCK TABLE {locked} IN ACCESS EXCLUSIVE MODE NOWAIT" + ))?; + // The tuple the mark recorded, re-read under the lock. Any publication + // after the mark moved one of these, so nothing is written. + let now = read_tuple(tx, schema)?; + if now != *tuple { + return Err(HistoryError::Conflict("Store moved after the mark")); + } + if digest(tx)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict("live heads moved after the mark")); + } + // Copy the owner (and a default ACL, when one exists) before the swap. + copy_owner(tx, schema, &scratch, "pack")?; + copy_owner(tx, schema, &scratch, "object_location")?; + tx.batch(&format!( + "DROP TABLE {schema}.object_location;\ + DROP TABLE {schema}.pack;\ + ALTER TABLE {scratch}.object_location SET SCHEMA {schema};\ + ALTER TABLE {scratch}.pack SET SCHEMA {schema};\ + CREATE TRIGGER pack_immutable BEFORE UPDATE OR DELETE ON {schema}.pack \ + FOR EACH ROW EXECUTE FUNCTION {schema}.pack_immutable();" + ))?; + verify_swap(tx, schema, mark, tuple, &built.dead)?; + let applied = scalar( + tx, + &format!("SELECT totals_applied FROM {schema}.gc_op WHERE id=1"), + Vec::new(), + )?; + if applied != 0 { + return Err(HistoryError::Integrity("compaction totals already applied")); + } + tx.query( + &format!( + "UPDATE {schema}.gc_op SET phase='swap',dead_canonical=$1,dead_stored=$2,dead_metadata=$3 WHERE id=1" + ), + vec![ + Param::I64(built.dead.canonical), + Param::I64(built.dead.stored), + Param::I64(built.dead.metadata), + ], + )?; + tx.query( + &format!( + "UPDATE {schema}.history_meta SET catalog_incarnation={} WHERE id=1", + tuple.generation + 1 + ), + Vec::new(), + )?; + tx.batch(&format!("DROP SCHEMA {scratch} CASCADE"))?; + let names: String = tx + .query( + "SELECT current_setting('synchronous_standby_names')", + Vec::new(), + )? + .first() + .ok_or(HistoryError::Integrity("standby names"))? + .get(0)?; + standby_check(tx, rank, &names)?; + Ok(Swapped { + physical_before, + physical_after: 0, + peak_temp: built.scratch_bytes, + dead: built.dead, + }) + }); + let result = match result { + Ok(swapped) => swapped, + Err(HistoryError::Busy) => { + // §8 step 5: roll back first, then drop the scratch schema in a + // new transaction, and report Busy with the holder detail. + let _ = drop_scratch(session, &scratch); + return Err(HistoryError::Busy); + } + Err(other) => return Err(other), + }; + let physical_after = relation_bytes(session, schema, &["pack", "object_location"])?; + Ok(Swapped { + physical_after, + ..result + }) +} + +fn tables_of(session: &Arc, schema: &str) -> HistoryResult> { + stage(session, |tx| { + let rows = tx.query( + "SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relkind='r' ORDER BY c.relname", + vec![Param::Text(schema.to_owned())], + )?; + rows.iter() + .map(|row| row.get::(0).map_err(HistoryError::from)) + .collect() + }) +} + +fn read_tuple(tx: &Transaction<'_>, schema: &str) -> HistoryResult { + let rows = tx.query( + &format!( + "SELECT h.catalog_incarnation,h.next_history_serial,p.next_pack_id,p.next_ordinal,\ + t.logical,t.canonical,t.stored,t.metadata,\ + (SELECT COALESCE(MAX(stamp),0) FROM {schema}.content_signature),\ + (SELECT COUNT(*) FROM {schema}.\"commit\")+(SELECT COUNT(*) FROM {schema}.layer) \ + FROM {schema}.history_meta h, {schema}.store_policy p, {schema}.store_totals t \ + WHERE h.id=1 AND p.id=1 AND t.id=1" + ), + Vec::new(), + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("Store tuple")); + } + let row = &rows[0]; + Ok(GcTuple { + generation: row.get(0)?, + next_history_serial: row.get(1)?, + next_pack_id: row.get(2)?, + next_ordinal: row.get(3)?, + logical: row.get(4)?, + canonical: row.get(5)?, + stored: row.get(6)?, + metadata: row.get(7)?, + stamp: row.get(8)?, + history_rows: row.get(9)?, + }) +} + +fn copy_owner(tx: &Transaction<'_>, schema: &str, scratch: &str, table: &str) -> HistoryResult<()> { + let rows = tx.query( + "SELECT pg_get_userbyid(c.relowner)::text, COALESCE(array_to_string(c.relacl,E'\\n'),'') \ + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname=$2", + vec![ + Param::Text(schema.to_owned()), + Param::Text(table.to_owned()), + ], + )?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("compaction owner")); + } + let owner: String = rows[0].get(0)?; + tx.batch(&format!( + "ALTER TABLE {scratch}.{} OWNER TO {}", + quote_ident(table), + quote_ident(&owner) + ))?; + let acl: String = rows[0].get(1)?; + for line in acl.lines().filter(|line| !line.is_empty()) { + if let Some(grant) = grant_of(line, schema, table) { + tx.batch(&grant)?; + } + } + Ok(()) +} + +/// One `relacl` entry as a `GRANT` on `schema.table`, or `None` when the +/// entry is the owner's own implicit grant (`owner=arwdDxt/owner`). +fn grant_of(line: &str, schema: &str, table: &str) -> Option { + let (grantee, rest) = line.split_once('=')?; + let (privileges, grantor) = rest.split_once('/')?; + if grantee.is_empty() { + return None; + } + // The owner's implicit self-grant is re-created by ALTER TABLE OWNER. + if grantee == grantor { + return None; + } + let names: Vec<&str> = privileges + .chars() + .filter_map(|letter| { + Some(match letter { + 'r' => "SELECT", + 'w' => "UPDATE", + 'a' => "INSERT", + 'd' => "DELETE", + 'D' => "TRUNCATE", + 'x' => "REFERENCES", + 't' => "TRIGGER", + _ => return None, + }) + }) + .collect(); + if names.is_empty() { + return None; + } + let target = if grantee.is_empty() { + "PUBLIC".to_owned() + } else { + quote_ident(grantee) + }; + Some(format!( + "GRANT {} ON TABLE {schema}.{} TO {target}", + names.join(","), + quote_ident(table) + )) +} + +fn verify_swap( + tx: &Transaction<'_>, + schema: &str, + mark: &MarkResult, + tuple: &GcTuple, + dead: &DeadTotals, +) -> HistoryResult<()> { + let objects = scalar( + tx, + &format!("SELECT count(*) FROM {schema}.object_location"), + Vec::new(), + )?; + let distinct = scalar( + tx, + &format!("SELECT count(DISTINCT object_id) FROM {schema}.object_location"), + Vec::new(), + )?; + let packs = scalar( + tx, + &format!("SELECT count(*) FROM {schema}.pack"), + Vec::new(), + )?; + if objects != mark.objects().len() as i64 + || distinct != objects + || packs != mark.objects().packs().len() as i64 + { + return Err(HistoryError::Integrity("compaction swapped count")); + } + let canonical = scalar( + tx, + &format!("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {schema}.object_location"), + Vec::new(), + )?; + let stored = scalar( + tx, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack"), + Vec::new(), + )?; + let metadata = scalar( + tx, + &format!("SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack WHERE domain=0"), + Vec::new(), + )?; + if canonical != tuple.canonical - dead.canonical + || stored != tuple.stored - dead.stored + || metadata != tuple.metadata - dead.metadata + { + return Err(HistoryError::Integrity("compaction swapped totals")); + } + let logical = scalar( + tx, + &format!("SELECT logical FROM {schema}.store_totals WHERE id=1"), + Vec::new(), + )?; + let sum = scalar( + tx, + &format!( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.layer)" + ), + Vec::new(), + )?; + if logical != sum { + return Err(HistoryError::Integrity( + "Store totals disagree with the history rows", + )); + } + if digest(tx)? != mark.live_heads_digest() { + return Err(HistoryError::Conflict("live heads moved after the mark")); + } + Ok(()) +} + +/// Phase C: delete only the persisted `gc_dead` members, settle the totals +/// once, and clear the op. The Store is usable at `G + 1` throughout. +fn cleanup( + handles: &PostgresHandles, + session: &Arc, + schema: &str, + dead: DeadTotals, + totals_applied: bool, + op_id: Option<[u8; 16]>, +) -> HistoryResult<()> { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.\"commit\" c WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='commit' AND d.id=c.commit_id);\ + DELETE FROM {schema}.layer l WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='layer' AND d.id=l.layer_id);\ + DELETE FROM {schema}.branch b WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='branch' AND d.id=b.branch_id);\ + DELETE FROM {schema}.layer_stack s WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='stack' AND d.id=s.layer_stack_id);\ + DELETE FROM {schema}.layer_lineage ll WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer l WHERE l.layer_id=ll.layer_id);\ + DELETE FROM {schema}.commit_lineage cl WHERE NOT EXISTS (SELECT 1 FROM {schema}.\"commit\" c WHERE c.commit_id=cl.commit_id);\ + DELETE FROM {schema}.absorbed a WHERE NOT EXISTS (SELECT 1 FROM {schema}.commit_lineage cl WHERE cl.run=a.run);\ + DELETE FROM {schema}.reach r WHERE r.key NOT IN (SELECT layer_stack_id FROM {schema}.layer_stack UNION SELECT run FROM {schema}.commit_lineage);\ + DELETE FROM {schema}.branch_retired br WHERE NOT EXISTS (SELECT 1 FROM {schema}.branch b WHERE b.branch_id=br.branch_id);\ + DELETE FROM {schema}.layer_stack_retired lr WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=lr.layer_stack_id);\ + DELETE FROM {schema}.path_removed pr WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=pr.layer_stack_id);\ + DELETE FROM {schema}.project_epoch pe WHERE NOT EXISTS (SELECT 1 FROM {schema}.layer_stack s WHERE s.layer_stack_id=pe.layer_stack_id);\ + DELETE FROM {schema}.content_signature cs WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='signature' AND d.id=int8send(cs.slot));\ + DELETE FROM {schema}.file_digest fd WHERE EXISTS (SELECT 1 FROM {schema}.gc_dead d WHERE d.kind='digest' AND d.id=fd.digest);\ + DELETE FROM {schema}.binding_hint_block;\ + DELETE FROM {schema}.init_operation;\ + DELETE FROM {schema}.init_entry;\ + DELETE FROM {schema}.init_host_file;" + ))?; + if !totals_applied { + tx.query( + &format!( + "UPDATE {schema}.store_totals SET \ + canonical=canonical-$1,stored=stored-$2,metadata=metadata-$3,logical=logical-$4 WHERE id=1" + ), + vec![ + Param::I64(dead.canonical), + Param::I64(dead.stored), + Param::I64(dead.metadata), + Param::I64(dead.logical), + ], + )?; + tx.query( + &format!( + "UPDATE {schema}.gc_op SET totals_applied=TRUE,phase='cleanup' WHERE id=1" + ), + Vec::new(), + )?; + } + let totals = tx.query( + &format!( + "SELECT logical,canonical,stored,metadata FROM {schema}.store_totals WHERE id=1" + ), + Vec::new(), + )?; + let logical: i64 = totals[0].get(0)?; + let canonical: i64 = totals[0].get(1)?; + let stored: i64 = totals[0].get(2)?; + let metadata: i64 = totals[0].get(3)?; + let sums = tx.query( + &format!( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {schema}.layer),\ + (SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {schema}.object_location),\ + (SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack),\ + (SELECT COALESCE(SUM(length)::bigint,0) FROM {schema}.pack WHERE domain=0)" + ), + Vec::new(), + )?; + let sum_logical: i64 = sums[0].get(0)?; + let sum_canonical: i64 = sums[0].get(1)?; + let sum_stored: i64 = sums[0].get(2)?; + let sum_metadata: i64 = sums[0].get(3)?; + if logical != sum_logical + || canonical != sum_canonical + || stored != sum_stored + || metadata != sum_metadata + { + return Err(HistoryError::Integrity("compaction settled totals")); + } + tx.batch(&format!( + "DELETE FROM {schema}.gc_dead; DELETE FROM {schema}.gc_op;" + ))?; + Ok(()) + })?; + if let Some(op) = op_id { + let _ = handles.history.gc_release(&op); + } else { + stage(session, |tx| { + tx.batch(&format!( + "DELETE FROM {schema}.gc_intent WHERE kind='compaction'" + ))?; + Ok(()) + })?; + } + Ok(()) +} + +/// One round after admission: mark, decide, build, swap, cleanup. +fn round( + handles: &PostgresHandles, + session: &Arc, + schema: &str, + request: &CompactRequest, +) -> HistoryResult { + let (mark, tuple) = handles.history.mark_gc()?; + load_visited(session, schema, &mark)?; + let stored = u64::try_from(total_stored(session, schema)?).unwrap_or(0); + let live = u64::try_from(live_stored(session, schema)?).unwrap_or(0); + let dead_bytes = stored.saturating_sub(live); + let scratch = scratch_name(&request.op_id); + let mark_dead = persist_mark(session, schema, &mark, &tuple, &scratch, request)?; + let generation = u64::try_from(tuple.generation).unwrap_or(0); + let kept = CompactKeptWaste { + dead_in_retained_packs_bytes: mark.objects().waste().dead_in_retained_packs_bytes(), + delta_bases_bytes: mark.objects().waste().delta_bases_bytes(), + delta_base_references: mark + .objects() + .waste() + .delta_base_references() + .iter() + .map(|id| *id.as_bytes()) + .collect(), + inode_blocks_bytes: mark.objects().waste().inode_blocks_bytes(), + }; + let rebuild = stored > 0 && dead_bytes.saturating_mul(THRESHOLD) >= stored; + if !rebuild { + clear_round(session, schema)?; + let _ = handles.history.gc_release(&request.op_id); + return Ok(report( + CompactPath::None, + generation, + generation, + dead_bytes, + dead_bytes, + 0, + 0, + 0, + kept, + 0, + CompactResidue::default(), + stored, + mark.history().len() as u64, + )); + } + stop( + request, + CompactFailpoint::PostgresAfterMark, + session, + schema, + "mark", + )?; + let mut built = build(session, schema, &scratch, &mark, &tuple)?; + built.dead.logical = mark_dead.logical; + stop( + request, + CompactFailpoint::PostgresAfterBuild, + session, + schema, + "build", + )?; + let swapped = swap(handles, session, schema, &tuple, &mark, &built)?; + stop( + request, + CompactFailpoint::PostgresAfterSwap, + session, + schema, + "swap", + )?; + let logical_discarded = u64::try_from(swapped.dead.logical).unwrap_or(0); + cleanup( + handles, + session, + schema, + swapped.dead, + false, + Some(request.op_id), + )?; + stop( + request, + CompactFailpoint::PostgresAfterCleanup, + session, + schema, + "cleanup", + )?; + let shrink = swapped + .physical_before + .saturating_sub(swapped.physical_after); + Ok(report( + CompactPath::Rebuild, + generation, + generation + 1, + dead_bytes, + dead_bytes, + logical_discarded, + 0, + shrink, + kept, + swapped.peak_temp, + CompactResidue::default(), + stored, + mark.history().len() as u64, + )) +} + +#[allow(clippy::too_many_arguments)] +fn report( + path: CompactPath, + generation_before: u64, + generation_after: u64, + dead_bytes: u64, + dead_refined: u64, + logical_discarded: u64, + backend_reusable: u64, + physical_shrink: u64, + kept_waste: CompactKeptWaste, + peak_temp_bytes: u64, + residue: CompactResidue, + total_stored: u64, + history_rows: u64, +) -> CompactReceipt { + let mut zero_reasons = Vec::new(); + if dead_bytes == 0 { + let reason = if total_stored == 0 { + "empty-store" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "deadBytesBound", + reason, + }); + zero_reasons.push(CompactZeroReason { + field: "deadBytesRefined", + reason, + }); + } + if logical_discarded == 0 { + let reason = if history_rows == 0 { + "no-history" + } else if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }; + zero_reasons.push(CompactZeroReason { + field: "logicalDiscarded", + reason, + }); + } + if physical_shrink == 0 { + zero_reasons.push(CompactZeroReason { + field: "physicalShrink", + reason: if path == CompactPath::None { + "none-path" + } else { + "no-waste" + }, + }); + } + zero_reasons.push(CompactZeroReason { + field: "backendReusable", + reason: "postgresql-in-place", + }); + if kept_waste.is_empty() { + zero_reasons.push(CompactZeroReason { + field: "keptWaste", + reason: "no-waste", + }); + } + if peak_temp_bytes == 0 { + zero_reasons.push(CompactZeroReason { + field: "peakTempBytes", + reason: "none-path", + }); + } + CompactReceipt { + path, + generation_before, + generation_after, + dead_bytes_bound: dead_bytes, + dead_bytes_refined: dead_refined, + logical_discarded, + backend_reusable, + physical_shrink, + kept_waste, + peak_temp_bytes, + residue, + estimate: true, + zero_reasons, + } +} + +impl PostgresHandles { + /// Runs one §8 PostgreSQL Store compaction on this Store's session. + /// + /// The call takes the Store's exclusive intent (a session advisory key) + /// through cleanup, marks at `G`, and on the rebuild path copies the two + /// payload tables, swaps them in one short transaction at `G + 1`, and + /// deletes only the persisted dead set. The caller keeps its session; the + /// Store is usable at `G + 1` from the swap on. + pub fn compact(&self, request: &CompactRequest) -> HistoryResult { + let session = self.storage.session.clone(); + let schema = session.settings.schema.clone(); + let _ = read_generation(&session, &schema)?; + if let Some(receipt) = settle_leftover(self, &session, &schema)? { + return Ok(receipt); + } + let intent = GcIntent::operation( + request.op_id, + GcIntentKind::Compaction, + GcScopeKind::Store, + self.history.catalog_id.as_slice().to_vec(), + ) + .owned( + request.owner_runtime.as_deref(), + request.owner_boot.as_deref(), + ) + .with_phase(Some("mark")); + self.history.gc_register(&intent)?; + let result = (|| { + let rank = effective_sync_rank(&session)?; + standby_precheck(&session, rank)?; + round(self, &session, &schema, request) + })(); + if result.is_err() { + let _ = self.history.gc_release(&request.op_id); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::{standby_satisfied, sync_name, sync_rank}; + + #[test] + fn the_standby_predicate_waits_only_for_a_named_standby() { + // Below remote_write, or with no names, no standby is required. + assert!(standby_satisfied(0, "", 0)); + assert!(standby_satisfied(1, "", 0)); + assert!(standby_satisfied(2, "", 0)); + assert!(standby_satisfied(4, " ", 0)); + // At or above remote_write with names, a streaming sync/quorum + // standby must exist. + assert!(!standby_satisfied(2, "FIRST 1 (s1)", 0)); + assert!(!standby_satisfied(4, "ANY 1 (s1)", 0)); + assert!(standby_satisfied(2, "FIRST 1 (s1)", 1)); + assert!(standby_satisfied(3, "ANY 2 (s1,s2)", 1)); + } + + #[test] + fn synchronous_commit_strength_is_the_spec_order() { + assert_eq!(sync_rank("off"), 0); + assert_eq!(sync_rank("local"), 1); + assert_eq!(sync_rank("remote_write"), 2); + assert_eq!(sync_rank("on"), 3); + assert_eq!(sync_rank("remote_apply"), 4); + assert_eq!(sync_name(4), "remote_apply"); + assert_eq!(sync_name(0), "off"); + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs index 98439899fc..74225ff555 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/mod.rs @@ -2,6 +2,7 @@ //! lane, objects, metadata and history statements, and Init's acquisition. pub(crate) mod access; pub(crate) mod acquisition; +pub(crate) mod compact; pub(crate) mod history; pub(crate) mod lane; mod mechanics; diff --git a/engine/crates/layerfs-persistence/src/backend/records.rs b/engine/crates/layerfs-persistence/src/backend/records.rs index 9efc76cd55..4cfb62853d 100644 --- a/engine/crates/layerfs-persistence/src/backend/records.rs +++ b/engine/crates/layerfs-persistence/src/backend/records.rs @@ -118,6 +118,8 @@ pub(crate) enum BackendError { #[cfg_attr(not(target_os = "linux"), allow(dead_code))] Cancelled, ReadOnly, + /// §10: the Store path no longer names the file this handle opened. + Stale, Capacity, Integrity, /// A SQLite Store of schema version 1 (before #91): refused at open with @@ -158,6 +160,8 @@ impl From for layerfs_history::HistoryError { Self::OwnershipUnavailable } BackendError::ReadOnly => Self::ContinuityUnavailable, + // §10: the file was replaced or moved under this handle. + BackendError::Stale => Self::StaleStore, BackendError::Capacity => Self::Capacity("persistence"), BackendError::Integrity => Self::Integrity("persistence"), BackendError::SchemaVersion1 => Self::Integrity(SCHEMA_VERSION_1), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs index b14fa8b4c0..61c3b99aef 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs @@ -114,28 +114,63 @@ impl CompactPath { #[doc(hidden)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CompactFailpoint { + /// Before the seal, while the open session, the writer lane and the + /// admission intent are still held. + BeforeSeal, + /// After the seal returns and before the sealed file is read back. + BeforeSealRead, /// After the seal and the sealed-state comparison, before any artifact. AfterSeal, + /// Inside the build, before the visited scratch or the copy is written. + BeforeCopy, + /// Inside the build, after `VACUUM INTO` produced `.compact` and + /// before the copy is pruned: the state a crash during the copy leaves. + DuringCopy, /// After the staged `.compact` is built and compacted. AfterBuild, + /// The staged Store's `VACUUM INTO` meets a bounded filesystem (ENOSPC). + CapacityAtBuild, /// After the staged file is switched to WAL and checkpointed. AfterWalSwitch, + /// Inside the verification, after the schema and row-count checks. + DuringVerify, /// After the immutable verification passes. AfterVerify, /// Immediately before the rename would run. BeforeInstall, + /// After the rename and before the directory fsync (rename/journal). + DuringRename, /// After the rename and directory fsync. AfterInstall, + /// Inside the settle, after the artifact set is removed and before the op + /// record. + DuringSettle, + /// PostgreSQL: after Phase M wrote the op and the dead set, before Phase A. + PostgresAfterMark, + /// PostgreSQL: after Phase A built and verified the scratch tables. + PostgresAfterBuild, + /// PostgreSQL: after Phase B committed the swap, before Phase C. + PostgresAfterSwap, + /// PostgreSQL: after Phase C settled the op. + PostgresAfterCleanup, } impl CompactFailpoint { /// The phase the sidecar records for this stop point. const fn phase(self) -> &'static str { match self { - Self::AfterSeal => "mark", - Self::AfterBuild => "build", - Self::AfterWalSwitch | Self::AfterVerify | Self::BeforeInstall => "verify", - Self::AfterInstall => "install", + Self::BeforeSeal | Self::BeforeSealRead | Self::AfterSeal => "mark", + Self::BeforeCopy + | Self::DuringCopy + | Self::AfterBuild + | Self::CapacityAtBuild + | Self::AfterWalSwitch => "build", + Self::DuringVerify | Self::AfterVerify | Self::BeforeInstall => "verify", + Self::DuringRename | Self::AfterInstall | Self::DuringSettle => "install", + Self::PostgresAfterMark => "mark", + Self::PostgresAfterBuild => "build", + Self::PostgresAfterSwap => "swap", + Self::PostgresAfterCleanup => "cleanup", } } } @@ -149,8 +184,21 @@ pub struct CompactRequest { pub owner_runtime: Option, /// Owning boot token, for the op and intent records. pub owner_boot: Option, + /// The owning process id, so `store.status` can tell a live run from a + /// leftover (§11). The kernel start time and boot id, when known, keep a + /// reused id from passing as the owner. + pub owner_pid: Option, + /// The owner's kernel start time (the lease's second field). + pub owner_pid_start: Option, + /// The owner's kernel boot id (the lease's third field). + pub owner_pid_boot: Option, /// A test-only stop point; production callers leave it `None`. pub failpoint: Option, + /// A test-only hook called after verification and immediately before the + /// §10 install fence, so a test can replace the Store under the sealed + /// handle. Production callers leave it `None`. + #[doc(hidden)] + pub before_install_hook: Option, } /// The `keptWaste` §11 part of a compaction report. @@ -210,6 +258,24 @@ pub struct CompactZeroReason { pub reason: &'static str, } +impl CompactZeroReason { + /// The §10 wire key this reason names: the §11 logical name rendered + /// snake_case, the same mapping the report serializer uses. + #[must_use] + pub fn wire_field(self) -> &'static str { + match self.field { + "deadBytesBound" => "dead_bytes_bound", + "deadBytesRefined" => "dead_bytes_refined", + "logicalDiscarded" => "logical_discarded", + "backendReusable" => "backend_reusable", + "physicalShrink" => "physical_shrink", + "keptWaste" => "kept_waste", + "peakTempBytes" => "peak_temp_bytes", + other => other, + } + } +} + /// The §11 report `store.compact` returns. #[derive(Clone, Debug, Eq, PartialEq)] pub struct CompactReceipt { @@ -248,6 +314,9 @@ struct RunOp { generation: i64, phase: String, started_ms: i64, + owner_pid: Option, + owner_pid_start: Option, + owner_pid_boot: Option, tuple: MarkTuple, live_heads_digest: [u8; 32], } @@ -256,6 +325,7 @@ impl RunOp { fn encode(&self) -> String { format!( "version=1\nop_id={}\ngeneration={}\nphase={}\nstarted_ms={}\n\ +owner_pid={}\nowner_pid_start={}\nowner_pid_boot={}\n\ catalog_id={}\nnext_history_serial={}\nnext_pack_id={}\nnext_ordinal={}\n\ mark_serial={}\nmark_pack_id={}\nmark_digest={}\nmark_totals={},{},{},{}\nmark_stamp={}\nhistory_rows={}\n\ live_heads_digest={}\n", @@ -263,6 +333,9 @@ live_heads_digest={}\n", self.generation, self.phase, self.started_ms, + self.owner_pid.map_or_else(String::new, |pid| pid.to_string()), + self.owner_pid_start.map_or_else(String::new, |start| start.to_string()), + self.owner_pid_boot.map_or_else(String::new, |boot| format!("{boot:032x}")), hex(&self.tuple.catalog_id), self.tuple.next_history_serial, self.tuple.next_pack_id, @@ -310,6 +383,18 @@ live_heads_digest={}\n", let generation: i64 = values.get("generation")?.parse().ok()?; let phase = (*values.get("phase")?).to_owned(); let started_ms: i64 = values.get("started_ms")?.parse().ok()?; + let owner_pid = values + .get("owner_pid") + .filter(|text| !text.is_empty()) + .and_then(|text| text.parse().ok()); + let owner_pid_start = values + .get("owner_pid_start") + .filter(|text| !text.is_empty()) + .and_then(|text| text.parse().ok()); + let owner_pid_boot = values + .get("owner_pid_boot") + .filter(|text| !text.is_empty()) + .and_then(|text| u128::from_str_radix(text, 16).ok()); let catalog_id = unhex_vec(values.get("catalog_id")?)?; let tuple = MarkTuple { catalog_id, @@ -330,12 +415,87 @@ live_heads_digest={}\n", generation, phase, started_ms, + owner_pid, + owner_pid_start, + owner_pid_boot, tuple, live_heads_digest, }) } } +/// The operation a Store's `.compact-op` records, as `store.status` +/// (T10.1) projects it: the operation identity, the generation the run started +/// at, its phase, its owner and whether its outcome is known. A settled run +/// removes the record, so [`compact_op_status`] returns `None`. +#[doc(hidden)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactOpStatus { + /// The 16-byte operation identity. + pub op_id: [u8; 16], + /// The generation the run started at. + pub generation: i64, + /// The recorded phase: `admission`, `mark`, `build`, `verify` or `install`. + pub phase: String, + /// When the run started, in Unix milliseconds. + pub started_ms: i64, + /// The owning process id, when the run recorded one. + pub owner_pid: Option, + /// The owner's kernel start time, when recorded. + pub owner_pid_start: Option, + /// The owner's kernel boot id, when recorded. + pub owner_pid_boot: Option, + /// True while the record remains: the run did not settle, so its outcome + /// is unknown. `store.status` reports `busy` while the owner lives and + /// `unresolved` once it is gone. + pub uncertain: bool, +} + +impl CompactOpStatus { + /// The `store.status` projection of this operation; `alive` says whether + /// the owning process is still running. A live owner is `busy`, a gone one + /// left an uncertain `unresolved` outcome. `uncertain` stays the record's + /// own fact: it is true for every unsettled run, live or leftover. + #[must_use] + pub fn status(&self, alive: bool) -> crate::report::CompactStatus { + // A read in the admission window still projects a §11 phase: the run + // is about to mark, and `admission` is not one of the four. + let phase = if self.phase == "admission" { + "mark".to_owned() + } else { + self.phase.clone() + }; + crate::report::CompactStatus { + id: format!("op-{}", hex(&self.op_id)), + name: "store.compact", + phase, + started: self.started_ms, + uncertain: self.uncertain, + state: if alive { "busy" } else { "unresolved" }, + phases: &crate::report::SQLITE_COMPACT_PHASES, + } + } +} + +/// Reads the operation `.compact-op` records, or `None` when the Store +/// has no compaction record. `store.status` (T10.1) uses this to show the +/// operation, its phase and its uncertain outcome; a settled run removes the +/// record (§9). +#[doc(hidden)] +#[must_use] +pub fn compact_op_status(store: &Path) -> Option { + RunOp::read(store).map(|op| CompactOpStatus { + op_id: op.op_id, + generation: op.generation, + phase: op.phase, + started_ms: op.started_ms, + owner_pid: op.owner_pid, + owner_pid_start: op.owner_pid_start, + owner_pid_boot: op.owner_pid_boot, + uncertain: true, + }) +} + fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -731,6 +891,9 @@ fn build( packs: &BTreeMap, layout: crate::SqlitePackLayout, synchronous: i64, + capacity: bool, + request: &CompactRequest, + op: &RunOp, ) -> HistoryResult { let store = &facts.path; let target = compact_path(store); @@ -740,7 +903,15 @@ fn build( return Err(HistoryError::Exists("staged compaction artifact")); } } + stop_at(request, CompactFailpoint::BeforeCopy, op, store)?; let visited_bytes = write_visited(&visited, mark, packs)?; + if capacity { + // §7 step 3 on a bounded filesystem: `VACUUM INTO` has already created + // the staged file when the write meets ENOSPC. The abort path removes + // it and the live Store is unchanged. + std::fs::write(&target, b"staged").map_err(io_error)?; + return Err(io_error(std::io::Error::from_raw_os_error(28))); + } let source = open_immutable(store)?; let target_text = target @@ -752,6 +923,8 @@ fn build( source .close() .map_err(|(_, error)| db_error(error))?; + // The copy produced the staged file but has not pruned or verified it. + stop_at(request, CompactFailpoint::DuringCopy, op, store)?; let connection = Connection::open_with_flags( &target, @@ -998,11 +1171,14 @@ metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", /// allocator tuple and the recomputed totals. `quick_check` and the full doctor /// stay separate commands. fn verify( + store: &Path, built: &Built, mark: &MarkResult, tuple: &MarkTuple, layout: crate::SqlitePackLayout, acquisition: crate::SqliteAcquisitionSchema, + request: &CompactRequest, + op: &RunOp, ) -> HistoryResult<()> { let connection = open_immutable(&built.target)?; schema::check( @@ -1029,6 +1205,7 @@ fn verify( if count("SELECT count(*) FROM kept_node")? != mark.history().len() as u64 { return Err(HistoryError::Integrity("staged Store history count")); } + stop_at(request, CompactFailpoint::DuringVerify, op, store)?; if file_heads_digest(&connection)? != mark.live_heads_digest() { return Err(HistoryError::Conflict( "staged Store live heads differ from the mark", @@ -1147,6 +1324,18 @@ fn sidecar_present(path: &Path) -> Option { /// §7 step 7's fence before the rename: no sidecar, the same `G`, the same /// catalog identity and the sealed `(size, mtime, change counter)`. fn install_fence(facts: &SealedFacts, tuple: &MarkTuple) -> HistoryResult<()> { + // §10: the path must still name the sealed file. A replacement is + // `StaleStore` before the rename could install over a different Store. + if let Some(sealed) = facts.identity { + match sealed.names(&facts.path) { + Ok(true) => {} + Ok(false) => return Err(HistoryError::StaleStore), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(HistoryError::StaleStore); + } + Err(_) => return Err(HistoryError::Integrity("Store identity at install")), + } + } for path in [&facts.path, &compact_path(&facts.path)] { if sidecar_present(path).is_some() { return Err(HistoryError::Conflict("Store sidecar at install")); @@ -1192,8 +1381,8 @@ fn sync_directory(store: &Path) -> HistoryResult<()> { Ok(()) } -/// Removes the fixed artifact set and reports what remains, measuring the -/// bytes before deletion. +/// Removes the fixed artifact set and reports only what remains, so a +/// successful settle has a zero residue (§11). fn clear_artifacts(store: &Path) -> CompactResidue { let mut bytes = 0u64; let mut left = Vec::new(); @@ -1201,8 +1390,10 @@ fn clear_artifacts(store: &Path) -> CompactResidue { let Ok(metadata) = std::fs::symlink_metadata(&path) else { continue; }; - bytes = bytes.saturating_add(metadata.len()); if std::fs::remove_file(&path).is_err() && std::fs::symlink_metadata(&path).is_ok() { + // §11: residue is what the settle left behind. An artifact that + // was removed is not residue, so its bytes are not counted. + bytes = bytes.saturating_add(metadata.len()); left.push( path.file_name() .map_or_else(|| path.display().to_string(), |name| { @@ -1220,25 +1411,28 @@ fn clear_artifacts(store: &Path) -> CompactResidue { /// Settles one run: the artifact set is removed first, the op record last /// (§9: the op is settled only after removal). -fn settle(store: &Path) -> CompactResidue { +fn settle(store: &Path, request: &CompactRequest, op: &RunOp) -> HistoryResult { let mut residue = clear_artifacts(store); - let op = op_path(store); - if std::fs::symlink_metadata(&op).is_ok() { - let bytes = std::fs::metadata(&op).map_or(0, |metadata| metadata.len()); - if std::fs::remove_file(&op).is_err() && std::fs::symlink_metadata(&op).is_ok() { + // §9: the op settles only after the artifact set is gone. A crash here + // leaves the installed `G + 1` and the record for the next run. + stop_at(request, CompactFailpoint::DuringSettle, op, store)?; + let record_path = op_path(store); + if std::fs::symlink_metadata(&record_path).is_ok() { + let bytes = std::fs::metadata(&record_path).map_or(0, |metadata| metadata.len()); + if std::fs::remove_file(&record_path).is_err() + && std::fs::symlink_metadata(&record_path).is_ok() + { residue.bytes = residue.bytes.saturating_add(bytes); - residue.artifacts.push( - op.file_name() - .map_or_else(|| op.display().to_string(), |name| { - name.to_string_lossy().into_owned() - }), - ); + residue.artifacts.push(record_path.file_name().map_or_else( + || record_path.display().to_string(), + |name| name.to_string_lossy().into_owned(), + )); } } if !residue.artifacts.is_empty() && residue.reason.is_none() { residue.reason = Some("the artifact set could not be removed".to_owned()); } - residue + Ok(residue) } /// Drops every artifact and the op record after a definite failure before the @@ -1250,6 +1444,16 @@ fn abort_before_install(store: &Path) { } } +/// Releases the admission intent of a definite failure before the seal and +/// then removes the artifact set and the op record. When the release itself +/// fails the op record is kept as the next run's recovery breadcrumb (§9): +/// deleting it would leave a live intent that no later call could settle. +fn abort_run(history: &crate::HistoryProvider, store: &Path, op_id: &[u8; 16]) { + if history.gc_release(op_id).is_ok() { + abort_before_install(store); + } +} + /// Judges and removes a leftover an earlier run recorded (§9): the artifact set /// it names and the admission intent it took. The caller already holds the /// writer lane, so a leftover owner is gone. @@ -1493,6 +1697,9 @@ impl SqliteHandles { generation: live.generation, phase: "admission".to_owned(), started_ms: now_ms(), + owner_pid: request.owner_pid, + owner_pid_start: request.owner_pid_start, + owner_pid_boot: request.owner_pid_boot, tuple: live.clone(), live_heads_digest: [0u8; 32], }; @@ -1509,26 +1716,40 @@ impl SqliteHandles { ) .with_phase(Some("mark")); history.gc_register(&intent)?; - let mark = history.mark()?; - let tuple = live_tuple(&session)?; - op.tuple = tuple.clone(); - op.live_heads_digest = mark.live_heads_digest(); - op.phase = "mark".to_owned(); - op.write(&store)?; - - let lengths = pack_lengths(&session)?; - let total = lengths.values().copied().fold(0u64, u64::saturating_add); - let mut live_stored = 0u64; - for pack in mark.objects().packs() { - let length = lengths - .get(pack) - .ok_or(HistoryError::Integrity("mark pack"))?; - live_stored = live_stored.saturating_add(*length); - } - let dead = total.saturating_sub(live_stored); + // §8: the registration, the mark and the totals read are one + // admitted sequence. A definite failure here releases the intent + // before the artifact set and the op record go, so the Store is + // left exactly as it was and no later call meets BusyOp. + let admitted = + (|| -> HistoryResult<(MarkResult, MarkTuple, u64, u64, CompactKeptWaste, u64)> { + let mark = history.mark()?; + let tuple = live_tuple(&session)?; + op.tuple = tuple.clone(); + op.live_heads_digest = mark.live_heads_digest(); + op.phase = "mark".to_owned(); + op.write(&store)?; + let lengths = pack_lengths(&session)?; + let total = lengths.values().copied().fold(0u64, u64::saturating_add); + let mut live_stored = 0u64; + for pack in mark.objects().packs() { + let length = lengths + .get(pack) + .ok_or(HistoryError::Integrity("mark pack"))?; + live_stored = live_stored.saturating_add(*length); + } + let dead = total.saturating_sub(live_stored); + let kept_waste = CompactKeptWaste::from_mark(mark.objects().waste()); + let history_rows = u64::try_from(tuple.history_rows).unwrap_or(0); + Ok((mark, tuple, total, dead, kept_waste, history_rows)) + })(); + let (mark, tuple, total, dead, kept_waste, history_rows) = match admitted { + Ok(admitted) => admitted, + Err(error) => { + abort_run(&history, &store, &op_id); + return Err(error); + } + }; let rebuild = total > 0 && dead.saturating_mul(10) >= total; - let kept_waste = CompactKeptWaste::from_mark(mark.objects().waste()); - let history_rows = u64::try_from(tuple.history_rows).unwrap_or(0); if !rebuild { // §3 `none`: mark only. The intent is cleared and the op record @@ -1553,8 +1774,27 @@ impl SqliteHandles { )); } + // §7 step 2: a stop here leaves the open session, the writer lane, + // the admission intent and the op record; the live Store is at `G`. + stop_at(request, CompactFailpoint::BeforeSeal, &op, &store)?; + // §7 step 2 onward: seal and rebuild. Only the session remains. - let object_packs = object_packs(&session)?; + let object_packs = match object_packs(&session) { + Ok(packs) => packs, + Err(error) => { + abort_run(&history, &store, &op_id); + return Err(error); + } + }; + // §8/§9: release the admission intent while the Store is still + // reachable. Every failure below runs after the seal, when this + // process can no longer run SQL on the Store; without this release + // a live intent would refuse every later GC operation. The held + // writer lane still excludes writers through the install, and a + // crash leaves the op record for the next run to settle. + if let Err(error) = history.gc_release(&op_id) { + return Err(error); + } drop(history); drop(storage); let mut guard = WriterGuard::new(lane.into_writer()); @@ -1565,6 +1805,8 @@ impl SqliteHandles { Err(error) => return Err(seal_error(error)), }; guard.put(writer); + // §7 step 1.5: the sealed file is about to be read back. + stop_at(request, CompactFailpoint::BeforeSealRead, &op, &store)?; // §7 step 1.5's fence: the seal must not have moved the generation // or the heads the mark recorded. @@ -1594,6 +1836,9 @@ impl SqliteHandles { } stop_at(request, CompactFailpoint::AfterSeal, &op, &store)?; + op.phase = "build".to_owned(); + op.write(&store)?; + let capacity = request.failpoint == Some(CompactFailpoint::CapacityAtBuild); let built = match build( &facts, &mark, @@ -1601,8 +1846,14 @@ impl SqliteHandles { &object_packs, layout, synchronous, + capacity, + request, + &op, ) { Ok(built) => built, + // A stop point leaves the crash state on purpose: never abort + // an artifact set a recorded run names (§9). + Err(HistoryError::UnknownOutcome) => return Err(HistoryError::UnknownOutcome), Err(error) => { abort_before_install(&store); return Err(error); @@ -1611,12 +1862,32 @@ impl SqliteHandles { stop_at(request, CompactFailpoint::AfterBuild, &op, &store)?; stop_at(request, CompactFailpoint::AfterWalSwitch, &op, &store)?; - if let Err(error) = verify(&built, &mark, &tuple, layout, acquisition_schema) { + op.phase = "verify".to_owned(); + if let Err(error) = op.write(&store) { + abort_before_install(&store); + return Err(error); + } + if let Err(error) = verify( + &store, + &built, + &mark, + &tuple, + layout, + acquisition_schema, + request, + &op, + ) { + if matches!(error, HistoryError::UnknownOutcome) { + return Err(error); + } abort_before_install(&store); return Err(error); } stop_at(request, CompactFailpoint::AfterVerify, &op, &store)?; + if let Some(hook) = request.before_install_hook { + hook(&facts.path); + } if let Err(error) = install_fence(&facts, &tuple) { abort_before_install(&store); return Err(error); @@ -1634,14 +1905,19 @@ impl SqliteHandles { let _ = op.write(&store); return Err(HistoryError::UnknownOutcome); } + // §7 step 7: the rename happened; a crash before the directory + // fsync leaves an installed-but-uncertain `G + 1`. + stop_at(request, CompactFailpoint::DuringRename, &op, &store)?; if let Err(_error) = sync_directory(&facts.path) { op.phase = "install".to_owned(); let _ = op.write(&store); return Err(HistoryError::UnknownOutcome); } + op.phase = "install".to_owned(); + let _ = op.write(&store); stop_at(request, CompactFailpoint::AfterInstall, &op, &store)?; guard.release()?; - let residue = settle(&store); + let residue = settle(&store, request, &op)?; let logical_discarded = u64::try_from(tuple.logical) .unwrap_or(0) .saturating_sub(built.logical); @@ -1663,3 +1939,29 @@ impl SqliteHandles { } } } + +#[cfg(test)] +mod tests { + use super::io_error; + use layerfs_history::HistoryError; + + /// §7 step 3 on a bounded filesystem: `ENOSPC`, `EDQUOT` and `EFBIG` are + /// the report's `Capacity` refusal, never a corruption verdict. + #[test] + fn bounded_filesystem_errors_are_capacity() { + for code in [28, 122, 27, 69] { + let error = io_error(std::io::Error::from_raw_os_error(code)); + assert!( + matches!(error, HistoryError::Capacity(_)), + "errno {code}: {error:?}" + ); + } + } + + /// Every other I/O failure keeps the integrity classification. + #[test] + fn another_io_error_is_integrity() { + let error = io_error(std::io::Error::from_raw_os_error(5)); + assert!(matches!(error, HistoryError::Integrity(_)), "{error:?}"); + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs new file mode 100644 index 0000000000..43eac9148c --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs @@ -0,0 +1,51 @@ +//! §10 Store-file identity: the `(dev, ino)` fence a SQLite handle applies +//! before it reads or writes, and the install fence re-applies to the sealed +//! source. +use std::path::Path; + +/// The `(device, inode)` pair naming one Store file at one moment (§10). +/// +/// A transaction compares the pair recorded at open with the pair the Store +/// path names now; a replacement or a move is `StaleStore`. Together with the +/// held writer lane this is the SQLite fence of §10: an admitted writer cannot +/// be in flight at install, and a connection that bypassed admission is caught +/// at its next transaction. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct FileIdentity { + /// The filesystem device holding the file. + pub(crate) dev: u64, + /// The file's inode number on that device. + pub(crate) ino: u64, +} + +impl FileIdentity { + /// Captures the identity of the file `path` currently names. + /// + /// `Ok(None)` means this platform has no `(dev, ino)` pair; the fence then + /// has nothing to compare and callers keep their other facts. `Err` is a + /// path that could not be inspected at all. + pub(crate) fn capture(path: &Path) -> Result, std::io::Error> { + let metadata = std::fs::metadata(path)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + Ok(Some(Self { + dev: metadata.dev(), + ino: metadata.ino(), + })) + } + #[cfg(not(unix))] + { + let _ = metadata; + Ok(None) + } + } + + /// §10: whether `path` still names this exact file. + pub(crate) fn names(&self, path: &Path) -> Result { + match Self::capture(path)? { + Some(current) => Ok(current == *self), + None => Ok(true), + } + } +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs index 1895897203..a23a4ce8cb 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod file_control; #[cfg(any(target_os = "macos", target_os = "linux"))] pub(crate) mod headroom; +pub(crate) mod identity; pub(crate) mod reclamation; pub(crate) mod seal; #[cfg(target_os = "macos")] diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index 178f404237..6370074a43 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -1,5 +1,6 @@ //! Sole-owner host seal; never available through the daemon's Store ports. use crate::backend::sqlite::lane::writer_admission::Writer; +use crate::backend::sqlite::lifecycle::identity::FileIdentity; use crate::backend::sqlite::session::connection::Session; use crate::backend::sqlite::session::query; use crate::backend::sqlite::session::rows; @@ -25,6 +26,8 @@ pub(crate) struct SealedFacts { pub(crate) modified: SystemTime, /// The database header change counter (big-endian bytes 24..28). pub(crate) change_counter: u32, + /// The file's §10 `(dev, ino)` while the lane was still held. + pub(crate) identity: Option, } impl SealedFacts { @@ -159,6 +162,10 @@ impl Session { released }; let change_counter = read_change_counter(&self.path)?; + let identity = + FileIdentity::capture(&self.path).map_err(|error| PersistenceError::Refused { + status: error.to_string(), + })?; Ok(( SealedFacts { path: self.path.clone(), @@ -167,6 +174,7 @@ impl Session { bytes, modified, change_counter, + identity, }, writer, )) diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index c080697dae..608efa6d47 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -3,6 +3,7 @@ use super::profile; use super::query; use super::rows; use crate::backend::records::BackendError; +use crate::backend::sqlite::lifecycle::identity::FileIdentity; use crate::SqlitePersistenceProfile; use rusqlite::{limits::Limit, Connection, OpenFlags}; use std::{ @@ -165,6 +166,9 @@ pub(crate) struct Session { pub(crate) profile: ConnectionProfile, pub(crate) layout: crate::SqlitePackLayout, pub(crate) acquisition: crate::SqliteAcquisitionSchema, + /// The `(dev, ino)` identity the path named when this handle opened (§10). + /// `None` on a platform without one; the fence then has nothing to compare. + pub(crate) identity: Option, pub(crate) path: PathBuf, } impl Session { @@ -331,6 +335,8 @@ impl Session { let background = (writable && selected == SqlitePersistenceProfile::Disposable) .then(|| super::background::Checkpointer::start(path)) .transpose()?; + // §10: the record every later transaction is fenced against. + let identity = FileIdentity::capture(path).map_err(|_| BackendError::Integrity)?; Ok(Self { diagnostic, background, @@ -347,6 +353,7 @@ impl Session { profile, layout, acquisition, + identity, }) } /// Stored schema version: the pack layout's, offset by the acquisition tables. @@ -361,4 +368,19 @@ impl Session { let w = *s.work.borrow(); Ok(w) } + /// §10: the Store path must still name the file this handle opened, or the + /// call is `StaleStore` before it reads or writes anything. + pub(crate) fn check_identity(&self) -> Result<(), BackendError> { + let Some(expected) = self.identity else { + return Ok(()); + }; + match expected.names(&self.path) { + Ok(true) => Ok(()), + Ok(false) => Err(BackendError::Stale), + // A removed Store is as stale as a replaced one: nothing this + // handle opened is left at the path to act on. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Err(BackendError::Stale), + Err(_) => Err(BackendError::Integrity), + } + } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index 640fc7b9d9..bd64d7a177 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -270,6 +270,9 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "gc_inode_blocks" => Ok(include_str!( "../../../../sql/sqlite/queries/history/gc_inode_blocks.sql" )), + "gc_tuple_read" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/gc_tuple_read.sql" + )), "kept_node_clear" => Ok(include_str!( "../../../../sql/sqlite/queries/history/kept_node_clear.sql" )), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs index 65a1fb1cbd..b06988dae6 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs @@ -148,6 +148,7 @@ impl Session { if state.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // An earlier uncertain transaction may still exist. It cannot be // inherited by an independently owned snapshot or guessed complete. if !state.connection.is_autocommit() { @@ -207,6 +208,7 @@ impl Session { if state.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // An earlier uncertain transaction cannot be inherited or guessed done. if !state.connection.is_autocommit() { state.quarantined = true; @@ -249,6 +251,7 @@ impl Session { if s.quarantined { return Err(BackendError::Unknown.into()); } + self.check_identity().map_err(E::from)?; // A held publication lane already owns the writer for its nested work. if writable && !turn.nested { // State owns this before BEGIN, including across a poisoned unwind. diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index d90a7291b1..219d7b16e8 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -22,8 +22,8 @@ mod removal; mod rows; mod transaction; pub use kept::{KeptHistory, KeptNode, KeptNodeId}; -pub use objects::{KeptObjects, KeptWaste, MarkResult}; pub(crate) use lineage_backfill::backfill_lineage; +pub use objects::{GcTuple, KeptObjects, KeptWaste, MarkResult}; pub use provider::HistoryProvider; pub(crate) use rows::{ branch_row as diagnostic_branch, commit_graph_row as diagnostic_commit_graph, diff --git a/engine/crates/layerfs-persistence/src/history/objects.rs b/engine/crates/layerfs-persistence/src/history/objects.rs index 5a2af90032..30dd25402c 100644 --- a/engine/crates/layerfs-persistence/src/history/objects.rs +++ b/engine/crates/layerfs-persistence/src/history/objects.rs @@ -37,6 +37,7 @@ const LOCATIONS: &str = "gc_object_locations"; const LIVE_STACKS: &str = "gc_live_stacks"; const LIVE_BRANCHES: &str = "gc_live_branches"; const INODE_BLOCKS: &str = "gc_inode_blocks"; +const TUPLE: &str = "gc_tuple_read"; /// The bytes a mark keeps that no retained root actually needs (§6.2, §11). /// @@ -153,6 +154,76 @@ impl MarkResult { } } +/// The Store tuple a compaction records as its mark's first statement and +/// re-checks before the swap (§8 Phase B): the generation, the serial and pack +/// allocators, the four Store totals, the signature stamp and the history row +/// count the mark ran at. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct GcTuple { + /// The catalog incarnation (the Store generation G). + pub generation: i64, + /// The next history serial the mark saw. + pub next_history_serial: i64, + /// store_policy.next_pack_id at the mark. + pub next_pack_id: i64, + /// store_policy.next_ordinal at the mark. + pub next_ordinal: i64, + /// store_totals.logical at the mark. + pub logical: i64, + /// store_totals.canonical at the mark. + pub canonical: i64, + /// store_totals.stored at the mark. + pub stored: i64, + /// store_totals.metadata at the mark. + pub metadata: i64, + /// The highest content_signature stamp at the mark. + pub stamp: i64, + /// count(commit) + count(layer) at the mark. + pub history_rows: i64, +} + +impl HistoryProvider { + /// The T7.2 mark with the §8 mark tuple read as its first statement. + /// + /// The tuple and the closure are one write transaction: a publication that + /// lands before the tuple is read is part of the mark, and one that lands + /// after it moves at least one tuple field, so Phase B abandons. + pub fn mark_gc(&self) -> HistoryResult<(MarkResult, GcTuple)> { + self.write(mark_gc) + } + + /// The §6.4 live-heads digest of the Store as it is now, without a mark. + pub fn live_heads_digest(&self) -> HistoryResult<[u8; 32]> { + self.read(live_heads_digest) + } +} + +fn mark_gc(tx: &Transaction<'_>) -> HistoryResult<(MarkResult, GcTuple)> { + let tuple = gc_tuple(tx)?; + let mark = mark(tx)?; + Ok((mark, tuple)) +} + +fn gc_tuple(tx: &Transaction<'_>) -> HistoryResult { + let rows = tx.query(TUPLE, Vec::::new())?; + if rows.len() != 1 { + return Err(HistoryError::Integrity("Store tuple")); + } + let row = &rows[0]; + Ok(GcTuple { + generation: cell(row, 0)?, + next_history_serial: cell(row, 1)?, + next_pack_id: cell(row, 2)?, + next_ordinal: cell(row, 3)?, + logical: cell(row, 4)?, + canonical: cell(row, 5)?, + stored: cell(row, 6)?, + metadata: cell(row, 7)?, + stamp: cell(row, 8)?, + history_rows: cell(row, 9)?, + }) +} + impl HistoryProvider { /// Recomputes the §6.1 history closure, then the §6.2 object closure over /// its retained Commit and Layer roots, and the §6.4 live-heads digest. diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 09d0321814..fe158ec820 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -8,6 +8,7 @@ mod storage; mod store; pub(crate) use store::config; mod history; +mod report; #[cfg(feature = "postgresql")] pub use backend::postgresql::access::{access as postgresql_access, PostgresAccess}; #[cfg(feature = "postgresql")] @@ -16,10 +17,13 @@ pub use backend::postgresql::status::{ }; pub use backend::sqlite::acquisition::AcquisitionProvider; pub use backend::sqlite::compact::{ - CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, - CompactResidue, CompactZeroReason, + compact_op_status, CompactFailpoint, CompactKeptWaste, CompactOpStatus, CompactPath, + CompactReceipt, CompactRequest, CompactResidue, CompactZeroReason, }; pub use backend::sqlite::heads::{heads_digest, read_heads}; +pub use report::{ + CompactGeneration, CompactReport, CompactReportSource, CompactStatus, SQLITE_COMPACT_PHASES, +}; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use backend::sqlite::lifecycle::headroom::STORE_HEADROOM_BYTES; pub use backend::sqlite::lifecycle::reclamation::{SpaceReclamation, RECLAMATION_PAGE_LIMIT}; diff --git a/engine/crates/layerfs-persistence/src/report.rs b/engine/crates/layerfs-persistence/src/report.rs new file mode 100644 index 0000000000..0c6a1fa80e --- /dev/null +++ b/engine/crates/layerfs-persistence/src/report.rs @@ -0,0 +1,124 @@ +//! The §11 compaction report and the `store.status` compaction projection. +//! +//! One shape serves every backend. A SQLite [`CompactReceipt`] projects into +//! [`CompactReport`] through [`CompactReportSource`], and T9's PostgreSQL +//! rebuild implements that trait for its own receipt, so both paths share one +//! wire shape and one serializer. §11 names are logical; the rendered keys are +//! snake_case (§10). +use crate::backend::sqlite::compact::{ + CompactKeptWaste, CompactPath, CompactReceipt, CompactResidue, CompactZeroReason, +}; + +/// The phases a SQLite compaction passes, in order (§11): a run that crashes +/// leaves its record at the phase it reached. +pub const SQLITE_COMPACT_PHASES: [&str; 4] = ["mark", "build", "verify", "install"]; + +/// The generation a compaction started at and, when it installed a rebuild, +/// the one it left (`none` keeps both equal). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompactGeneration { + /// The generation the mark ran at. + pub before: u64, + /// The installed generation; equal to `before` on the `none` path. + pub after: u64, +} + +/// The §11 report `store.compact` returns, backend-neutral. +/// +/// SQLite fills it from [`CompactReceipt`]; T9's PostgreSQL rebuild fills the +/// same fields from its own receipt. A backend-only field is deliberately not +/// defined here: PostgreSQL reports its own zeros through the same keys. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactReport { + /// `none` or `rebuild`. + pub path: CompactPath, + /// The generation before and after. + pub generation: CompactGeneration, + /// Bytes in wholly dead packs after the metadata walk. + pub dead_bytes_bound: u64, + /// The same bytes after the payload sweep. + pub dead_bytes_refined: u64, + /// The collected history rows' `logical` bytes. + pub logical_discarded: u64, + /// Free pages × page size: the build's artifact, or the live `none` Store. + pub backend_reusable: u64, + /// Old file size minus the new one. + pub physical_shrink: u64, + /// Bytes the mark keeps that no retained root needs. + pub kept_waste: CompactKeptWaste, + /// Peak temporary bytes the run needed. + pub peak_temp_bytes: u64, + /// The artifact set left after settle; empty on success. + pub residue: CompactResidue, + /// §13: the report is an estimate, not a measurement. + pub estimate: bool, + /// One reason per zero report field, from the closed vocabulary. + pub zero_reasons: Vec, +} + +/// A backend receipt that projects to the one §11 [`CompactReport`]. +/// +/// SQLite's [`CompactReceipt`] implements it here. T9's PostgreSQL rebuild +/// implements it for the receipt of its Phase M/A/B/C run, which keeps the +/// PostgreSQL path on this shared shape without a PostgreSQL-only field. +pub trait CompactReportSource { + /// Projects this receipt as the §11 report. + fn compact_report(&self) -> CompactReport; +} + +impl CompactReportSource for CompactReceipt { + fn compact_report(&self) -> CompactReport { + CompactReport { + path: self.path, + generation: CompactGeneration { + before: self.generation_before, + after: self.generation_after, + }, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + logical_discarded: self.logical_discarded, + backend_reusable: self.backend_reusable, + physical_shrink: self.physical_shrink, + kept_waste: self.kept_waste.clone(), + peak_temp_bytes: self.peak_temp_bytes, + residue: self.residue.clone(), + estimate: self.estimate, + zero_reasons: self.zero_reasons.clone(), + } + } +} + +impl From<&CompactReceipt> for CompactReport { + fn from(receipt: &CompactReceipt) -> Self { + receipt.compact_report() + } +} + +impl From for CompactReport { + fn from(receipt: CompactReceipt) -> Self { + receipt.compact_report() + } +} + +/// The operation `store.status` shows while a compaction runs, or after a dead +/// owner left one (§11): `mark`, `build`, `verify` or `install`. +/// +/// It is read from `.compact-op` alone; the Store database, its +/// artifacts and the directory listing are never opened for it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactStatus { + /// The operation identity as `op-`. + pub id: String, + /// The call that owns it: `store.compact`. + pub name: &'static str, + /// The phase the run recorded. + pub phase: String, + /// When the run started, in Unix milliseconds. + pub started: i64, + /// Whether the outcome is unknown: a settled run removes its record. + pub uncertain: bool, + /// `busy` while the owner lives, `unresolved` once it is gone. + pub state: &'static str, + /// The backend's phases, in order, for a progress view. + pub phases: &'static [&'static str], +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json new file mode 100644 index 0000000000..c67cf099d2 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json @@ -0,0 +1,407 @@ +{ + "digestVector": { + "absentHeadDigest": "6864c4f336ed6ccfb6ea864bd565b23b97fbbff571e6b627d0043bce7218a724", + "commit": [ + "121c28e0f8ea5f06e9f3a5ad08f7f2bd7b2ad1afd677d57099991f41678ab19ec1" + ], + "retiredBranchDigest": "dafcdd21820991184e6f21e7c80cbdd0f034fdc1d62ebf9502aeae381770c857", + "retiredStack": [ + "3107070707070707070707070707070707" + ] + }, + "emptyNone": { + "mark": { + "liveHeadsDigest": "3556f439f3ea4d21d189c484e3c117492779e5c62249481882cd6011cf1a13a9", + "objectCount": 0, + "packs": [] + }, + "receipt": { + "backendReusable": 0, + "deadBytesBound": 0, + "deadBytesRefined": 0, + "estimate": true, + "generationAfter": 7, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 0, + "deltaBases": { + "bytes": 0, + "references": [] + }, + "inodeBlocksBytes": 0 + }, + "logicalDiscarded": 0, + "path": "none", + "peakTempBytes": 0, + "physicalShrink": 0, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "deadBytesBound", + "reason": "empty-store" + }, + { + "field": "deadBytesRefined", + "reason": "empty-store" + }, + { + "field": "logicalDiscarded", + "reason": "no-history" + }, + { + "field": "physicalShrink", + "reason": "none-path" + }, + { + "field": "backendReusable", + "reason": "none-path" + }, + { + "field": "keptWaste", + "reason": "no-waste" + }, + { + "field": "peakTempBytes", + "reason": "none-path" + } + ] + }, + "totalsAfter": { + "canonical": 0, + "logical": 0, + "metadata": 0, + "stored": 0 + }, + "totalsBefore": { + "canonical": 0, + "logical": 0, + "metadata": 0, + "stored": 0 + } + }, + "noDeadNone": { + "commit": [ + "120dee8be25262ff5277cf6682a1b66b1157c2f1aa1ed551e3a3c140ec4d41e250" + ], + "layer": [ + "3228ecf26544a2fea9ffeecc6829ab584e2a4fa4f5e7be920696d98ed8d843e749" + ], + "mark": { + "liveHeadsDigest": "b04ee485d7c6002f22a8ab6d12c206ad84254435fb1067e952883c8f3012128d", + "objectCount": 8, + "packs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ] + }, + "receipt": { + "backendReusable": 32768, + "deadBytesBound": 0, + "deadBytesRefined": 0, + "estimate": true, + "generationAfter": 7, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 0, + "deltaBases": { + "bytes": 0, + "references": [] + }, + "inodeBlocksBytes": 190 + }, + "logicalDiscarded": 0, + "path": "none", + "peakTempBytes": 0, + "physicalShrink": 0, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "deadBytesBound", + "reason": "no-waste" + }, + { + "field": "deadBytesRefined", + "reason": "no-waste" + }, + { + "field": "logicalDiscarded", + "reason": "none-path" + }, + { + "field": "physicalShrink", + "reason": "none-path" + }, + { + "field": "peakTempBytes", + "reason": "none-path" + } + ] + }, + "totalsAfter": { + "canonical": 890, + "logical": 410, + "metadata": 808, + "stored": 936 + }, + "totalsBefore": { + "canonical": 890, + "logical": 410, + "metadata": 808, + "stored": 936 + } + }, + "rebuild": { + "after": { + "branchRetired": [ + "1101010101010101010101010101010101" + ], + "deadPacksGone": true, + "fileDigestCollected": false, + "fileDigestRetained": true, + "fullDoctor": "clean", + "layerStackRetired": 0, + "liveHeadsDigestUnchanged": true, + "liveReadable": true + }, + "fixture": { + "branches": { + "collected": [ + "1103030303030303030303030303030303", + "1104040404040404040404040404040404" + ], + "live": [ + "1102020202020202020202020202020202" + ], + "retainedRetired": [ + "1101010101010101010101010101010101" + ] + }, + "commits": { + "collected": [ + "12504118308814458073cf694c832277a9a3edf25438d6c29033749d7d4b6851e9", + "12f41169281d83c82e136fb7a69850b3d72ea90ef637dd4145bdbf1077cbafbb45" + ], + "retained": [ + "12ae744d795440a3f12341548e1f1ae118e384450d2454aa6e1bf952fc5fa5b9e5", + "129c35a564ebb33405e16e5ebc548aa5e01640914517db7a7a2948c0c22e8600e6", + "128512651dcea9f37272d401d22f19aefcac65d46f4a66dcf9596078ce4f1efb2f" + ] + }, + "deadBytesBound": 182635, + "deltaBaseReferences": [ + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85" + ], + "deltaChainPackIds": [ + 25, + 26, + 27 + ], + "inodeBlocksBytes": 665, + "layers": { + "collected": [ + "32c89ce4fa2bda2b0fc9ef449faa224fe05895352e5333ec7fe9fb4e35fb8d5409", + "32e7c02e338721609c58801a6195a8f9ececb7875b3715a6356f811534f9185dda" + ], + "retained": [ + "32e2fce8a15669a223b821c5a12720915f959a3bd8b678843a2d4c836f49973d79", + "3228fe19ff26677849bef9617a26fba8c9d70b6bb0f30f4ceb35dac89164bba157", + "32219e07e8572a40e056647a2db0febee7cff5e951a91974ce130bf7534d4f5ed9" + ] + }, + "objects": { + "dead": [ + "04da3ee0d5c26526b82eaacaa3611fb5ee51b53938b15b9a6e0c2ce23fac79aa", + "03cdb1082aefd8630de9ded4491f70107d1194879c3725f47604e61d6ce8464f", + "b41d84c180ddbf811874a45b54f6bb5929351b9d922e3eae16030b93ef0c0465", + "93934455f81c303589782508b25a6fc6cb5577041d09f9997dbdf9a4443da90e", + "1ccae865928d331b9f40f6b9781de1a8f2fc131c79a46b18de977a92e1fc41f6", + "d330e6950c00b9c6e5214a911a0b7438590ce84c9f732d9a5dcf0337a3ee3b3c", + "9e1ae9f693108d70236c4b834eae7066e95a1c6c88319598521b3a30178b335b", + "9b4f54e459f725c48c96be90af12c6918e3135964d38629313fe09fa548c2c63", + "c260b43bb74755139dd041a0906db269d0ed8039181fcb0d1ec6226ae1e1638e", + "1d0d2e8c5f735a3437e84a60119ae1daa08a34d769969c460a7a7695361bd88b", + "aa2d83fe8c3e636f1c8086bcf96bdee0081bd564ed730f627e1bf130d3f6725b", + "c7885caa9ac150fca12ca3352cec6e1a9ab8d585332e3def1ccb084cdb710368", + "8e65a309ef2a363965849a9b19bb3b005e565d4a678f09893d721a8031d9977e", + "cc9d4bace21b47206e4cacdacc1dcce819210b79787c33a71c1bdfd553bca024", + "1352f9c655388061372fe949b37530b1284bc50c4cd7f979ff25d848e42a43ac", + "968c266c9fd518965edb975d40714441b48b142ea1bef48ad6164da39f307a80", + "773f0e9363afec9a9bc0294e492e84e9c2985de50fcb21294b6762774309e5b5", + "a96b6772f5b6c00682d3e8545e583c6f99f3e404015efb471a8c5ccb4d42f1e4", + "90bd063fc258fce027928d7d05334f076868c9fa5effe998c09a18c79b9b3c1b" + ], + "live": [ + "bea0e9f6fdfdb07ff631ac8213c85caf98cba1ff2aeb342fe7f99ff210b1498e", + "f7b81686c824c3de2c23077b02f554936618e20621719a0682c9430358290f73", + "4ed7a5abe47b79b9c281d7bbd914e858162d5d005c3aff0e05c5b1bf484e3403", + "3df6d13237972839afaa61737e92f27e60793cc25d61b7f8515b6f8374d3dc2d", + "43652a535c7f61a5caef1d1574a661b4c19b02d848d6463430d933920c148152", + "b57fc5ee3a4f8270aecd33c137a39b68336669fa1d06f11acb8f02e6ff6e6849", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85", + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "3806265ed5cd11005a9918e03c34377954012805cdfe908877678c1395111776", + "01fd2163213bdea11e96e3fb9f27ea76a6112497895d7e4751f541509d5b7e25" + ] + }, + "packs": { + "dead": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 19, + 20, + 21, + 22, + 23, + 24, + 29, + 30, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52 + ], + "live": [ + 13, + 14, + 15, + 16, + 17, + 18, + 25, + 26, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43 + ], + "shared": 28, + "sharedDeadGroupBytes": 265, + "sharedLength": 562 + }, + "stacks": { + "collected": [ + "3102020202020202020202020202020202" + ], + "live": [ + "3101010101010101010101010101010101" + ] + } + }, + "mark": { + "liveHeadsDigest": "d3a3eadd25ce23512bc358a91b3195ee8d29ee027f83840d567dc6540f47bab9", + "objectCount": 23, + "packs": [ + 13, + 14, + 15, + 16, + 17, + 18, + 25, + 26, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43 + ] + }, + "receipt": { + "backendReusable": 0, + "deadBytesBound": 182635, + "deadBytesRefined": 182635, + "estimate": true, + "generationAfter": 8, + "generationBefore": 7, + "keptWaste": { + "deadInRetainedPacksBytes": 265, + "deltaBases": { + "bytes": 571, + "references": [ + "d5f0864e883f3636c1d1e60fd9a40c4a7942776d635958b01d89dfe914953d50", + "e7069dfd5e4a6720d9e7a8dfb74e3eda95b0cc6c798a8942d0b956ac919e8c85" + ] + }, + "inodeBlocksBytes": 665 + }, + "logicalDiscarded": 160, + "path": "rebuild", + "peakTempBytes": 389120, + "physicalShrink": 221184, + "residue": { + "artifacts": [], + "bytes": 0, + "reason": null + }, + "zeroReasons": [ + { + "field": "backendReusable", + "reason": "no-waste" + } + ] + }, + "totalsAfter": { + "canonical": 134988, + "logical": 1090, + "metadata": 2024, + "stored": 134670 + }, + "totalsBefore": { + "canonical": 317453, + "logical": 1250, + "metadata": 3644, + "stored": 317305 + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs new file mode 100644 index 0000000000..acdcebce6a --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs @@ -0,0 +1,582 @@ +//! #41 T9: the §8 PostgreSQL rebuild. A compaction takes the Store-scoped +//! exclusive intent, records the mark tuple, builds the retained payload in a +//! scratch schema, swaps `pack`/`object_location` at `G -> G + 1`, deletes only +//! the persisted `gc_dead` set, settles the totals once, and leaves no +//! scratch schema or op row. A leftover at `G` is removed; one at `G + 1` +//! finishes Phase C (lost-reply resume). A conflicting lock is `Busy` (55P03). +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRequest, + CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, LayerStackId, + LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use support::{branch_id, name, stack, BINDING}; + +/// Drops the test's schema when the test ends, pass or fail. +struct Schema { + url: String, + selection: PostgresSelection, +} +impl Schema { + fn new(url: &str, tag: &str) -> Self { + let name = format!("lfs_t9_{tag}_{}", std::process::id()); + let selection = PostgresSelection::new(url, name); + let schema = Self { + url: url.to_owned(), + selection, + }; + schema.drop_schema(); + schema + } + fn open(&self) -> PostgresHandles { + PostgresHandles::open_writable(&self.selection, BINDING, [71; 32]).expect("Store open") + } + fn create(&self) -> PostgresHandles { + PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) + .expect("Store creation") + } + fn client(&self) -> postgres::Client { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); + client + .batch_execute(&format!("SET search_path TO {}", self.selection.schema)) + .expect("search path"); + client + } + fn count(&self, table: &str) -> i64 { + let sql = format!("SELECT count(*) FROM {}.{}", self.selection.schema, table); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn count_where(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn scalar(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("scalar").get(0) + } + fn exists(&self, sql: &str) -> bool { + self.scalar(sql) != 0 + } + fn scratch_exists(&self, op: u8) -> bool { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + self.exists(&format!( + "SELECT (CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname='{scratch}') THEN 1 ELSE 0 END)::bigint" + )) + } + fn drop_scratch(&self, op: u8) { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!("DROP SCHEMA IF EXISTS {scratch} CASCADE")); + } + } + fn drop_schema(&self) { + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!( + "DROP SCHEMA IF EXISTS {} CASCADE", + self.selection.schema + )); + } + } +} +impl Drop for Schema { + fn drop(&mut self) { + self.drop_schema(); + } +} + +fn catalog() -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + } +} + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &PostgresHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &PostgresHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: Vec::new(), + }) + .unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &PostgresHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: stack(tag), + name: name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, +) -> layerfs_history::CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn located(handles: &PostgresHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn generation(handles: &PostgresHandles) -> u64 { + handles.history.incarnation() +} + +/// The live generation from the Store row (the open session caches the value +/// it opened at, as it does on SQLite). +fn db_generation(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT catalog_incarnation FROM {s}.history_meta WHERE id=1")) + .unwrap() +} + +fn clean(handles: &PostgresHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +fn assert_settled(schema: &Schema, op: u8) { + assert_eq!(schema.count("gc_op"), 0, "gc_op is cleared on settle"); + assert_eq!(schema.count("gc_dead"), 0, "gc_dead is cleared on settle"); + assert_eq!( + schema.count_where("SELECT count(*) FROM {s}.gc_intent WHERE kind='compaction'"), + 0, + "the compaction intent is released" + ); + assert!( + !schema.scratch_exists(op), + "the recorded scratch schema is dropped" + ); + let logical = schema.scalar("SELECT logical FROM {s}.store_totals WHERE id=1"); + let sum = schema.scalar( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.layer)", + ); + assert_eq!(logical, sum, "store_totals.logical tracks the history rows"); + let canonical = schema.scalar("SELECT canonical FROM {s}.store_totals WHERE id=1"); + let canonical_sum = + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"); + assert_eq!(canonical, canonical_sum, "store_totals.canonical"); + let stored = schema.scalar("SELECT stored FROM {s}.store_totals WHERE id=1"); + let stored_sum = schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack"); + assert_eq!(stored, stored_sum, "store_totals.stored"); + let metadata = schema.scalar("SELECT metadata FROM {s}.store_totals WHERE id=1"); + let metadata_sum = + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"); + assert_eq!(metadata, metadata_sum, "store_totals.metadata"); +} + +#[test] +fn an_empty_store_takes_the_none_path() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "empty"); + let handles = schema.create(); + let before = generation(&handles); + let receipt = handles.compact(&request(1)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "backendReusable" && zero.reason == "postgresql-in-place")); + assert_settled(&schema, 1); + assert_eq!(generation(&handles), before); + handles.history.mark().unwrap(); +} + +#[test] +fn a_live_store_takes_the_none_path() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "live"); + let handles = schema.create(); + let genesis = filesystem_root(&handles, 1); + initialize(&handles, 1, genesis); + let before = generation(&handles); + let receipt = handles.compact(&request(2)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert_eq!(receipt.generation_after, before); + assert_eq!(receipt.physical_shrink, 0); + assert_settled(&schema, 2); + assert!(located(&handles, genesis)); +} + +#[test] +fn a_wholly_dead_pack_is_rebuilt_away_and_totals_settle() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "rebuild"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"nobody reaches this object"); + assert!(located(&handles, dead)); + let before = generation(&handles); + let receipt = handles.compact(&request(3)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.dead_bytes_bound > 0); + assert_eq!(receipt.backend_reusable, 0); + assert_settled(&schema, 3); + assert_eq!(db_generation(&schema), before + 1); + assert!(!located(&handles, dead), "the dead object is collected"); + assert!(located(&handles, live), "the retained object survives"); + clean(&schema.open()); +} + +#[test] +fn retained_roots_survive_a_rebuild_that_collects_a_deleted_project() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "retained"); + let handles = schema.create(); + let live_genesis = filesystem_root(&handles, 1); + let live_commit = filesystem_root(&handles, 2); + let live = initialize(&handles, 1, live_genesis); + let work = branch_id(1); + fork(&handles, live.id, work, 1); + let retained_commit = commit(&handles, work, live_commit); + publish(&handles, live.id, work); + + let dead_genesis = filesystem_root(&handles, 3); + let dead_commit = filesystem_root(&handles, 4); + let gone = initialize(&handles, 2, dead_genesis); + let dead_branch = branch_id(2); + fork(&handles, gone.id, dead_branch, 2); + let collected_commit = commit(&handles, dead_branch, dead_commit); + publish(&handles, gone.id, dead_branch); + handles.history.mark_stack_deleted(gone.id, 5_000).unwrap(); + + let before = generation(&handles); + let receipt = handles.compact(&request(4)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert!( + receipt.logical_discarded > 0, + "the deleted project's rows are collected" + ); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 4); + // The retained project and its history survive. + assert!(handles.history.layer_stack(live.id).unwrap().is_some()); + assert!(handles.history.branch(work).unwrap().is_some()); + assert!(handles.history.commit(retained_commit).unwrap().is_some()); + assert!(located(&handles, live_genesis)); + assert!(located(&handles, live_commit)); + // The deleted project's history and objects are gone. + assert!(handles.history.layer_stack(gone.id).unwrap().is_none()); + assert!(handles.history.commit(collected_commit).unwrap().is_none()); + assert!(!located(&handles, dead_genesis)); + assert!(!located(&handles, dead_commit)); + clean(&schema.open()); +} + +#[test] +fn a_lost_swap_reply_is_resumed_by_the_next_compact() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "lost"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack collected by the first round"); + let before = generation(&handles); + let error = handles + .compact(&request_at(5, CompactFailpoint::PostgresAfterSwap)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + // The swap committed: the Store is at G + 1 and the op still names Phase C. + assert_eq!(db_generation(&schema), before + 1); + assert_eq!(schema.count("gc_op"), 1); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + let receipt = handles.compact(&request(6)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 6); + assert_eq!(db_generation(&schema), before + 1); + clean(&schema.open()); +} + +#[test] +fn the_same_session_resumes_an_uncertain_round() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "same"); + schema.drop_scratch(11); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the resumed round collects"); + let before = generation(&handles); + let error = handles + .compact(&request_at(11, CompactFailpoint::PostgresAfterMark)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + // The same session is the recorded owner and is idle; it adopts its own + // uncertain round instead of terminating itself. + let receipt = handles.compact(&request(12)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 12); + clean(&schema.open()); +} + +#[test] +fn a_stale_owner_backend_is_terminated_before_the_next_round() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "terminate"); + schema.drop_scratch(9); + let first = schema.create(); + let live = filesystem_root(&first, 1); + initialize(&first, 1, live); + let _dead = whole_file(&first, b"a pack the adopted round collects"); + let before = generation(&first); + let error = first + .compact(&request_at(9, CompactFailpoint::PostgresAfterMark)) + .unwrap_err(); + assert!(matches!( + error, + layerfs_history::HistoryError::UnknownOutcome + )); + let pid = schema.scalar("SELECT pid FROM {s}.gc_op WHERE id=1"); + assert!(pid > 0, "the op records its owner backend"); + // The recorded owner is alive but idle: the next round terminates that + // (pid, backend_start) and adopts the op as a dead owner (§9, §12). + let second = schema.open(); + let receipt = second.compact(&request(10)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_eq!(db_generation(&schema), before + 1); + assert_settled(&schema, 10); + let mut gone = false; + for _ in 0..50 { + if schema.scalar(&format!( + "SELECT count(*) FROM pg_stat_activity WHERE pid={pid}" + )) == 0 + { + gone = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + assert!(gone, "the stale owner backend is terminated"); + clean(&schema.open()); +} + +#[test] +fn an_exclusive_table_lock_makes_the_swap_busy() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "busy"); + schema.drop_scratch(7); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + // Phase A never touches store_totals; the swap's ACCESS EXCLUSIVE NOWAIT + // does, so this lock is met exactly at the swap (SQLSTATE 55P03). + let mut blocker = schema.client(); + blocker + .batch_execute(&format!( + "BEGIN; LOCK TABLE {}.binding_hint_block IN ACCESS EXCLUSIVE MODE;", + schema.selection.schema + )) + .unwrap(); + let error = handles.compact(&request(7)).unwrap_err(); + blocker.batch_execute("ROLLBACK").unwrap(); + assert!( + matches!(error, layerfs_history::HistoryError::Busy), + "a 55P03 lock at the swap is Busy, got {error:?}" + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs index 6b4578cded..8445f4f917 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -10,15 +10,18 @@ use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, Obje use layerfs_history::{ AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, - LayerStackId, LayerStackRecord, StackInitialization, + HistoryError, LayerStackId, LayerStackRecord, StackInitialization, }; use layerfs_persistence::{ - CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, + compact_op_status, CompactFailpoint, CompactPath, CompactReport, CompactReportSource, + CompactRequest, PersistenceConfig, SqliteAcquisitionSchema, SqliteHandles, SqlitePackLayout, SqlitePersistenceProfile, }; use layerfs_storage::{ + port::acquisition::{Acquisition, Begin}, port::PackPersistence, Storage, StoragePolicy, }; +use std::cell::RefCell; use std::path::{Path, PathBuf}; use support::Temp; @@ -433,6 +436,117 @@ fn an_installed_but_unsettled_run_is_judged_at_g_plus_one() { assert!(!compact_path(&path).exists()); } +#[test] +fn the_report_projects_every_section_11_field() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let receipt = handles.compact(&request(30)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let report = CompactReport::from(&receipt); + assert_eq!(report.path, CompactPath::Rebuild); + assert_eq!(report.generation.before, before); + assert_eq!(report.generation.after, before + 1); + assert_eq!(report.dead_bytes_bound, receipt.dead_bytes_bound); + assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); + assert_eq!(report.logical_discarded, receipt.logical_discarded); + assert_eq!(report.backend_reusable, receipt.backend_reusable); + assert_eq!(report.physical_shrink, receipt.physical_shrink); + assert_eq!(report.kept_waste, receipt.kept_waste); + assert_eq!(report.peak_temp_bytes, receipt.peak_temp_bytes); + assert_eq!(report.residue, receipt.residue); + assert!(report.estimate); + assert_eq!(report.zero_reasons, receipt.zero_reasons); + assert!( + report + .zero_reasons + .iter() + .all(|zero| zero.wire_field() != "dead_bytes_bound"), + "a rebuild with dead bytes does not explain a zero" + ); + // `From` and the shared source trait project the same report. + assert_eq!(report, receipt.compact_report()); +} + +#[test] +fn the_empty_store_report_explains_its_zeros() { + let (_temp, handles, path) = create_store(); + let receipt = handles.compact(&request(31)).unwrap(); + let report = CompactReport::from(&receipt); + assert_eq!(report.path, CompactPath::None); + assert_eq!(report.generation.before, report.generation.after); + assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); + assert!(report.estimate); + assert!( + report.zero_reasons.iter().any(|zero| { + zero.field == "deadBytesBound" + && zero.wire_field() == "dead_bytes_bound" + && zero.reason == "empty-store" + }), + "the empty store names its zero: {:?}", + report.zero_reasons + ); + assert!(report.residue.artifacts.is_empty()); + assert!(compact_op_status(&path).is_none(), "none settles its op"); +} + +#[test] +fn a_leftover_op_reports_its_phase_and_state() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let before = generation(&handles); + let error = handles + .compact(&CompactRequest { + op_id: [21; 16], + owner_pid: Some(std::process::id()), + failpoint: Some(CompactFailpoint::AfterBuild), + ..CompactRequest::default() + }) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + let op = compact_op_status(&path).expect("the leftover op record is readable"); + assert_eq!(op.op_id, [21; 16]); + assert_eq!(op.generation, i64::try_from(before).unwrap()); + assert_eq!(op.phase, "build"); + assert_eq!(op.owner_pid, Some(std::process::id())); + assert!(op.started_ms > 0); + + let busy = op.status(true); + assert_eq!(busy.id, "op-15151515151515151515151515151515"); + assert_eq!(busy.name, "store.compact"); + assert_eq!(busy.phase, "build"); + assert_eq!(busy.started, op.started_ms); + assert!(busy.uncertain, "an unsettled record is uncertain"); + assert_eq!(busy.state, "busy"); + assert_eq!( + busy.phases, + &["mark", "build", "verify", "install"][..], + "the SQLite phase order" + ); + + let leftover = op.status(false); + assert!(leftover.uncertain); + assert_eq!(leftover.state, "unresolved"); + assert_eq!(leftover.phase, "build"); + + // The next run settles the recorded leftover: artifact, intent and op go. + let reopened = reopen(&path); + let receipt = reopened.compact(&request(22)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert!(compact_op_status(&path).is_none(), "the op record is settled"); + assert!(receipt.residue.artifacts.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.reason.is_none()); + assert!(!compact_path(&path).exists()); +} + #[test] fn generation_increases_once_per_installed_rebuild() { let (_temp, handles, path) = create_store(); @@ -454,3 +568,298 @@ fn generation_increases_once_per_installed_rebuild() { assert!(located(&final_handles, first)); clean(&final_handles); } + +// --- Fences, thresholds, grouped and acquisition layouts (T8.1) --- + +thread_local! { + /// The Store file the install-fence test renames over the sealed path. + static REPLACEMENT: RefCell> = const { RefCell::new(None) }; +} + +/// The test-only hook CompactRequest::before_install_hook runs: replace the +/// Store path with the prepared file after the seal and before the rename. +fn replace_store(path: &Path) { + let replacement = REPLACEMENT + .with(|slot| slot.borrow_mut().take()) + .expect("a prepared replacement"); + std::fs::rename(&replacement, path).expect("replace the sealed Store"); +} + +/// The stored length of every pack, read after the Store is closed. +fn pack_lengths(path: &Path) -> Vec { + let connection = + rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("read-only Store"); + let mut statement = connection + .prepare("SELECT length FROM pack ORDER BY pack_id") + .expect("pack rows"); + let rows = statement + .query_map([], |row| row.get::<_, i64>(0)) + .expect("pack lengths"); + rows.map(|row| u64::try_from(row.expect("a pack length")).expect("nonnegative")) + .collect() +} + +/// Every acquisition working row the rebuild must clear. +fn acquisition_rows(path: &Path) -> i64 { + let connection = + rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("read-only Store"); + ["init_operation", "init_entry", "init_host_file"] + .iter() + .map(|table| { + connection + .query_row(&format!("SELECT count(*) FROM {table}"), [], |row| { + row.get::<_, i64>(0) + }) + .expect("acquisition row count") + }) + .sum() +} + +fn create_with( + layout: SqlitePackLayout, + acquisition: SqliteAcquisitionSchema, +) -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-layout"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_pack_layout(layout) + .with_sqlite_acquisition(acquisition), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +/// One Store whose live whole-file roots are retained and whose dead +/// whole-file packs no retained root reaches. +/// +/// Every object has the same canonical length, so every pack has the same +/// stored length: the wholly dead byte ratio is dead / (live + dead) exactly. +fn ratio_store(live: usize, dead: usize) -> (Temp, SqliteHandles, PathBuf) { + let (temp, handles, path) = create_store(); + for index in 0..live { + let root = whole_file(&handles, format!("live-{index:06}").as_bytes()); + initialize(&handles, u8::try_from(index + 1).unwrap(), root); + } + for index in 0..dead { + let _ = whole_file(&handles, format!("dead-{index:06}").as_bytes()); + } + (temp, handles, path) +} + +#[test] +fn nine_percent_wholly_dead_bytes_take_the_none_path() { + let (_temp, handles, path) = ratio_store(91, 9); + let before = generation(&handles); + let receipt = handles.compact(&request(20)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_after, before); + assert!(!compact_path(&path).exists()); + assert!(!op_path(&path).exists()); + let lengths = pack_lengths(&path); + assert_eq!(lengths.len(), 100, "one pack per object: {lengths:?}"); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 9 * unit); + assert!( + 9 * unit * 10 < 100 * unit, + "9% wholly dead bytes are below the threshold" + ); +} + +#[test] +fn ten_percent_wholly_dead_bytes_rebuild_at_the_threshold() { + let (_temp, handles, path) = ratio_store(90, 10); + let before = generation(&handles); + let receipt = handles.compact(&request(21)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + let lengths = pack_lengths(&path); + assert_eq!( + lengths.len(), + 90, + "ten dead packs are collected: {lengths:?}" + ); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 10 * unit); + assert!( + 10 * unit * 10 >= (90 + 10) * unit, + "exactly 10% wholly dead bytes take the rebuild path" + ); +} + +#[test] +fn thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold() { + let (_temp, handles, path) = ratio_store(70, 30); + let before = generation(&handles); + let receipt = handles.compact(&request(22)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let lengths = pack_lengths(&path); + assert_eq!(lengths.len(), 70, "thirty dead packs are collected"); + let unit = lengths[0]; + assert!(lengths.iter().all(|length| *length == unit)); + assert_eq!(receipt.dead_bytes_bound, 30 * unit); + assert!(30 * unit * 10 > (70 + 30) * unit); +} + +#[test] +fn a_store_replaced_between_seal_and_install_is_stale() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"a pack the rebuild would collect"); + + // A second valid Store at the same generation: replacing the path with it + // changes the (dev, ino) pair and leaves size, mtime and counter unrelated. + let (_replacement_temp, replacement_handles, replacement_path) = create_store(); + let replacement_generation = generation(&replacement_handles); + replacement_handles.seal().unwrap(); + REPLACEMENT.with(|slot| *slot.borrow_mut() = Some(replacement_path)); + + let request = CompactRequest { + op_id: [23; 16], + before_install_hook: Some(replace_store), + ..CompactRequest::default() + }; + let error = handles.compact(&request).unwrap_err(); + assert!(matches!(error, HistoryError::StaleStore), "{error:?}"); + + // No install: every artifact is gone and the replacement is untouched. + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), replacement_generation); +} + +#[test] +fn a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + let (_replacement_temp, replacement_handles, replacement_path) = create_store(); + replacement_handles.seal().unwrap(); + std::fs::rename(&replacement_path, &path).unwrap(); + + // The handle still owns the old inode; the fence refuses before anything runs. + let error = handles.history.mark().unwrap_err(); + assert!(matches!(error, HistoryError::StaleStore), "{error:?}"); +} + +#[test] +fn a_capacity_failure_leaves_the_store_unchanged() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let error = handles + .compact(&request_at(24, CompactFailpoint::CapacityAtBuild)) + .unwrap_err(); + assert!(matches!(error, HistoryError::Capacity(_)), "{error:?}"); + assert!(!compact_path(&path).exists()); + assert!(!visited_path(&path).exists()); + assert!(!op_path(&path).exists()); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before, "the old Store is unchanged"); + assert!(located(&reopened, dead), "nothing was collected"); + let receipt = reopened.compact(&request(25)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +fn layout_round_trip(layout: SqlitePackLayout) { + let (_temp, handles, path) = create_with(layout, SqliteAcquisitionSchema::Absent); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + // The none path: the stored layout opens and a mark-only run leaves it. + let receipt = handles.compact(&request(26)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + let reopened = reopen(&path); + assert_eq!(reopened.profile().pack_layout, layout); + + // A rebuild collects a wholly dead pack and preserves the layout. + let dead = whole_file(&reopened, b"a wholly dead pack in a grouped layout"); + let before = generation(&reopened); + let receipt = reopened.compact(&request(27)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let final_handles = reopen(&path); + assert_eq!(final_handles.profile().pack_layout, layout); + assert!(located(&final_handles, live)); + assert!(!located(&final_handles, dead)); + clean(&final_handles); +} + +#[test] +fn group_rows_layout_opens_compacts_and_takes_none() { + layout_round_trip(SqlitePackLayout::GroupRows); +} + +#[test] +fn group_rows_indexed_layout_opens_compacts_and_takes_none() { + layout_round_trip(SqlitePackLayout::GroupRowsIndexed); +} + +#[test] +fn acquisition_layout_opens_compacts_and_clears_working_rows() { + let (_temp, handles, path) = + create_with(SqlitePackLayout::GroupRows, SqliteAcquisitionSchema::Tables); + assert_eq!( + handles.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + // A live acquisition operation: the rebuild must clear its working rows. + let _owner = handles + .acquisition + .begin(&Begin { + source_device: 1, + source_inode: 2, + stack: [3; 16], + scope: ObjectId::for_bytes(b"acquisition scope"), + }) + .unwrap(); + + // The none path leaves the acquisition tables and their working rows. + let receipt = handles.compact(&request(28)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + let reopened = reopen(&path); + assert_eq!( + reopened.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + + // A rebuild clears the working rows and keeps the acquisition tables. + let dead = whole_file(&reopened, b"a wholly dead pack with acquisition tables"); + let before = generation(&reopened); + let receipt = reopened.compact(&request(29)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + let final_handles = reopen(&path); + assert_eq!( + final_handles.profile().acquisition, + SqliteAcquisitionSchema::Tables + ); + assert!(located(&final_handles, live)); + assert!(!located(&final_handles, dead)); + assert_eq!(acquisition_rows(&path), 0, "working rows are cleared"); + clean(&final_handles); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs new file mode 100644 index 0000000000..79751dd869 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs @@ -0,0 +1,577 @@ +//! #41 T12.2: the §9 crash matrix for SQLite `store.compact`. +//! +//! Each case aborts at one named failpoint, reopens the Store and asserts the +//! documented crash state: wholly old at `G` or wholly new at `G + 1`, the +//! exact artifact set, the live objects still readable, the admission record +//! and the op/phase/uncertain `store.status` projection. It then runs the next +//! `store.compact` and asserts it settles (installed) or rebuilds (pre-install) +//! and leaves a clean Store. +//! +//! The expectations below are backend-neutral data; a PostgreSQL harness can +//! reuse the same table with its own failpoints and its own artifact set. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CatalogId, CommitId, CommitMeta, CommitOutcome, + CommitRequest, CommitVia, ForkRequest, ForkSource, GcIntentKind, GcScopeKind, HistoryCatalog, + HistoryCatalogConfig, LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + compact_op_status, CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, + SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use std::path::{Path, PathBuf}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-crash-binding"; + +/// One crash point: the failpoint, the phase `store.status` must show, whether +/// the rebuild is already installed at `G + 1`, and which staged artifacts the +/// crash leaves beside the Store. +#[derive(Clone, Copy, Debug)] +struct Case { + failpoint: CompactFailpoint, + phase: &'static str, + installed: bool, + target: bool, + visited: bool, +} + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-crash"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: [u8; 16], failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: op, + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact") +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-op") +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-visited") +} + +fn catalog_id() -> CatalogId { + CatalogId::derive(BINDING).expect("catalog identity") +} + +fn intents(handles: &SqliteHandles) -> Vec { + handles + .history + .gc_intents(GcScopeKind::Store, catalog_id().as_slice()) + .unwrap() +} + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + save.accept(object.clone()).unwrap(); + save.finish().unwrap(); + object.id() +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: Vec::new(), + }) + .unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + let root_id = object.id(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + for object in [directory, inode, object] { + save.accept(object).unwrap(); + } + save.finish().unwrap(); + root_id +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +#[allow(dead_code)] +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +#[allow(dead_code)] +fn commit(handles: &SqliteHandles, branch: BranchId, candidate_root: ObjectId) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: layerfs_content::filesystem::root::profile_id(), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +#[allow(dead_code)] +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { .. } => {} + other => panic!("publish: {other:?}"), + } +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn object_bytes(handles: &SqliteHandles, id: ObjectId) -> Vec { + let storage = Storage::new(handles.storage.clone()).unwrap(); + storage + .reader() + .unwrap() + .read_objects(&[id]) + .unwrap() + .into_iter() + .next() + .expect("one object") +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +fn assert_no_sidecars(store: &Path, case: Case) { + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(store, suffix).exists(), + "sidecar {suffix} beside {} at {case:?}", + store.display() + ); + } +} + +fn exercise(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + let op = [41u8; 16]; + let error = handles + .compact(&request_at(op, case.failpoint)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + + // `store.status` (T10.1) projects the recorded operation, its phase and + // its uncertain outcome; the engine reads them from the op record. + let status = compact_op_status(&path).expect("the op record remains"); + assert_eq!(status.op_id, op); + assert_eq!(status.generation, i64::try_from(before).unwrap()); + assert_eq!(status.phase, case.phase, "phase at {case:?}"); + assert!( + status.uncertain, + "an unfinished run is uncertain at {case:?}" + ); + + // The exact artifact set the point leaves beside the live Store. + assert_eq!( + compact_path(&path).exists(), + case.target, + "staged Store at {case:?}" + ); + assert_eq!( + visited_path(&path).exists(), + case.visited, + "visited scratch at {case:?}" + ); + assert!(op_path(&path).exists(), "op record at {case:?}"); + assert_no_sidecars(&path, case); + assert_no_sidecars(&compact_path(&path), case); + + let reopened = reopen(&path); + + // The admission intent is released before the seal (T-G5), so only a + // crash before that release still leaves it; a rebuilt Store was copied + // without it (`build` clears `gc_intent`). + let listed = intents(&reopened); + if case.failpoint == CompactFailpoint::BeforeSeal { + assert_eq!(listed.len(), 1, "admission intent at {case:?}"); + assert_eq!(listed[0].op_id, op); + assert_eq!(listed[0].kind, GcIntentKind::Compaction); + assert_eq!(listed[0].scope_kind, GcScopeKind::Store); + } else { + assert!(listed.is_empty(), "the intent was released at {case:?}"); + } + + // Wholly old or wholly new: never a mixed Store. + let expected_generation = if case.installed { before + 1 } else { before }; + assert_eq!( + generation(&reopened), + expected_generation, + "generation at {case:?}" + ); + assert!(located(&reopened, live), "live root at {case:?}"); + assert_eq!( + object_bytes(&reopened, live), + live_bytes, + "live bytes readable at {case:?}" + ); + assert_eq!( + located(&reopened, dead), + !case.installed, + "the dead pack at {case:?}" + ); + assert!( + reopened.history.layer_stack(stack.id).unwrap().is_some(), + "retained history readable at {case:?}" + ); + + // A crash during the copy can tear the staged file. The next run must + // remove the recorded set instead of trusting it. + if case.failpoint == CompactFailpoint::DuringCopy { + let target = compact_path(&path); + std::fs::OpenOptions::new() + .write(true) + .open(&target) + .unwrap() + .set_len(17) + .unwrap(); + std::fs::write(with_suffix(&target, "-journal"), b"torn copy journal").unwrap(); + } + // A crash partway through settle can leave a subset of the recorded set. + // Put one back to prove the next run removes the whole set. + if case.failpoint == CompactFailpoint::DuringSettle { + std::fs::write( + with_suffix(&compact_path(&path), "-journal"), + b"partial settle", + ) + .unwrap(); + std::fs::write(visited_path(&path), b"partial settle").unwrap(); + } + + // The next run judges the leftover and reaches a clean state: a rebuild + // from `G` before install, a settle at `G + 1` after it. + let receipt = reopened.compact(&request(42)).unwrap(); + if case.installed { + assert_eq!(receipt.path, CompactPath::None, "settle at {case:?}"); + assert_eq!(receipt.generation_before, before + 1); + assert_eq!(receipt.generation_after, before + 1); + } else { + assert_eq!(receipt.path, CompactPath::Rebuild, "rebuild at {case:?}"); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + } + assert!( + receipt.residue.artifacts.is_empty(), + "residue at {case:?}: {:?}", + receipt.residue + ); + + assert!(compact_op_status(&path).is_none(), "op settled at {case:?}"); + assert!(!compact_path(&path).exists(), "target removed at {case:?}"); + assert!(!visited_path(&path).exists(), "visited removed at {case:?}"); + assert!(!op_path(&path).exists(), "op removed at {case:?}"); + assert_no_sidecars(&path, case); + + let final_handles = reopen(&path); + assert_eq!(generation(&final_handles), before + 1, "final generation"); + assert!(located(&final_handles, live), "final live root"); + assert_eq!( + object_bytes(&final_handles, live), + live_bytes, + "final bytes" + ); + assert!(!located(&final_handles, dead), "the dead pack is collected"); + assert!(intents(&final_handles).is_empty(), "the intent is released"); + // Closure and the full doctor both pass on the settled Store. + final_handles.history.mark().unwrap(); + clean(&final_handles); +} + +macro_rules! crash_at { + ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal) => { + #[test] + fn $name() { + exercise(Case { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + }); + } + }; +} + +crash_at!( + crash_before_seal, + CompactFailpoint::BeforeSeal, + "mark", + false, + false, + false +); +crash_at!( + crash_before_seal_read, + CompactFailpoint::BeforeSealRead, + "mark", + false, + false, + false +); +crash_at!( + crash_after_seal, + CompactFailpoint::AfterSeal, + "mark", + false, + false, + false +); +crash_at!( + crash_before_copy, + CompactFailpoint::BeforeCopy, + "build", + false, + false, + false +); +crash_at!( + crash_during_copy, + CompactFailpoint::DuringCopy, + "build", + false, + true, + true +); +crash_at!( + crash_after_build, + CompactFailpoint::AfterBuild, + "build", + false, + true, + true +); +crash_at!( + crash_after_wal_switch, + CompactFailpoint::AfterWalSwitch, + "build", + false, + true, + true +); +crash_at!( + crash_during_verify, + CompactFailpoint::DuringVerify, + "verify", + false, + true, + true +); +crash_at!( + crash_after_verify, + CompactFailpoint::AfterVerify, + "verify", + false, + true, + true +); +crash_at!( + crash_before_install, + CompactFailpoint::BeforeInstall, + "verify", + false, + true, + true +); +crash_at!( + crash_during_rename, + CompactFailpoint::DuringRename, + "install", + true, + false, + true +); +crash_at!( + crash_after_install, + CompactFailpoint::AfterInstall, + "install", + true, + false, + true +); +crash_at!( + crash_during_settle, + CompactFailpoint::DuringSettle, + "install", + true, + false, + false +); diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs new file mode 100644 index 0000000000..3107ef5ccc --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs @@ -0,0 +1,1597 @@ +//! #41 T12.1: SQLite `store.compact` fixtures and the golden receipt. +//! +//! One transcript pins the §11 report for all three outcomes: an empty Store +//! (`none`/`empty-store`), a Store with no wholly dead pack (`none`/`no-waste`) +//! and a Store with retained history plus known dead objects, packs and delta +//! bases (`rebuild`). The fixture is built only through the public APIs; the +//! golden JSON is compared byte for byte, so a change to any reported value is +//! deliberate. The same file pins the §6.4 live-heads digest vector (fixed ids, +//! a retired Branch and stack, an absent head). +//! +//! Covered matrix rows (SQLite half where the row is cross-backend): +//! AC-04 (deterministic arm), AC-05, AC-17, AC-18, AC-25 (Branch/stack markers), +//! AC-38 (successful settle), AC-40 (SQLite half), AC-47 (`none`/success), +//! AC-48 (empty Store and rebuild arms), AC-52, AC-56 (SQLite half), AC-58. +//! +//! Update the golden deliberately with `UPDATE_GOLDEN=1 cargo test -p +//! layerfs-persistence --test sqlite_compact_fixtures`; the second run checks +//! the new file. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{ + encode_whole_file_payload, + file::mapping::{self, ExtentNode, ExtentSlice, FileState}, + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + root::profile_id, + scope_for_seed, FilesystemRoot, PathName, + }, + inode_leaf::{ + encode_inode_value, InodeKind, InodeLeaf, InodeLeafRow, InodeMetadata, InodeValue, + LEAF_ROW_BYTES, + }, + FinalizedObject, ObjectId, ObjectRole, +}; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, HistoryCatalog, LayerId, + LayerRecord, LayerStackId, LayerStackRecord, StackInitialization, +}; +use layerfs_persistence::{ + diagnostic::{Observation, Options, Outcome}, + CompactPath, CompactReceipt, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_storage::{ + encoding::{encode_full, encode_prefix, raw_payload, CompressionWorkspace}, + location::{ObjectLocation, PackDomain}, + pack::{assemble, build_group, layout, PackLane}, + port::{PackPersistence, Publication, PublishedPack, Reserve}, + save::SaveProfile, + Storage, StorageCapacities, +}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use support::doctor::Fixture; +use support::Temp; + +const BINDING: &[u8] = b"doctor-adversarial"; +const CURSOR: [u8; 32] = [71; 32]; +const SCOPE_SEED: [u8; 32] = [11; 32]; +const GOLDEN: &str = include_str!("fixtures/sqlite_compact_golden.json"); + +// A deterministic pseudo-random payload: the dead packs must not compress away +// to nothing, and the same bytes must produce the same receipt on every run. +fn pseudo_bytes(len: usize, seed: u8) -> Vec { + let mut state = u64::from(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.to_le_bytes()); + } + out.truncate(len); + out +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + CURSOR, + ) + .unwrap() +} + +fn store_path(temp: &Temp) -> PathBuf { + temp.join("store.sqlite") +} + +fn sidecar(store: &Path, suffix: &str) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +// ---- fixture objects ------------------------------------------------------- + +fn whole_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + f.put( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .id() +} + +/// One chunked file's content root (a FileState): the representation a file +/// at or above the small-file threshold takes, and the only kind whose +/// file_digest row the Store records. +fn chunked_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + let mut extents = Vec::new(); + for part in bytes.chunks(32_768) { + let chunk = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(part).unwrap(), + ) + .id(); + extents.push(ExtentSlice::new(chunk, 0, part.len() as u32).unwrap()); + } + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: bytes.len() as u64, + extents, + }, + true, + ) + .unwrap(), + ) + .id(); + f.put( + ObjectRole::FileState, + mapping::encode_file_state(FileState { + logical_len: bytes.len() as u64, + extent_count: (bytes.len() as u64).div_ceil(32_768), + tree_level: 0, + profile_id: mapping::profile_id(), + mapping_root: leaf, + }) + .unwrap(), + ) + .id() +} + +/// One valid filesystem tree: a root directory whose only entry is a regular +/// file whose content root is `content`. The inode leaf is pooled, so the +/// fixture also owns `inode_value_block` rows that every rebuild keeps. +fn tree(f: &Fixture, content: ObjectId, seed: u8) -> ObjectId { + let directory = f + .put( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: vec![(PathName::new(&format!("file-{seed}")).unwrap(), 2)], + }) + .unwrap(), + ) + .id(); + let mut values = Vec::new(); + let mut rows = Vec::new(); + for (serial, kind, count, root, mode) in [ + (1u64, InodeKind::Directory, 0u64, directory, 0o755), + (2u64, InodeKind::RegularFile, 1u64, content, 0o644), + ] { + let value = encode_inode_value(InodeValue { + kind, + namespace_ref_count: count, + content_root: root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode, + mtime_seconds: 0, + mtime_nanoseconds: 0, + }), + }); + values.push(value); + rows.push(InodeLeafRow { serial, value }); + } + let inode = f.inode( + InodeLeaf { + subtree_bytes: LEAF_ROW_BYTES as u64 * rows.len() as u64, + rows, + } + .encode() + .unwrap(), + values, + ); + f.put( + ObjectRole::FilesystemRoot, + FilesystemRoot::new(profile_id(), scope_for_seed(SCOPE_SEED), 1, inode) + .unwrap() + .encode() + .unwrap(), + ) + .id() +} + +/// Publishes one pack holding each object as its own group, so a retained pack +/// can hold one reached and one unreached record. +fn publish_groups(f: &Fixture, objects: &[FinalizedObject]) -> (i64, Vec) { + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let mut workspace = CompressionWorkspace::new().unwrap(); + let mut profile = SaveProfile::default(); + let groups: Vec<_> = objects + .iter() + .map(|object| { + let record = encode_full( + object.canonical(), + object.role(), + &capacities, + &mut workspace, + &mut profile, + ) + .unwrap(); + build_group(PackLane::WholeFile, &[record.record], None).unwrap() + }) + .collect(); + let body = assemble(PackLane::WholeFile, &groups).unwrap(); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .unwrap(); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body.clone()), + )], + objects: objects + .iter() + .enumerate() + .map(|(group, object)| ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: group, + record_number: 0, + }) + .collect(), + ..Publication::default() + }) + .unwrap(); + (reserved.first_pack_id, body) +} + +/// Publishes `target` as a PREFIX record over `base`, asserting the encoder +/// kept the delta (a full record would hide the base from the mark). +fn prefix_record(f: &Fixture, target: &FinalizedObject, base: &FinalizedObject, base_id: ObjectId) { + let capacities = StorageCapacities::from_policy(f.policy).unwrap(); + let record = encode_prefix( + target.canonical(), + target.role(), + base_id, + raw_payload(base.canonical(), base.role()).unwrap(), + &capacities, + &mut CompressionWorkspace::new().unwrap(), + &mut SaveProfile::default(), + ) + .unwrap(); + assert_eq!( + record.base, + Some(base_id), + "the fixture needs a PREFIX record, not a full one" + ); + f.record(target, record.lane, record.record, Vec::new()); +} + +// ---- fixture history ------------------------------------------------------ + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: scope_for_seed(SCOPE_SEED).object(), + profile: profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn fork(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); +} + +fn commit( + handles: &SqliteHandles, + branch: BranchId, + candidate_root: ObjectId, + logical: u64, +) -> CommitId { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let expected_root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + let meta = CommitMeta { + statistics: CommitStatistics { + logical, + ..CommitStatistics::ZERO + }, + ..CommitMeta::new(CommitVia::Files) + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: snapshot.head_commit, + expected_base: snapshot.base_layer, + expected_root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta, + force: false, + }) + .unwrap() + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &SqliteHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +// ---- raw Store facts ------------------------------------------------------ + +#[derive(Clone, Copy, Debug, Default)] +struct Totals { + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, +} + +fn totals(path: &Path) -> Totals { + let connection = rusqlite::Connection::open(path).unwrap(); + let (logical, canonical, stored, metadata): (i64, i64, i64, i64) = connection + .query_row( + "SELECT logical,canonical,stored,metadata FROM store_totals WHERE id=1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + Totals { + logical: logical as u64, + canonical: canonical as u64, + stored: stored as u64, + metadata: metadata as u64, + } +} + +#[derive(Debug)] +struct PackFacts { + total: u64, + live_bytes: u64, + dead_bytes: u64, + live: Vec, + dead: Vec, +} + +fn pack_facts(path: &Path, live_packs: &[i64]) -> PackFacts { + let connection = rusqlite::Connection::open(path).unwrap(); + let mut statement = connection + .prepare("SELECT pack_id, length FROM pack ORDER BY pack_id") + .unwrap(); + let rows = statement + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))) + .unwrap(); + let mut facts = PackFacts { + total: 0, + live_bytes: 0, + dead_bytes: 0, + live: Vec::new(), + dead: Vec::new(), + }; + for row in rows { + let (pack, length) = row.unwrap(); + facts.total += length as u64; + if live_packs.contains(&pack) { + facts.live.push(pack); + facts.live_bytes += length as u64; + } else { + facts.dead.push(pack); + } + } + facts.dead_bytes = facts.total - facts.live_bytes; + facts +} + +fn scalar(path: &Path, sql: &str, param: impl rusqlite::ToSql) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection + .query_row(sql, [param], |row| row.get(0)) + .unwrap() +} + +fn scalar0(path: &Path, sql: &str) -> i64 { + let connection = rusqlite::Connection::open(path).unwrap(); + connection.query_row(sql, [], |row| row.get(0)).unwrap() +} + +fn pack_of(path: &Path, id: ObjectId) -> Option { + let connection = rusqlite::Connection::open(path).unwrap(); + match connection.query_row( + "SELECT pack_id FROM object_location WHERE object_id=?1", + [id.as_bytes().as_slice()], + |row| row.get::<_, i64>(0), + ) { + Ok(pack) => Some(pack), + Err(rusqlite::Error::QueryReturnedNoRows) => None, + Err(error) => panic!("object location: {error}"), + } +} + +fn object_count(path: &Path, id: ObjectId) -> i64 { + scalar( + path, + "SELECT count(*) FROM object_location WHERE object_id=?1", + id.as_bytes().as_slice(), + ) +} + +fn pack_length(path: &Path, pack: i64) -> u64 { + scalar(path, "SELECT length FROM pack WHERE pack_id=?1", pack) as u64 +} + +fn pack_row(path: &Path, pack: i64) -> i64 { + scalar(path, "SELECT count(*) FROM pack WHERE pack_id=?1", pack) +} + +fn branch_row(path: &Path, id: BranchId) -> i64 { + scalar( + path, + "SELECT count(*) FROM branch WHERE branch_id=?1", + id.as_slice(), + ) +} + +fn stack_row(path: &Path, id: LayerStackId) -> i64 { + scalar( + path, + "SELECT count(*) FROM layer_stack WHERE layer_stack_id=?1", + id.as_slice(), + ) +} + +fn retired_branches(path: &Path) -> Vec { + let connection = rusqlite::Connection::open(path).unwrap(); + let mut statement = connection + .prepare("SELECT branch_id FROM branch_retired ORDER BY branch_id") + .unwrap(); + let rows = statement + .query_map([], |row| row.get::<_, Vec>(0)) + .unwrap(); + rows.map(|row| BranchId::from_slice(&row.unwrap()).unwrap()) + .collect() +} + +fn retired_stacks(path: &Path) -> i64 { + scalar0(path, "SELECT count(*) FROM layer_stack_retired") +} + +fn digest_rows(path: &Path, digest: &[u8; 32]) -> i64 { + scalar( + path, + "SELECT count(*) FROM file_digest WHERE digest=?1", + digest.as_slice(), + ) +} + +fn inode_block_bytes(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(length(body)),0) FROM inode_value_block", + ) as u64 +} + +fn sum_pack_lengths(path: &Path) -> u64 { + scalar0(path, "SELECT COALESCE(SUM(length),0) FROM pack") as u64 +} + +fn sum_canonical(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(canonical_length),0) FROM object_location", + ) as u64 +} + +fn sum_metadata_packs(path: &Path) -> u64 { + scalar0( + path, + "SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0", + ) as u64 +} + +fn sum_history_logical(path: &Path) -> u64 { + (scalar0(path, "SELECT COALESCE(SUM(logical),0) FROM [commit]") + + scalar0(path, "SELECT COALESCE(SUM(logical),0) FROM layer")) as u64 +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn audit(handles: &SqliteHandles) -> Vec { + let mut out = Vec::new(); + handles + .diagnose( + Options { + quick: false, + ..Options::default() + }, + &mut |observation| out.push(observation), + ) + .unwrap(); + out +} + +fn assert_doctor_clean(handles: &SqliteHandles) { + let out = audit(handles); + assert!( + !out.iter() + .any(|observation| matches!(observation.outcome, Outcome::Fail | Outcome::Unknown)), + "the compacted Store must pass the full doctor: {out:?}" + ); +} + +fn assert_no_artifacts(store: &Path) { + for suffix in [ + ".compact", + ".compact-journal", + ".compact-wal", + ".compact-shm", + ".compact-visited", + ".compact-visited-journal", + ".compact-vacuum", + ".compact-op", + ".compact-op.tmp", + ] { + assert!( + !sidecar(store, suffix).exists(), + "compaction artifact {suffix} remains" + ); + } + let name = store.file_name().unwrap().to_string_lossy().to_string(); + for entry in std::fs::read_dir(store.parent().unwrap()).unwrap() { + let entry = entry.unwrap(); + let file = entry.file_name().to_string_lossy().to_string(); + if file == name { + continue; + } + let rest = file.strip_prefix(&name).unwrap_or(&file); + assert!( + matches!(rest, "-wal" | "-shm" | "-journal"), + "unexpected file beside the Store: {file}" + ); + } +} + +// ---- JSON ----------------------------------------------------------------- + +enum Json { + Arr(Vec), + Bool(bool), + Null, + Num(u64), + Obj(Vec<(&'static str, Json)>), + Str(String), +} + +impl Json { + fn render(&self) -> String { + let mut out = String::new(); + self.write(&mut out, 0); + out.push('\n'); + out + } + + fn write(&self, out: &mut String, depth: usize) { + match self { + Self::Bool(value) => out.push_str(if *value { "true" } else { "false" }), + Self::Null => out.push_str("null"), + Self::Num(value) => out.push_str(&value.to_string()), + Self::Str(value) => { + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + character if character.is_control() => { + out.push_str(&format!("\\u{:04x}", character as u32)); + } + character => out.push(character), + } + } + out.push('"'); + } + Self::Arr(items) => { + if items.is_empty() { + out.push_str("[]"); + return; + } + out.push_str("[\n"); + for (index, item) in items.iter().enumerate() { + pad(out, depth + 1); + item.write(out, depth + 1); + if index + 1 < items.len() { + out.push(','); + } + out.push('\n'); + } + pad(out, depth); + out.push(']'); + } + Self::Obj(fields) => { + if fields.is_empty() { + out.push_str("{}"); + return; + } + let mut sorted: Vec<_> = fields.iter().collect(); + sorted.sort_by(|left, right| left.0.cmp(right.0)); + out.push_str("{\n"); + for (index, (key, value)) in sorted.iter().enumerate() { + pad(out, depth + 1); + out.push('"'); + out.push_str(key); + out.push_str("\": "); + value.write(out, depth + 1); + if index + 1 < sorted.len() { + out.push(','); + } + out.push('\n'); + } + pad(out, depth); + out.push('}'); + } + } + } +} + +fn pad(out: &mut String, depth: usize) { + for _ in 0..depth { + out.push_str(" "); + } +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(DIGITS[(byte >> 4) as usize] as char); + out.push(DIGITS[(byte & 15) as usize] as char); + } + out +} + +fn strings(items: impl IntoIterator) -> Json { + Json::Arr(items.into_iter().map(Json::Str).collect()) +} + +fn object_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_bytes()))) +} + +fn byte_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(&id))) +} + +fn branch_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn commit_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn layer_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn stack_ids(ids: impl IntoIterator) -> Json { + strings(ids.into_iter().map(|id| hex(id.as_slice()))) +} + +fn numbers(items: impl IntoIterator) -> Json { + Json::Arr( + items + .into_iter() + .map(|item| Json::Num(item as u64)) + .collect(), + ) +} + +fn totals_json(total: Totals) -> Json { + Json::Obj(vec![ + ("canonical", Json::Num(total.canonical)), + ("logical", Json::Num(total.logical)), + ("metadata", Json::Num(total.metadata)), + ("stored", Json::Num(total.stored)), + ]) +} + +fn receipt_json(receipt: &CompactReceipt) -> Json { + Json::Obj(vec![ + ("backendReusable", Json::Num(receipt.backend_reusable)), + ("deadBytesBound", Json::Num(receipt.dead_bytes_bound)), + ("deadBytesRefined", Json::Num(receipt.dead_bytes_refined)), + ("estimate", Json::Bool(receipt.estimate)), + ("generationAfter", Json::Num(receipt.generation_after)), + ("generationBefore", Json::Num(receipt.generation_before)), + ( + "keptWaste", + Json::Obj(vec![ + ( + "deadInRetainedPacksBytes", + Json::Num(receipt.kept_waste.dead_in_retained_packs_bytes), + ), + ( + "deltaBases", + Json::Obj(vec![ + ("bytes", Json::Num(receipt.kept_waste.delta_bases_bytes)), + ( + "references", + byte_ids(receipt.kept_waste.delta_base_references.iter().copied()), + ), + ]), + ), + ( + "inodeBlocksBytes", + Json::Num(receipt.kept_waste.inode_blocks_bytes), + ), + ]), + ), + ("logicalDiscarded", Json::Num(receipt.logical_discarded)), + ("path", Json::Str(receipt.path.as_str().to_owned())), + ("peakTempBytes", Json::Num(receipt.peak_temp_bytes)), + ("physicalShrink", Json::Num(receipt.physical_shrink)), + ( + "residue", + Json::Obj(vec![ + ( + "artifacts", + strings(receipt.residue.artifacts.iter().cloned()), + ), + ("bytes", Json::Num(receipt.residue.bytes)), + ( + "reason", + match &receipt.residue.reason { + Some(reason) => Json::Str(reason.clone()), + None => Json::Null, + }, + ), + ]), + ), + ( + "zeroReasons", + Json::Arr( + receipt + .zero_reasons + .iter() + .map(|zero| { + Json::Obj(vec![ + ("field", Json::Str(zero.field.to_owned())), + ("reason", Json::Str(zero.reason.to_owned())), + ]) + }) + .collect(), + ), + ), + ]) +} + +fn mark_json(digest: [u8; 32], objects: usize, packs: &[i64]) -> Json { + Json::Obj(vec![ + ("liveHeadsDigest", Json::Str(hex(&digest))), + ("objectCount", Json::Num(objects as u64)), + ("packs", numbers(packs.iter().copied())), + ]) +} + +fn check_golden(actual: &Json) { + let rendered = actual.render(); + if std::env::var_os("UPDATE_GOLDEN").is_some() { + let path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/sqlite_compact_golden.json"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, &rendered).unwrap(); + eprintln!("updated golden transcript {}", path.display()); + return; + } + assert_eq!( + rendered, GOLDEN, + "the SQLite compaction golden transcript changed. Review every value; if \ + the change is deliberate, rerun with UPDATE_GOLDEN=1." + ); +} + +/// The §6.4 live-heads vector, written out independently of the mark: the +/// fixed prefix, the non-retired stacks in identity order as `(17, 33)`, then +/// the marker-free Branches in identity order as `(17, 33-or-33-zero, 33)`. +fn expected_heads_digest( + stacks: &[(LayerStackId, LayerId)], + branches: &[(BranchId, Option, LayerId)], +) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"layerfs/live-heads/v1\0"); + let mut stacks = stacks.to_vec(); + stacks.sort_by(|left, right| left.0.as_slice().cmp(right.0.as_slice())); + for (stack, head) in stacks { + assert_eq!(stack.as_slice().len(), 17, "stack id width"); + assert_eq!(head.as_slice().len(), 33, "layer id width"); + hasher.update(stack.as_slice()); + hasher.update(head.as_slice()); + } + let mut branches = branches.to_vec(); + branches.sort_by(|left, right| left.0.as_slice().cmp(right.0.as_slice())); + for (branch, commit, base) in branches { + assert_eq!(branch.as_slice().len(), 17, "branch id width"); + assert_eq!(base.as_slice().len(), 33, "base layer id width"); + hasher.update(branch.as_slice()); + match commit { + Some(commit) => { + assert_eq!(commit.as_slice().len(), 33, "commit id width"); + hasher.update(commit.as_slice()); + } + None => { + hasher.update(&[0u8; 33]); + } + } + hasher.update(base.as_slice()); + } + *hasher.finalize().as_bytes() +} + +// ---- cases ---------------------------------------------------------------- + +fn empty_none_case() -> Json { + let f = Fixture::new(); + let Fixture { temp, handles, .. } = f; + let path = store_path(&temp); + let before = totals(&path); + assert_eq!( + ( + before.logical, + before.canonical, + before.stored, + before.metadata + ), + (0, 0, 0, 0) + ); + let generation_before = generation(&handles); + let receipt = handles.compact(&request(0x70)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, 0); + assert_eq!(receipt.logical_discarded, 0); + assert_eq!(receipt.physical_shrink, 0); + assert_eq!(receipt.peak_temp_bytes, 0); + assert!(receipt.estimate); + assert!(receipt.kept_waste.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "empty-store")); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesRefined" && zero.reason == "empty-store")); + assert_no_artifacts(&path); + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before); + assert_doctor_clean(&reopened); + let digest = reopened.history.mark().unwrap().live_heads_digest(); + let after = totals(&path); + Json::Obj(vec![ + ("mark", mark_json(digest, 0, &[])), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +fn no_dead_none_case() -> Json { + let f = Fixture::new(); + let genesis_content = whole_file(&f, b"the genesis tree content"); + let genesis = tree(&f, genesis_content, 7); + let stack = initialize(&f.handles, 1, genesis); + let branch = support::branch_id(1); + fork(&f.handles, stack.id, branch, 1); + let commit_content = whole_file(&f, b"a retained commit tree content"); + let commit_root = tree(&f, commit_content, 8); + let commit_id = commit(&f.handles, branch, commit_root, 200); + let layer = publish(&f.handles, stack.id, branch); + + let Fixture { temp, handles, .. } = f; + let path = store_path(&temp); + let before = totals(&path); + let generation_before = generation(&handles); + let mark = handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + assert!(mark.objects().waste().inode_blocks_bytes() > 0); + let facts = pack_facts(&path, mark.objects().packs()); + assert_eq!(facts.dead_bytes, 0, "no wholly dead pack exists"); + let digest = mark.live_heads_digest(); + let live_packs = mark.objects().packs().to_vec(); + let receipt = handles.compact(&request(0x71)).unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before); + assert_eq!(receipt.dead_bytes_bound, 0); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert_eq!( + receipt.logical_discarded, 0, + "the none path collects nothing" + ); + assert_eq!(receipt.physical_shrink, 0); + assert_eq!(receipt.peak_temp_bytes, 0); + assert!(receipt.estimate); + assert!(receipt.residue.artifacts.is_empty()); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "deadBytesBound" && zero.reason == "no-waste")); + assert_no_artifacts(&path); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before); + assert_doctor_clean(&reopened); + assert!(reopened.history.commit(commit_id).unwrap().is_some()); + assert!(reopened.history.layer(layer.id).unwrap().is_some()); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + digest, + "the none path leaves the heads unchanged" + ); + let after = totals(&path); + Json::Obj(vec![ + ("commit", commit_ids([commit_id])), + ("layer", layer_ids([layer.id])), + ("mark", mark_json(digest, mark.objects().len(), &live_packs)), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +struct FixtureIds { + stacks_live: Vec, + stacks_collected: Vec, + branches_live: Vec, + branches_retained_retired: Vec, + branches_collected: Vec, + commits_retained: Vec, + commits_collected: Vec, + layers_retained: Vec, + layers_collected: Vec, + objects_live: Vec, + objects_dead: Vec, + delta_bases_sorted: Vec, + delta_chain: Vec<(ObjectId, i64)>, + delta_target: ObjectId, + live_group: ObjectId, + dead_group: ObjectId, + shared_pack: i64, + shared_pack_length: u64, + shared_dead_group: u64, + digest_collected: [u8; 32], + digest_retained: [u8; 32], +} + +fn rebuild_fixture() -> (Temp, SqliteHandles, FixtureIds) { + let f = Fixture::new(); + + // The dead packs: unreferenced whole files, each in its own pack. + let mut objects_dead = Vec::new(); + for index in 0..12u8 { + let payload = pseudo_bytes(4096, 0xA0 + index); + objects_dead.push(whole_file(&f, &payload)); + } + // AC-18 needs chunked file roots: the digest floor is the small-file + // threshold, so a WholeFile root is below it and records no row. + let retained_chunked_payload = pseudo_bytes(131_072, 0x77); + let retained_chunked = chunked_file(&f, &retained_chunked_payload); + let collected_chunked_payload = pseudo_bytes(131_072, 0x88); + let collected_chunked = chunked_file(&f, &collected_chunked_payload); + let digest_collected = *blake3::hash(&collected_chunked_payload).as_bytes(); + let digest_retained = *blake3::hash(&retained_chunked_payload).as_bytes(); + + // A depth-2 delta chain: base2 <- base1 <- target. Only the target is a + // retained root, so both bases are kept only as delta bases. + let base2_payload = pseudo_bytes(512, 0x11); + let base1_payload = [base2_payload.as_slice(), b" v2"].concat(); + let target_payload = [base1_payload.as_slice(), b" v3"].concat(); + let base2 = whole_file(&f, &base2_payload); + let base2_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base2_payload).unwrap(), + ) + .unwrap(); + let base1_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base1_payload).unwrap(), + ) + .unwrap(); + prefix_record(&f, &base1_object, &base2_object, base2); + let base1 = base1_object.id(); + let target_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&target_payload).unwrap(), + ) + .unwrap(); + prefix_record(&f, &target_object, &base1_object, base1); + let delta_target = target_object.id(); + + // One retained pack holding one reached and one unreached record. + let live_payload = pseudo_bytes(256, 0x55); + let dead_payload = pseudo_bytes(256, 0x66); + let live_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&live_payload).unwrap(), + ) + .unwrap(); + let dead_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&dead_payload).unwrap(), + ) + .unwrap(); + let (shared_pack, shared_body) = publish_groups(&f, &[live_group.clone(), dead_group.clone()]); + // The shared pack's live object is the genesis tree's file content. + let shared_header = layout::parse_header(&shared_body).unwrap(); + let dead_view = layout::group_view(&shared_body, shared_header, 1).unwrap(); + let shared_dead_group = (dead_view.end - dead_view.start) as u64; + let shared_pack_length = pack_length(&store_path(&f.temp), shared_pack); + + // Content objects of the trees that are expected to be collected. + let collected_content_a4 = collected_chunked; + let collected_content_b0 = whole_file(&f, b"deleted project genesis content"); + let collected_content_b1 = whole_file(&f, b"deleted project commit content"); + let content_c3 = whole_file(&f, b"live branch commit content"); + + let root_a0 = tree(&f, live_group.id(), 0); + let root_a1 = tree(&f, delta_target, 1); + let root_a2 = tree(&f, retained_chunked, 2); + let root_a3 = tree(&f, content_c3, 3); + let root_a4 = tree(&f, collected_content_a4, 4); + let root_b0 = tree(&f, collected_content_b0, 5); + let root_b1 = tree(&f, collected_content_b1, 6); + + let stack_a = initialize(&f.handles, 1, root_a0); + let branch_published = support::branch_id(1); + fork(&f.handles, stack_a.id, branch_published, 1); + let commit_1 = commit(&f.handles, branch_published, root_a1, 120); + let layer_1 = publish(&f.handles, stack_a.id, branch_published); + let commit_2 = commit(&f.handles, branch_published, root_a2, 240); + let layer_2 = publish(&f.handles, stack_a.id, branch_published); + + let branch_live = support::branch_id(2); + fork(&f.handles, stack_a.id, branch_live, 2); + let commit_3 = commit(&f.handles, branch_live, root_a3, 360); + + let branch_retired = support::branch_id(3); + fork(&f.handles, stack_a.id, branch_retired, 3); + let commit_4 = commit(&f.handles, branch_retired, root_a4, 70); + + let stack_b = initialize(&f.handles, 2, root_b0); + let branch_b = support::branch_id(4); + fork(&f.handles, stack_b.id, branch_b, 4); + let commit_5 = commit(&f.handles, branch_b, root_b1, 40); + let layer_b = publish(&f.handles, stack_b.id, branch_b); + + // A retired Branch a retained Layer still names keeps its row and marker. + f.handles + .history + .mark_branch_removed(branch_published, 1_000) + .unwrap(); + // A retired Branch nothing retained reaches is collected whole. + f.handles + .history + .mark_branch_removed(branch_retired, 1_500) + .unwrap(); + // A deleted project goes whole, marker included. + f.handles + .history + .mark_stack_deleted(stack_b.id, 2_000) + .unwrap(); + + // AC-18: one digest root no retained root reaches, one it does. + f.handles + .history + .record_file_digests(&[ + FileDigest { + root: *collected_chunked.as_bytes(), + length: collected_chunked_payload.len() as u64, + digest: digest_collected, + }, + FileDigest { + root: *retained_chunked.as_bytes(), + length: retained_chunked_payload.len() as u64, + digest: digest_retained, + }, + ]) + .unwrap(); + + let path = store_path(&f.temp); + let delta_chain: Vec<(ObjectId, i64)> = [base2, base1, delta_target] + .iter() + .map(|id| (*id, pack_of(&path, *id).expect("a delta-chain pack"))) + .collect(); + let mut delta_bases_sorted = vec![base1, base2]; + delta_bases_sorted.sort(); + let mut objects_dead = objects_dead; + objects_dead.push(dead_group.id()); + objects_dead.extend([ + root_a4, + root_b0, + root_b1, + collected_content_a4, + collected_content_b0, + collected_content_b1, + ]); + let objects_live = vec![ + root_a0, + root_a1, + root_a2, + root_a3, + live_group.id(), + retained_chunked, + base2, + base1, + delta_target, + content_c3, + ]; + let ids = FixtureIds { + stacks_live: vec![stack_a.id], + stacks_collected: vec![stack_b.id], + branches_live: vec![branch_live], + branches_retained_retired: vec![branch_published], + branches_collected: vec![branch_retired, branch_b], + commits_retained: vec![commit_1, commit_2, commit_3], + commits_collected: vec![commit_4, commit_5], + layers_retained: vec![stack_a.head_layer, layer_1.id, layer_2.id], + layers_collected: vec![stack_b.head_layer, layer_b.id], + objects_live, + objects_dead, + delta_bases_sorted, + delta_chain, + delta_target, + live_group: live_group.id(), + dead_group: dead_group.id(), + shared_pack, + shared_pack_length, + shared_dead_group, + digest_collected, + digest_retained, + }; + let Fixture { temp, handles, .. } = f; + (temp, handles, ids) +} + +fn rebuild_case() -> Json { + let (temp, handles, ids) = rebuild_fixture(); + let path = store_path(&temp); + let before = totals(&path); + let generation_before = generation(&handles); + + let mark = handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + let expected_digest = expected_heads_digest( + &[(ids.stacks_live[0], *ids.layers_retained.last().unwrap())], + &[( + ids.branches_live[0], + Some(ids.commits_retained[2]), + *ids.layers_retained.last().unwrap(), + )], + ); + assert_eq!( + mark.live_heads_digest(), + expected_digest, + "the mark hashes the §6.4 vector" + ); + let live_packs = mark.objects().packs().to_vec(); + let facts = pack_facts(&path, &live_packs); + let inode_blocks = inode_block_bytes(&path); + assert_eq!(mark.objects().waste().inode_blocks_bytes(), inode_blocks); + assert_eq!( + mark.objects().waste().dead_in_retained_packs_bytes(), + ids.shared_dead_group, + "the unreached group of the retained pack is keptWaste" + ); + assert_eq!( + mark.objects().waste().delta_base_references(), + ids.delta_bases_sorted.as_slice(), + "a depth-2 chain reports both bases, in identity order" + ); + let expected_bases: Vec<[u8; 32]> = ids + .delta_bases_sorted + .iter() + .map(|id| *id.as_bytes()) + .collect(); + assert!( + mark.objects().waste().delta_bases_bytes() > 0, + "the bases are kept only as bases" + ); + for id in &ids.objects_live { + assert!(mark.objects().contains(*id), "live object {id} is retained"); + } + for id in &ids.objects_dead { + assert!( + !mark.objects().contains(*id), + "dead object {id} is collected" + ); + } + assert_eq!(facts.dead_bytes, facts.total - facts.live_bytes); + assert!( + facts.dead_bytes * 10 >= facts.total, + "the fixture must take the rebuild path: {} dead of {}", + facts.dead_bytes, + facts.total + ); + + let receipt = handles.compact(&request(0x72)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before + 1); + assert_eq!(receipt.dead_bytes_bound, facts.dead_bytes); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + assert!(receipt.estimate); + assert_eq!( + receipt.kept_waste.dead_in_retained_packs_bytes, + ids.shared_dead_group + ); + assert_eq!(receipt.kept_waste.inode_blocks_bytes, inode_blocks); + assert_eq!(receipt.kept_waste.delta_base_references, expected_bases); + assert!(receipt.kept_waste.delta_bases_bytes > 0); + assert_eq!(receipt.residue.bytes, 0); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert_no_artifacts(&path); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), generation_before + 1); + let after = totals(&path); + assert_eq!( + receipt.logical_discarded, + before.logical - after.logical, + "logicalDiscarded is the history logical delta" + ); + assert_eq!(before.logical - after.logical, 160); // C4 + stack B genesis + C5 + Lb + assert_eq!(after.stored, sum_pack_lengths(&path)); + assert_eq!(after.canonical, sum_canonical(&path)); + assert_eq!(after.metadata, sum_metadata_packs(&path)); + assert_eq!(after.logical, sum_history_logical(&path)); + + // Live objects are readable, including the delta target through its chain. + let storage = Storage::new(reopened.storage.clone()).unwrap(); + let reader = storage.reader().unwrap(); + let canonicals = reader + .read_objects(&[ids.delta_target, ids.live_group]) + .unwrap(); + assert_eq!(canonicals.len(), 2); + assert!(!canonicals[0].is_empty()); + assert!(!canonicals[1].is_empty()); + for id in &ids.objects_live { + assert!(located(&reopened, *id), "live object {id} survives"); + } + for id in &ids.objects_dead { + assert!(!located(&reopened, *id), "dead object {id} is gone"); + } + for pack in &facts.dead { + assert_eq!(pack_row(&path, *pack), 0, "dead pack {pack} is removed"); + } + for pack in &facts.live { + assert_eq!(pack_row(&path, *pack), 1, "live pack {pack} stays"); + } + + // The retained pack keeps the unreached group: repacking is out of 2.0. + assert_eq!(pack_of(&path, ids.live_group), Some(ids.shared_pack)); + assert_eq!(object_count(&path, ids.live_group), 1); + assert_eq!(object_count(&path, ids.dead_group), 0); + assert_eq!(pack_length(&path, ids.shared_pack), ids.shared_pack_length); + // AC-17: a copied pack keeps its identity. + for (id, pack) in &ids.delta_chain { + assert_eq!( + pack_of(&path, *id), + Some(*pack), + "pack_id of {id} is preserved" + ); + } + + // Retained history and markers survive; collected rows take their markers. + for id in &ids.commits_retained { + assert!( + reopened.history.commit(*id).unwrap().is_some(), + "commit {id}" + ); + } + for id in &ids.commits_collected { + assert!( + reopened.history.commit(*id).unwrap().is_none(), + "commit {id}" + ); + } + for id in &ids.layers_retained { + assert!(reopened.history.layer(*id).unwrap().is_some(), "layer {id}"); + } + for id in &ids.layers_collected { + assert!(reopened.history.layer(*id).unwrap().is_none(), "layer {id}"); + } + for id in &ids.stacks_live { + assert!(reopened.history.layer_stack(*id).unwrap().is_some()); + assert_eq!(stack_row(&path, *id), 1); + } + for id in &ids.stacks_collected { + assert!(reopened.history.layer_stack(*id).unwrap().is_none()); + assert_eq!(stack_row(&path, *id), 0); + } + for id in &ids.branches_live { + assert!(reopened.history.branch(*id).unwrap().is_some()); + } + assert_eq!(retired_branches(&path), ids.branches_retained_retired); + for id in &ids.branches_collected { + assert_eq!(branch_row(&path, *id), 0, "collected branch row {id}"); + } + assert_eq!(retired_stacks(&path), 0); + + // AC-18: a digest row survives only when a retained root reaches the root. + assert_eq!(digest_rows(&path, &ids.digest_retained), 1); + assert_eq!(digest_rows(&path, &ids.digest_collected), 0); + + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + mark.live_heads_digest(), + "S9: the live-heads digest matches before and after" + ); + + let fixture = Json::Obj(vec![ + ( + "branches", + Json::Obj(vec![ + ( + "collected", + branch_ids(ids.branches_collected.iter().copied()), + ), + ("live", branch_ids(ids.branches_live.iter().copied())), + ( + "retainedRetired", + branch_ids(ids.branches_retained_retired.iter().copied()), + ), + ]), + ), + ( + "commits", + Json::Obj(vec![ + ( + "collected", + commit_ids(ids.commits_collected.iter().copied()), + ), + ("retained", commit_ids(ids.commits_retained.iter().copied())), + ]), + ), + ("deadBytesBound", Json::Num(facts.dead_bytes)), + ( + "deltaBaseReferences", + object_ids(ids.delta_bases_sorted.iter().copied()), + ), + ( + "deltaChainPackIds", + numbers(ids.delta_chain.iter().map(|(_, pack)| *pack)), + ), + ("inodeBlocksBytes", Json::Num(inode_blocks)), + ( + "layers", + Json::Obj(vec![ + ("collected", layer_ids(ids.layers_collected.iter().copied())), + ("retained", layer_ids(ids.layers_retained.iter().copied())), + ]), + ), + ( + "objects", + Json::Obj(vec![ + ("dead", object_ids(ids.objects_dead.iter().copied())), + ("live", object_ids(ids.objects_live.iter().copied())), + ]), + ), + ( + "packs", + Json::Obj(vec![ + ("dead", numbers(facts.dead.iter().copied())), + ("live", numbers(facts.live.iter().copied())), + ("shared", Json::Num(ids.shared_pack as u64)), + ("sharedDeadGroupBytes", Json::Num(ids.shared_dead_group)), + ("sharedLength", Json::Num(ids.shared_pack_length)), + ]), + ), + ( + "stacks", + Json::Obj(vec![ + ("collected", stack_ids(ids.stacks_collected.iter().copied())), + ("live", stack_ids(ids.stacks_live.iter().copied())), + ]), + ), + ]); + let after_state = Json::Obj(vec![ + ("branchRetired", branch_ids(retired_branches(&path))), + ("deadPacksGone", Json::Bool(true)), + ("fileDigestCollected", Json::Bool(false)), + ("fileDigestRetained", Json::Bool(true)), + ("layerStackRetired", Json::Num(retired_stacks(&path) as u64)), + ("liveReadable", Json::Bool(true)), + ("liveHeadsDigestUnchanged", Json::Bool(true)), + ("fullDoctor", Json::Str("clean".to_owned())), + ]); + Json::Obj(vec![ + ("after", after_state), + ("fixture", fixture), + ( + "mark", + mark_json(mark.live_heads_digest(), mark.objects().len(), &live_packs), + ), + ("receipt", receipt_json(&receipt)), + ("totalsAfter", totals_json(after)), + ("totalsBefore", totals_json(before)), + ]) +} + +fn digest_vector_case() -> Json { + let f = Fixture::new(); + // Out-of-order creation: the digest orders by identity, not creation. + let root_9 = tree(&f, whole_file(&f, b"digest stack nine content"), 9); + let stack_9 = initialize(&f.handles, 9, root_9); + let root_3 = tree(&f, whole_file(&f, b"digest stack three content"), 10); + let stack_3 = initialize(&f.handles, 3, root_3); + let root_7 = tree(&f, whole_file(&f, b"digest retired stack content"), 11); + let stack_7 = initialize(&f.handles, 7, root_7); + f.handles + .history + .mark_stack_deleted(stack_7.id, 7_000) + .unwrap(); + + let branch_9 = support::branch_id(9); + fork(&f.handles, stack_9.id, branch_9, 9); + let branch_3 = support::branch_id(3); + fork(&f.handles, stack_3.id, branch_3, 3); + let branch_5 = support::branch_id(5); + fork(&f.handles, stack_9.id, branch_5, 5); + let commit_root = tree(&f, whole_file(&f, b"digest commit content"), 12); + let commit_id = commit(&f.handles, branch_3, commit_root, 90); + + // branch_5 has never committed: an absent head is hashed as 33 zero bytes. + let with_empty_head = f.handles.history.mark().unwrap().live_heads_digest(); + assert_eq!( + with_empty_head, + f.handles.history.mark().unwrap().live_heads_digest(), + "the fixed fixture pins one digest" + ); + assert_eq!( + with_empty_head, + expected_heads_digest( + &[ + (stack_3.id, stack_3.head_layer), + (stack_9.id, stack_9.head_layer) + ], + &[ + (branch_3, Some(commit_id), stack_3.head_layer), + (branch_5, None, stack_9.head_layer), + (branch_9, None, stack_9.head_layer), + ], + ), + "an absent head is 33 zero bytes" + ); + // A retired Branch is absent, not zeroed. + f.handles + .history + .mark_branch_removed(branch_5, 5_000) + .unwrap(); + let retired_branch = f.handles.history.mark().unwrap().live_heads_digest(); + assert_ne!( + with_empty_head, retired_branch, + "a retired Branch leaves the digest instead of hashing zeros" + ); + assert_eq!( + retired_branch, + expected_heads_digest( + &[ + (stack_3.id, stack_3.head_layer), + (stack_9.id, stack_9.head_layer) + ], + &[ + (branch_3, Some(commit_id), stack_3.head_layer), + (branch_9, None, stack_9.head_layer), + ], + ), + "a retired Branch row is absent, not zeroed" + ); + assert_eq!( + retired_branch, + f.handles.history.mark().unwrap().live_heads_digest() + ); + Json::Obj(vec![ + ("absentHeadDigest", Json::Str(hex(&with_empty_head))), + ("commit", commit_ids([commit_id])), + ("retiredBranchDigest", Json::Str(hex(&retired_branch))), + ("retiredStack", stack_ids([stack_7.id])), + ]) +} + +#[test] +fn the_sqlite_compaction_receipts_match_the_golden_transcript() { + let transcript = Json::Obj(vec![ + ("digestVector", digest_vector_case()), + ("emptyNone", empty_none_case()), + ("noDeadNone", no_dead_none_case()), + ("rebuild", rebuild_case()), + ]); + check_golden(&transcript); +} diff --git a/engine/crates/layerfs-store-service/src/methods/branch/mod.rs b/engine/crates/layerfs-store-service/src/methods/branch/mod.rs index 63177f6be8..92fc82863e 100644 --- a/engine/crates/layerfs-store-service/src/methods/branch/mod.rs +++ b/engine/crates/layerfs-store-service/src/methods/branch/mod.rs @@ -7,6 +7,7 @@ mod get_commit; mod list_commits; mod merge; mod publish; +mod remove; pub use commits::commits; pub use diff_branches::diff_branches; pub use diff_commits::diff_commits; @@ -15,3 +16,4 @@ pub use get_commit::get_commit; pub use list_commits::list_commits; pub use merge::merge; pub use publish::publish; +pub use remove::remove; diff --git a/engine/crates/layerfs-store-service/src/methods/branch/remove.rs b/engine/crates/layerfs-store-service/src/methods/branch/remove.rs new file mode 100644 index 0000000000..1bcdf5c80b --- /dev/null +++ b/engine/crates/layerfs-store-service/src/methods/branch/remove.rs @@ -0,0 +1,10 @@ +//! `branch.remove`. +use crate::methods::args::{project, text, Answer}; +use crate::{history::removal, StoreService}; +use layerfs_bridge::control::SdkValue; + +/// `remove({ project, branch })`: hide one Branch by its Store marker. +pub fn remove(service: &StoreService, args: &SdkValue) -> Answer { + let stack = project(service, args)?; + removal::remove(&service.store, &stack, text(args, "branch")?) +} diff --git a/engine/crates/layerfs-store-service/src/methods/dispatch.rs b/engine/crates/layerfs-store-service/src/methods/dispatch.rs index 73e1cf034a..5c96f216f9 100644 --- a/engine/crates/layerfs-store-service/src/methods/dispatch.rs +++ b/engine/crates/layerfs-store-service/src/methods/dispatch.rs @@ -29,6 +29,7 @@ pub fn call(service: &StoreService, call: &SdkCall) -> Answer { "branch.get_commit" => branch::get_commit(service, args), "branch.merge" => branch::merge(service, args), "branch.publish" => branch::publish(service, args), + "branch.remove" => branch::remove(service, args), "branch.diff_commits" => branch::diff_commits(service, args), "branch.diff_branches" => branch::diff_branches(service, args), "fs.read" => fs::read(service, args), diff --git a/engine/crates/layerfs-store-service/src/methods/file/glob.rs b/engine/crates/layerfs-store-service/src/methods/file/glob.rs index bde2596c77..065a2e1bd6 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/glob.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/glob.rs @@ -17,6 +17,8 @@ pub fn glob(service: &StoreService, args: &SdkValue) -> Answer { return Err(refusal); } } + // The page-independent `removed_count`: the hidden entries this pattern + // would return anywhere under `path`, not only direct children. let removed = seen.hidden_matches(&reader, &snap, &path, pattern)?; if args.get("page").is_some() { return paged_reads::pin( diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs index ef60b2cf4c..f388a1b1a5 100644 --- a/engine/crates/layerfs-store-service/src/read/visibility.rs +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -132,7 +132,10 @@ impl Visibility { /// The number of hidden entries directly inside `dir` whose entries exist /// at this snapshot: the `removed_count` of a listing, independent of the - /// page. A marker on a path below a direct child is not counted here. + /// page. A listing returns `dir`'s direct entries, so a marker below one + /// of them is not a hidden result and is not counted; a glob counts the + /// matched hidden entries at any depth (`hidden_matches`). Each call + /// therefore counts exactly the entries its own result would omit. pub fn hidden_children( &self, reader: &StorePorts, @@ -168,8 +171,12 @@ impl Visibility { } /// The number of hidden entries at or below `path` that the glob `pattern` - /// matches: the `removed_count` of a glob, independent of the page. Only - /// hidden subtrees are walked, so a snapshot with no marker counts nothing. + /// matches: the `removed_count` of a glob, independent of the page. It + /// counts every hidden entry the glob would otherwise return, at any depth + /// under the query root, not only the root's direct children: the pattern + /// decides which descendants are results. A listing's `hidden_children` + /// does the same for its direct entries. Only hidden subtrees are walked, + /// so a snapshot with no marker counts nothing. pub fn hidden_matches( &self, reader: &StorePorts, @@ -212,6 +219,10 @@ impl Visibility { found.insert(full.clone()); } if entry.value.kind == InodeKind::Directory { + // A hidden directory's descendants are hidden by the same + // governing frontier and are glob results whenever the pattern + // reaches them, so walk the hidden subtree and count each + // match; `found` deduplicates a marker on a descendant. walked.push(relative.to_owned()); walk::walk_within(reader, snap, &full, None, |child, _| { if rule.matches_path(child) { diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index f15aec7227..36925dcd95 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -207,6 +207,9 @@ fn change( // already whiteouts a hidden path, so its absence there is ambiguous. // The base view distinguishes a hidden path (a `Removed` refusal for // `edit` and a cp source) from a genuinely absent one (`NoSuchPath`). + // `rm` and `mv`-from read only the private overlay, where a hidden path is + // already a whiteout, so they report it as absent: `NoSuchPath`, this + // method table's wire spelling of §4.3's `NotFound` for those two calls. let base = workspace .snapshot() .map_err(crate::write::mutation::failure)?; @@ -240,6 +243,7 @@ fn change( }, Change::Mkdir { path, mode } => mutation.mkdir(path, *mode), Change::Chmod { path, mode } => mutation.chmod(path, *mode), + // `rm` treats a hidden path as absent for the same reason. Change::Rm { path } => mutation.remove(path), Change::Cp { from, to } => match mutation.copy(from, to, index as u64) { Err(refusal) if refusal.code == "NoSuchPath" => match removed(from)? { @@ -248,6 +252,9 @@ fn change( }, other => other, }, + // A hidden `from` is absent to the overlay, so `mv`-from answers + // `NoSuchPath` (see the `NotFound` mapping above); unlike + // `edit`/`cp`-from it does not consult the base view. Change::Mv { from, to } => mutation.rename(from, to), })() .map_err(|mut e| { diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs b/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs new file mode 100644 index 0000000000..6fdcc1bbc8 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_hidden_semantics.rs @@ -0,0 +1,290 @@ +//! T5.8: hidden `rm`/`mv` sources and the glob `removed_count`. +//! +//! §4.3 says a `file.mv` from a hidden path is `NotFound` and a `file.rm` of +//! a hidden path is `NotFound`. This method table spells a missing path +//! `NoSuchPath`, so those two calls report exactly what a never-existing path +//! reports; `cp`-from and `edit` consult the base view and answer `Removed`. +//! A listing counts the hidden entries among its direct results; a glob counts +//! every hidden entry the pattern would return anywhere under its query root. +//! Both totals are page-independent and `removed_count` is on every page. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::fs; +use support::{items, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, branch: &str, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text(branch)), ("path", text(path))], + ) +} + +/// The one `Removed` detail path and its frontier. +fn hidden_path(refusal: &SdkRefusal, path: &str) -> i64 { + assert_eq!(refusal.code, "Removed", "{refusal:?}"); + let Some(SdkValue::List(hidden)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + assert_eq!(hidden.len(), 1, "{refusal:?}"); + assert_eq!(hidden[0].get_text("path"), Some(path), "{refusal:?}"); + hidden[0].get_int("frontier_serial").unwrap() +} + +fn glob(fixture: &Fixture, pattern: &str, path: &str) -> SdkValue { + fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text(pattern)), + ("path", text(path)), + ], + ) +} + +fn paged_glob(fixture: &Fixture, pattern: &str, page: i64, page_size: i64) -> SdkValue { + fixture.call( + "fs.glob", + vec![ + ("branch", text("work")), + ("pattern", text(pattern)), + ("path", text("/")), + ("page", SdkValue::Int(page)), + ("pageSize", SdkValue::Int(page_size)), + ], + ) +} + +fn matched(value: &SdkValue) -> Vec<&str> { + items(value) + .iter() + .map(|item| match item { + SdkValue::Text(text) => text.as_str(), + other => panic!("glob item is not a path: {other:?}"), + }) + .collect() +} + +#[test] +fn hidden_rm_and_mv_sources_are_absent_while_cp_and_edit_are_removed() { + let fixture = Fixture::new("t58-hidden-writes", |source| { + fs::write(source.join("a.txt"), "a\n").unwrap(); + fs::write(source.join("b.txt"), "b\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f.txt"), "f\n").unwrap(); + 4 + }); + for branch in ["rm", "mv", "cp", "edit"] { + fixture.fork(branch); + } + remove(&fixture, "/a.txt"); + remove(&fixture, "/d"); + + // §4.3's `NotFound` for a hidden `rm`/`mv`-from is this wire spelling: + // those calls read the private overlay, where a hidden path is already a + // whiteout, and answer like a never-existing path (`NoSuchPath`). + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/a.txt"))], + ) + .expect_err("hidden rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.mv", + vec![ + ("branch", text("mv")), + ("from", text("/a.txt")), + ("to", text("/moved.txt")), + ], + ) + .expect_err("hidden mv source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A hidden directory and its descendants are absent to `rm`/`mv` too. + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/d"))], + ) + .expect_err("hidden directory rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.rm", + vec![("branch", text("rm")), ("path", text("/d/f.txt"))], + ) + .expect_err("hidden child rm"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.mv", + vec![ + ("branch", text("mv")), + ("from", text("/d/f.txt")), + ("to", text("/moved.txt")), + ], + ) + .expect_err("hidden child mv source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // `cp`-from and `edit` consult the base tree, so a hidden source is + // `Removed` with the governing path and frontier, not `NoSuchPath`. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/a.txt")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("hidden edit"); + hidden_path(&refusal, "/a.txt"); + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/d/f.txt")), + ("oldText", text("f")), + ("newText", text("z")), + ], + ) + .expect_err("hidden child edit"); + hidden_path(&refusal, "/d/f.txt"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/a.txt")), + ("to", text("/copy.txt")), + ], + ) + .expect_err("hidden cp source"); + hidden_path(&refusal, "/a.txt"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/d")), + ("to", text("/copy")), + ], + ) + .expect_err("hidden cp directory"); + hidden_path(&refusal, "/d"); + + // A genuinely absent path is still `NoSuchPath` for `cp`/`edit`: the + // hidden and the absent cases stay distinguishable where the spec asks. + let refusal = fixture + .try_call( + "fs.edit", + vec![ + ("branch", text("edit")), + ("path", text("/missing")), + ("oldText", text("a")), + ("newText", text("z")), + ], + ) + .expect_err("absent edit"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + let refusal = fixture + .try_call( + "fs.cp", + vec![ + ("branch", text("cp")), + ("from", text("/missing")), + ("to", text("/copy")), + ], + ) + .expect_err("absent cp source"); + assert_eq!(refusal.code, "NoSuchPath", "{refusal:?}"); + + // A refused call leaves the branch and the destination untouched. + assert_eq!( + read(&fixture, "cp", "/copy.txt").unwrap_err().code, + "NoSuchPath" + ); + assert_eq!( + read(&fixture, "mv", "/moved.txt").unwrap_err().code, + "NoSuchPath" + ); + assert_eq!( + read(&fixture, "rm", "/b.txt").unwrap().get_text("text"), + Some("b\n") + ); + fixture.cleanup(); +} + +#[test] +fn glob_removed_count_counts_the_hidden_results_at_any_depth_on_every_page() { + let fixture = Fixture::new("t58-glob-count", |source| { + fs::write(source.join("a.txt"), "a\n").unwrap(); + fs::write(source.join("b.txt"), "b\n").unwrap(); + fs::create_dir(source.join("d")).unwrap(); + fs::write(source.join("d/f.txt"), "f\n").unwrap(); + fs::create_dir(source.join("d/sub")).unwrap(); + fs::write(source.join("d/sub/g.txt"), "g\n").unwrap(); + fs::create_dir(source.join("e")).unwrap(); + fs::write(source.join("e/h.txt"), "h\n").unwrap(); + 8 + }); + fixture.fork("work"); + remove(&fixture, "/a.txt"); + remove(&fixture, "/d"); + + // `/*` returns direct children only: the hidden file and directory are + // omitted and counted, but the hidden entries below `/d` are not results + // of this pattern and are not counted. + let top = glob(&fixture, "/*", "/"); + assert_eq!(matched(&top), vec!["/b.txt", "/e"], "{top:?}"); + assert_eq!(top.get_int("removed_count"), Some(2), "{top:?}"); + + // `/**` returns the whole queried subtree: every hidden entry the pattern + // would return counts, including `/d`'s descendants at any depth. + let all = glob(&fixture, "/**", "/"); + assert_eq!(matched(&all), vec!["/b.txt", "/e", "/e/h.txt"], "{all:?}"); + assert_eq!(all.get_int("removed_count"), Some(5), "{all:?}"); + + // A pattern that names hidden descendants counts those two, not the + // hidden directory itself (which this pattern does not match). + let below = glob(&fixture, "/d/*", "/"); + assert!(matched(&below).is_empty(), "{below:?}"); + assert_eq!(below.get_int("removed_count"), Some(2), "{below:?}"); + + // `removed_count` is present at 0 when the pattern matches no hidden entry. + let visible_only = glob(&fixture, "/b*", "/"); + assert_eq!(matched(&visible_only), vec!["/b.txt"], "{visible_only:?}"); + assert_eq!( + visible_only.get_int("removed_count"), + Some(0), + "{visible_only:?}" + ); + + // The count is page-independent: every page of the same glob reports the + // same total while its items carry only the visible matches. + let first = paged_glob(&fixture, "/**", 1, 2); + assert_eq!(matched(&first), vec!["/b.txt", "/e"], "{first:?}"); + assert_eq!(first.get_int("removed_count"), Some(5), "{first:?}"); + let second = paged_glob(&fixture, "/**", 2, 2); + assert_eq!(matched(&second), vec!["/e/h.txt"], "{second:?}"); + assert_eq!(second.get_int("removed_count"), Some(5), "{second:?}"); + fixture.cleanup(); +}