diff --git a/.github/workflows/store-ext4.yml b/.github/workflows/store-ext4.yml new file mode 100644 index 0000000000..cffa8038de --- /dev/null +++ b/.github/workflows/store-ext4.yml @@ -0,0 +1,203 @@ +# Store lane: the Store suites, on a filesystem that carries SQLite's locks. +# +# A Store opens only on a directory that is private, owned by the test user and +# on a local filesystem whose locks reach every opener: ext4, xfs or btrfs on +# Linux (engine/crates/layerfs-store-service/src/kernel_guard.rs). The Store +# suites keep their Stores in TMPDIR and in $CARGO_TARGET_DIR/tmp, so both must +# be on such a filesystem, and an overlay checkout or a tmpfs /tmp is refused. +# tools/ci/test_store_ext4.sh refuses any other directory before cargo starts +# and points TMPDIR, CARGO_TARGET_DIR and CARGO_HOME at the one it is given; +# this lane only provides that directory. See +# docs/architecture/engine/store/testing.md. +# +# It adds no test selection of its own: every target it runs is already in the +# push or nightly lane chosen by tools/ci_tests.py. It runs them under the +# environment contract their guard requires. +# +# Reports only, like the other lanes (AGENTS.md §4): never a required check. +# The runner's /mnt is a separate ext4 data disk while /tmp is tmpfs, so the +# prepare step asserts the filesystem: a runner image whose /mnt is not ext4 +# fails there, with its own message, instead of as a guard refusal inside a +# suite. +name: store-ext4 + +on: + push: + branches: [main] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + pull_request: + # off for pull requests into layerfs_v2_0_0, like the other lanes + branches-ignore: [layerfs_v2_0_0] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + schedule: + # After nightly.yml's 04:23 UTC run; the two lanes never share runner time. + - cron: "47 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: store-ext4-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + # CI needs no debugger: smaller objects, much shorter links. + CARGO_PROFILE_TEST_DEBUG: "0" + # Link the test binaries with mold instead of GNU ld, as ci.yml's test job + # does. Scoped to this target, so the repository's aarch64 rustflags + # (.cargo/config.toml) stay. + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C link-arg=-fuse-ld=mold" + # The harness requires this and validates it before cargo starts. /mnt is the + # runner's ext4 data disk; never /tmp, which is tmpfs there. + LAYERFS_TEST_TMPDIR: /mnt/layerfs-store-tests + +jobs: + quick: + name: store ext4 (--quick) + # The push lane: one or two targets per suite, across the four suites. + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + # /mnt belongs to root on the runner image and the tests run as the + # runner, so hand the directory over before the harness probes it. + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # No Swatinem/rust-cache here: the harness moves CARGO_HOME and + # CARGO_TARGET_DIR onto LAYERFS_TEST_TMPDIR, so the checkout-scoped cache + # the other lanes restore is never read. The cold build is the price of + # running the suites where their guard admits them. + - name: Run the Store suites (--quick) + run: tools/ci/test_store_ext4.sh --quick + + full-postgresql: + name: store ext4 (--full, PostgreSQL 17) + # The scheduled and manual lane: every Store suite, with the PostgreSQL + # cases on. A manual dispatch runs this job only, which is the full sweep. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + # A libpq connection string, not a URL (testing.md). A service + # container's published port is 127.0.0.1 on the runner. The server is + # scratch -- each PostgreSQL test creates and drops its own schema -- and + # lives only for this job. Setting this adds `--features postgresql` to + # every suite in the harness. + LAYERFS_TEST_POSTGRES_URL: host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=disable + # The full-mode doctor calls amcheck's verify_heapam and bt_index_check; + # the step below creates it in this database. The physical doctor's two + # extra servers (an amcheck-free database and a physical standby) are + # not available in this job; see testing.md. Set so the diagnostic test + # asserts relation coverage is Pass instead of only printing it. + LAYERFS_TEST_EXPECT_AMCHECK: "1" + services: + postgres: + # PostgreSQL 17 is the supported version for these cases: the schema + # needs 16 or newer for STORAGE PLAIN, and the suites are qualified on + # 17.x (docs/architecture/engine/store/testing.md). + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + # Trust authentication: a throwaway server, reachable from this job's + # steps only, and the tests create and drop their own schemas. + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=postgres --dbname=postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # The service health check gates this job on the server answering. The + # engine never installs extensions: a full-mode audit calls amcheck's + # verify_heapam and bt_index_check and reports every relation as + # Unknown when the extension is absent, while the compaction suite + # asserts a clean full audit. A fresh cluster therefore needs these + # contrib extensions before any suite connects (T-G14). The server is + # scratch, and this is the one database every suite reaches. + - name: Create the PostgreSQL extensions + run: | + set -euo pipefail + # The runner image does not ship psql; the distro client reaches the + # PostgreSQL 17 service container. + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends postgresql-client + psql --version + psql "$LAYERFS_TEST_POSTGRES_URL" -X -v ON_ERROR_STOP=1 <<'SQL' + CREATE EXTENSION IF NOT EXISTS amcheck; + CREATE EXTENSION IF NOT EXISTS pgstattuple; + SQL + # Fail here, with a readable list, rather than as five Unknown + # relation checks inside a compaction suite. + psql "$LAYERFS_TEST_POSTGRES_URL" -X -A -t -v ON_ERROR_STOP=1 \ + -c "SELECT extname FROM pg_catalog.pg_extension WHERE extname IN ('amcheck','pgstattuple') ORDER BY extname" + # The suites own their schemas; the harness reports the mode it took. + - name: Run the Store suites (--full) + run: tools/ci/test_store_ext4.sh --full diff --git a/Cargo.lock b/Cargo.lock index 3ba6ad4ac3..94c8fbe9fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -999,6 +999,7 @@ dependencies = [ "layerfs-telemetry", "libc", "nix", + "postgres", ] [[package]] diff --git a/api/crates/layerfs-api/tests/scope_references.rs b/api/crates/layerfs-api/tests/scope_references.rs index 86aa60c205..51bdba4264 100644 --- a/api/crates/layerfs-api/tests/scope_references.rs +++ b/api/crates/layerfs-api/tests/scope_references.rs @@ -193,10 +193,13 @@ fn fork_refuses_every_reference_outside_the_namespace_before_any_effect() { let mut given = vec![p()]; given.extend(args.iter().cloned()); let refused = call(&home, Some(&scoped), "branch.fork", &given).expect_err(what); - assert_eq!(refused.0, "InvalidInput", "{what}: {refused:?}"); - // the two composed Branch names (cases 0 and 3) name no Branch and - // are refused by the Store; the others never reach it - if n != 0 && n != 3 { + // The two composed Branch names (cases 0 and 3) name no Branch: the + // Store answers the typed `NoSuchBranch`; every other reference is + // refused by scope before it reaches the Store. + if n == 0 || n == 3 { + assert_eq!(refused.0, "NoSuchBranch", "{what}: {refused:?}"); + } else { + assert_eq!(refused.0, "InvalidInput", "{what}: {refused:?}"); assert!( refused.1.contains("out of scope: branch"), "{what}: {refused:?}" diff --git a/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs new file mode 100644 index 0000000000..9e53c90749 --- /dev/null +++ b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs @@ -0,0 +1,426 @@ +//! AC-01 (§12 crash injection, §9): `kill -9` the **real process** that +//! carries `store.compact`, not a re-execution of this test binary. +//! +//! The in-process matrix (`engine/crates/layerfs-persistence/tests/ +//! sqlite_compact_crash.rs`) stops at a failpoint inside the test process, and +//! its `real_process` arm re-executes the *test binary*, so the killed process +//! is still a test child. This test starts the built `layerfs` command with +//! `store compact`, waits through the SQLite engine's Unix-socket gate until +//! that process stands inside the run, and `SIGKILL`s it there. Per #352 +//! F-020 no daemon serves Store calls: the caller's process performs the +//! compaction, so this is the process the §9 crash describes. (It is still not +//! a long-lived `layerfs worker` serving a machine's Store.) +//! +//! Every arm reopens the Store through the CLI and asserts the §9 crash state: +//! wholly old at `G` or wholly new at `G + 1`, the exact artifact set, the +//! sidecars a killed session leaves, the `store.status` projection (phase, +//! `uncertain`, `unresolved`), the next `store.compact` settling the leftover, +//! the retained layer still readable, and a clean `doctor`. The `G` baseline +//! is the §3 `none` run before the dead pack exists, so the crash arms start +//! from a Store with exactly one wholly dead pack. +//! +//! Linux only: the arm asserts `SIGKILL` delivery through +//! `ExitStatusExt::signal`. A macOS build has no gate in the engine, so the +//! hook is inert there by construction. +#![cfg(target_os = "linux")] +mod common; + +use common::{layerfs, run, run_with_stdin, tempdir, text, LAYERFS}; +use layerfs_api::json; +use layerfs_bridge::control::SdkValue; +use std::io::Read; +use std::os::unix::net::UnixListener; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +/// The engine's gate: a run that reaches the named failpoint connects to this +/// socket and blocks there. Both variables are read by the production +/// `store.compact` path only when the gate is present. +const GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +const FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// A short socket path: the sandbox's own root is longer than `SUN_LEN`. +fn gate_socket() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + PathBuf::from(format!( + "/tmp/layerfs-g65-gate-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) +} + +/// One crash point: the failpoint, the phase `store.status` must project, +/// whether the rebuild is already installed at `G + 1`, and which staged +/// artifacts the kill leaves beside the Store. +#[derive(Clone, Copy, Debug)] +struct Arm { + failpoint: &'static str, + phase: &'static str, + installed: bool, + target: bool, + visited: bool, + /// A kill before the seal ends the process with the writer session's WAL + /// on disk (`-wal`, `-shm`); every later point has closed it. + live_sidecars: bool, +} + +/// `layerfs ` on `home`, parsing the one-line JSON result. +fn call(home: &Path, words: &[&str]) -> SdkValue { + let out = run(&mut layerfs(home, words)); + assert_eq!( + out.status.code(), + Some(0), + "{words:?}: {}", + text(&out.stderr) + ); + json::parse(text(&out.stdout).trim()).unwrap_or_else(|error| { + panic!("{words:?}: {error:?}: {}", text(&out.stdout)); + }) +} + +/// `store.compact s`, the call whose process the crash arms kill. +fn compact(home: &Path) -> SdkValue { + call(home, &["store", "compact", "s"]) +} + +fn generations(report: &SdkValue) -> (u64, u64) { + let generation = report.get("generation").expect("the report's generation"); + let before = generation.get_int("before").expect("generation.before"); + let after = generation.get_int("after").expect("generation.after"); + (before as u64, after as u64) +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +/// Kills `store.compact` for real: waits until the process is stopped inside +/// the run at `failpoint`, then `SIGKILL`s it. The gate keeps the wait from +/// being a sleep, and a process that ends early (a refusal) fails the arm with +/// its own stderr. +fn crash_and_kill(home: &Path, failpoint: &'static str) { + let gate = gate_socket(); + let _ = std::fs::remove_file(&gate); + let listener = UnixListener::bind(&gate).expect("the crash gate socket"); + listener + .set_nonblocking(true) + .expect("a nonblocking crash gate"); + let mut child = Command::new(LAYERFS) + .args(["store", "compact", "s"]) + .env("LAYERFS_HOME", home) + .env(GATE_ENV, &gate) + .env(FAILPOINT_ENV, failpoint) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("the real store.compact process"); + + let deadline = Instant::now() + Duration::from_secs(60); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the child's status") { + let mut stderr = String::new(); + if let Some(mut pipe) = child.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!("store.compact ended before the gate ({failpoint}): {status}: {stderr}"); + } + assert!( + Instant::now() < deadline, + "store.compact never reached the gate ({failpoint})" + ); + std::thread::yield_now(); + } + Err(error) => panic!("crash gate accept: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(60))) + .expect("a bounded gate read"); + let mut ready = [0u8; 1]; + let mut reader = &stream; + reader + .read_exact(&mut ready) + .expect("the child reached the crash gate"); + assert_eq!(ready[0], b'R', "the child stopped at the crash gate"); + + // The real §9 crash: SIGKILL the process that performs the compaction. + child.kill().expect("SIGKILL the store.compact process"); + let status = child.wait().expect("the killed process exits"); + assert_eq!( + status.signal(), + Some(9), + "the store.compact process was SIGKILLed: {status:?}" + ); + drop(stream); + let _ = std::fs::remove_file(&gate); +} + +/// The fixture: a Store, one project, a Branch whose commit writes a large +/// blob, and then a `§3 none` baseline that names `G` without moving it. +fn fixture(tag: &str) -> (PathBuf, PathBuf, u64) { + let home = tempdir(&format!("g65-home-{tag}")); + let source = tempdir(&format!("g65-src-{tag}")); + std::fs::write(source.join("a.txt"), b"the retained bytes\n").unwrap(); + call(&home, &["store", "create", "s"]); + call(&home, &["project", "init", "s/p", source.to_str().unwrap()]); + call(&home, &["branch", "fork", "s/p", "dev", "main"]); + // A large blob so its pack is wholly dead once `dev` is soft-removed, and + // large enough that the rebuild threshold (`dead >= 10% of total`) holds. + let big: Vec = (0..256 * 1024).map(|at| (at % 251) as u8).collect(); + let written = run_with_stdin( + &mut layerfs(&home, &["file", "write", "s/p", "dev", "/big"]), + &big, + ); + assert!(written.status.success(), "{}", text(&written.stderr)); + + // Nothing is dead yet: §3 `none`, so the generation does not move. This + // names `G`, the generation every crash arm starts from. + let baseline = compact(&home); + assert_eq!(baseline.get_text("path"), Some("none"), "{baseline:?}"); + let (before, after) = generations(&baseline); + assert_eq!(before, after, "a `none` run moves no generation"); + + // One wholly dead pack: the Branch that wrote `/big` is soft-removed and + // its objects become unreachable; the next run must reclaim them. + call(&home, &["branch", "remove", "s/p", "dev"]); + (home, source, before) +} + +fn exercise(arm: Arm) { + let (home, source, generation) = fixture(&arm.failpoint.to_lowercase()); + let store = home.join("store/s/store.sqlite"); + assert!(store.is_file(), "{}", store.display()); + + crash_and_kill(&home, arm.failpoint); + + // §9 crash state, read before any handle opens the Store: the exact + // artifact set beside the live file, and the sidecars the kill leaves. + assert_eq!( + with_suffix(&store, ".compact").exists(), + arm.target, + "staged Store at {}", + arm.failpoint + ); + assert_eq!( + with_suffix(&store, ".compact-visited").exists(), + arm.visited, + "visited scratch at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, ".compact-op").exists(), + "op record at {}", + arm.failpoint + ); + if arm.live_sidecars { + assert!( + with_suffix(&store, "-wal").exists(), + "the killed session's WAL at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, "-shm").exists(), + "the killed session's shared memory at {}", + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + if arm.live_sidecars && suffix != "-journal" { + continue; + } + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} beside the Store at {}", + arm.failpoint + ); + assert!( + !with_suffix(&with_suffix(&store, ".compact"), suffix).exists(), + "sidecar {suffix} beside the staged Store at {}", + arm.failpoint + ); + } + + // `store.status` (§11) projects the recorded run: the phase it stood in, + // an uncertain outcome, and an unresolved state because its owner died. + let observed = call(&home, &["store", "status", "s"]); + let op = observed.get("op").expect("the op projection"); + assert!(op.get("id").is_some(), "{observed:?}"); + assert_eq!(op.get_text("phase"), Some(arm.phase), "{observed:?}"); + assert_eq!( + op.get("uncertain"), + Some(&SdkValue::Bool(true)), + "{observed:?}" + ); + assert_eq!( + observed.get_text("state"), + Some("unresolved"), + "{observed:?}" + ); + + // The next run judges the leftover: a rebuild from `G` before install, a + // settle at `G + 1` after it, and no residue either way. + let recovered = compact(&home); + if arm.installed { + assert_eq!(recovered.get_text("path"), Some("none"), "{recovered:?}"); + assert_eq!( + generations(&recovered), + (generation + 1, generation + 1), + "{recovered:?}" + ); + } else { + assert_eq!(recovered.get_text("path"), Some("rebuild"), "{recovered:?}"); + assert_eq!( + generations(&recovered), + (generation, generation + 1), + "{recovered:?}" + ); + } + let residue = recovered.get("residue").expect("the residue"); + assert_eq!(residue.get_int("bytes"), Some(0), "{recovered:?}"); + assert!( + matches!(residue.get("artifacts"), Some(SdkValue::List(items)) if items.is_empty()), + "{recovered:?}" + ); + + // Nothing is left beside the Store, and the op record is settled. + for path in [ + with_suffix(&store, ".compact"), + with_suffix(&store, ".compact-visited"), + with_suffix(&store, ".compact-op"), + ] { + assert!( + !path.exists(), + "{} left at {}", + path.display(), + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} left at {}", + arm.failpoint + ); + } + let settled = call(&home, &["store", "status", "s"]); + assert!( + settled.get("op").and_then(|op| op.get("id")).is_none(), + "{settled:?}" + ); + assert_ne!(settled.get_text("state"), Some("unresolved"), "{settled:?}"); + + // The retained layer still reads, and the home passes the doctor: the + // dead pack is collected while the live one is intact. + let stats = call(&home, &["project", "stats", "s/p"]); + assert!(stats.get_int("files").unwrap_or(0) >= 1, "{stats:?}"); + call(&home, &["file", "stat", "s/p", "main", "/a.txt"]); + let doctor = call(&home, &["doctor"]); + assert_eq!(doctor.get("ok"), Some(&SdkValue::Bool(true)), "{doctor:?}"); + + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&source).unwrap(); +} + +macro_rules! sigkill_at { + ($name:ident, $failpoint:literal, $phase:literal, $installed:literal, $target:literal, $visited:literal, $live_sidecars:literal) => { + #[test] + fn $name() { + exercise(Arm { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + live_sidecars: $live_sidecars, + }); + } + }; +} + +// The mark phase: the open session and admission intent at `BeforeSeal` (the +// kill leaves the writer's WAL sidecars), the sealed Store at `AfterSeal`. +sigkill_at!( + sigkill_before_seal, + "BeforeSeal", + "mark", + false, + false, + false, + true +); +sigkill_at!( + sigkill_after_seal, + "AfterSeal", + "mark", + false, + false, + false, + false +); +// The build: immediately before `VACUUM INTO` (SQLite runs that copy as one +// uninterruptible call), and after the staged Store is built. +sigkill_at!( + sigkill_before_vacuum, + "BeforeVacuum", + "build", + false, + false, + true, + false +); +sigkill_at!( + sigkill_after_build, + "AfterBuild", + "build", + false, + true, + true, + false +); +// The verification and the install boundary. +sigkill_at!( + sigkill_during_verify, + "DuringVerify", + "verify", + false, + true, + true, + false +); +sigkill_at!( + sigkill_before_install, + "BeforeInstall", + "verify", + false, + true, + true, + false +); +// Installed at `G + 1`: after the rename, and partway through the settle. +sigkill_at!( + sigkill_after_install, + "AfterInstall", + "install", + true, + false, + true, + false +); +sigkill_at!( + sigkill_during_settle, + "DuringSettle", + "install", + true, + false, + false, + false +); diff --git a/api/scripts/gen-coverage.mjs b/api/scripts/gen-coverage.mjs index 03b36f7fca..55474981db 100644 --- a/api/scripts/gen-coverage.mjs +++ b/api/scripts/gen-coverage.mjs @@ -101,9 +101,12 @@ function usage(words) { /** What the command line of `spec` must be, by the table: a content or JSON positional is read from * stdin or `--from`, `on_progress` is `--progress`, the output functions are `--stream`. */ function expectedCli(spec) { + const options = spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)); + // the command frames a call with a stdout sink as events: `--events`, last + if (spec.options.some((o) => o.name === "stdout")) options.push("events"); return { positionals: spec.args.filter((a) => !piped(a)).map((a) => ({ name: a.name, optional: a.optional })), - options: spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)), + options, from: spec.args.some(piped), }; } diff --git a/control/crates/layerfs-control/tests/project_delete_worker.rs b/control/crates/layerfs-control/tests/project_delete_worker.rs index 1138ae2e63..6e301384a5 100644 --- a/control/crates/layerfs-control/tests/project_delete_worker.rs +++ b/control/crates/layerfs-control/tests/project_delete_worker.rs @@ -20,7 +20,7 @@ use layerfs_store::sqlite::create::create; use layerfs_store_service::history::lookup; use std::net::TcpListener; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; static NEXT: AtomicU64 = AtomicU64::new(0); @@ -298,10 +298,11 @@ fn the_worker_path_admits_before_the_sidecar_and_releases_after_the_marker() { } /// A fake worker on `listener`: answers its secret, answers each request with -/// `reply`, and shares back the requests it read. -fn fake_worker( +/// `reply`, and shares back the requests it read. A `None` answer drops the +/// connection without a reply, as a lost answer does. +fn fake_worker_with( listener: TcpListener, - reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, + reply: impl Fn(&SdkValue) -> Option + Send + 'static, ) -> Arc>> { let seen: Arc>> = Arc::default(); let requests = seen.clone(); @@ -325,12 +326,22 @@ fn fake_worker( }; let answer = reply(&request); requests.lock().unwrap().push(request); - let _ = write_frame(&mut stream, &answer); + if let Some(answer) = answer { + let _ = write_frame(&mut stream, &answer); + } } }); seen } +/// [`fake_worker_with`] of a worker that always answers. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Arc>> { + fake_worker_with(listener, move |request| Some(reply(request))) +} + /// The `op` of one request the fake worker read. fn op_of(request: &SdkValue) -> &str { request.get_text("op").unwrap_or("") @@ -548,3 +559,94 @@ fn hex_decode(text: &str) -> Vec { .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) .collect() } + +#[test] +fn a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([6; 16]); + let home = caller_home("fake-lost", &address, stack); + // The first marker attempt loses its answer; the second (the retry) + // retires the Stack with the sidecar's `deleted_ms`. + let attempts = Arc::new(AtomicU32::new(0)); + let count = attempts.clone(); + let seen = fake_worker_with(listener, move |request| match op_of(request) { + "project.delete.admit" => Some(admitted_for(stack)), + "project.delete" => { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + None + } else { + Some(removed_for(stack, 42)) + } + } + "project.delete.release" => Some(released_for(request)), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("the marker answer was lost"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + // The sidecar stays `pending`, naming the admission, and the entry stays + // for the retry: an unknown outcome is never reported clean. + let intent = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays"); + assert_eq!( + intent.state, + layerfs_store::project::delete::IntentState::Pending + ); + let op = intent.op.expect("the sidecar names its admission"); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let first = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), first); + assert_eq!(sent_op(&requests[2]), first); + assert_eq!(hex_decode(&first), op.to_vec()); + drop(requests); + + // The retry re-registers the same identity and finishes the entry. + let outcome = remove(&home).expect("the retry settles the unknown outcome"); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None, + "the retry unlinked the sidecar" + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release", + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let same: Vec<&str> = requests[3..].iter().map(sent_op_ref).collect(); + assert_eq!(same, [first.as_str(), first.as_str(), first.as_str()]); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The `op_id` a request's args carry, borrowed. +fn sent_op_ref(request: &SdkValue) -> &str { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") +} diff --git a/control/crates/layerfs-store/Cargo.toml b/control/crates/layerfs-store/Cargo.toml index 0c2a6ad807..830dbf4cd3 100644 --- a/control/crates/layerfs-store/Cargo.toml +++ b/control/crates/layerfs-store/Cargo.toml @@ -31,3 +31,9 @@ postgresql = [ ] # https:// remotes: Cloudflare R2, MinIO behind TLS. tls = ["layerfs-bucket/tls"] + +# The control-level PostgreSQL `store.status` test drives a real server and +# inserts the rows a compaction leaves; this is the pinned client the +# persistence PostgreSQL feature already locks, no new version. +[dev-dependencies] +postgres = "=0.19.14" diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs index 3a7514b54f..0a20fc12c7 100644 --- a/control/crates/layerfs-store/src/compact.rs +++ b/control/crates/layerfs-store/src/compact.rs @@ -21,14 +21,52 @@ use layerfs_home::events::Events; use layerfs_home::home::paths; use layerfs_home::outcome::{failed, refused, Outcome}; use layerfs_persistence::compact_metrics::{self, CompactRefusal}; -use layerfs_persistence::{ - CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, -}; #[cfg(feature = "postgresql")] use layerfs_persistence::PostgresHandles; +use layerfs_persistence::{ + CompactFailpoint, CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; use layerfs_store_service::history::{admission, lookup}; use std::path::Path; +/// The T-G65 crash harness's gate: the real `layerfs store compact` process it +/// starts carries this socket, and the SQLite engine stops at the failpoint +/// below instead of reporting an unknown outcome. Ordinary callers set +/// neither variable, so a run carries no failpoint. +const CRASH_GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +/// The stop point the crash harness names; honored only beside the gate. +const CRASH_FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// The failpoint the crash harness named, or `None` in every ordinary run. +/// +/// A failpoint alone would stop the run in-process; the gate makes the stop +/// observable to the harness, which then `SIGKILL`s this process. Requiring +/// both keeps the hook inert for production callers and for builds without +/// the engine's Unix-socket gate. +#[cfg(unix)] +fn crash_failpoint() -> Option { + std::env::var_os(CRASH_GATE_ENV)?; + match std::env::var(CRASH_FAILPOINT_ENV).ok()?.as_str() { + "BeforeSeal" => Some(CompactFailpoint::BeforeSeal), + "AfterSeal" => Some(CompactFailpoint::AfterSeal), + "BeforeVacuum" => Some(CompactFailpoint::BeforeVacuum), + "AfterBuild" => Some(CompactFailpoint::AfterBuild), + "DuringVerify" => Some(CompactFailpoint::DuringVerify), + "BeforeInstall" => Some(CompactFailpoint::BeforeInstall), + "DuringRename" => Some(CompactFailpoint::DuringRename), + "AfterInstall" => Some(CompactFailpoint::AfterInstall), + "DuringSettle" => Some(CompactFailpoint::DuringSettle), + _ => None, + } +} + +/// No gate, no failpoint: a stop would hang the run in-process. +#[cfg(not(unix))] +fn crash_failpoint() -> Option { + None +} + /// `store.compact(name)`: reclaim one sqlite Store's dead bytes, answering /// the §11 report. pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { @@ -84,6 +122,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { owner_pid: Some(std::process::id()), owner_pid_start, owner_pid_boot, + failpoint: crash_failpoint(), ..CompactRequest::default() }) .map_err(lookup::history)?; diff --git a/control/crates/layerfs-store/src/postgresql/connect.rs b/control/crates/layerfs-store/src/postgresql/connect.rs index 67228c5731..f070ca35bc 100644 --- a/control/crates/layerfs-store/src/postgresql/connect.rs +++ b/control/crates/layerfs-store/src/postgresql/connect.rs @@ -313,3 +313,27 @@ pub fn status(record: &StoreRecord) -> SdkValue { crate::postgresql::open::status(&url, &[SCHEMA.to_owned()]) .unwrap_or_else(|refusal| SdkValue::map([("unavailable", SdkValue::Text(refusal.message))])) } + +/// The Store's compaction operation as `store.status` (T10.5) renders it: +/// `{id, name, phase, started, uncertain, state, phases}`, or null when the +/// Store has none. One status query reads the Store-side `gc_op` and the +/// compaction `gc_intent`; no Store session is opened, no schema is changed +/// and no artifact or scratch schema is read. An unreachable server or a +/// query failure is null here: the `server` field already reports reach. +pub fn compact_status(record: &StoreRecord) -> SdkValue { + #[cfg(feature = "postgresql")] + { + let Ok(url) = url(record) else { + return SdkValue::Null; + }; + match layerfs_persistence::postgresql_compact_status(&url, SCHEMA) { + Ok(Some(op)) => crate::report::compact_status(&op.status()), + _ => SdkValue::Null, + } + } + #[cfg(not(feature = "postgresql"))] + { + let _ = record; + SdkValue::Null + } +} diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs index b780d22e9b..49f5cda341 100644 --- a/control/crates/layerfs-store/src/project/delete.rs +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -330,6 +330,13 @@ fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool ]) } +/// The side effects and the project-scope admission one finishing attempt +/// runs against: together, so `mark_and_finish` stays a small call. +struct Finish<'a> { + effects: &'a mut dyn Effects, + admission: Admission, +} + /// The mutating part of `delete_with` after the project scope is admitted: /// the identity save, the sidecar, the marker and the entry removal (spec §4). fn mark_and_finish( @@ -339,9 +346,12 @@ fn mark_and_finish( stack: LayerStackId, intent: Option, mut deleted_ms: i64, - effects: &mut impl Effects, - admission: Admission, + finish: Finish<'_>, ) -> Outcome { + let Finish { + effects, + admission, + } = finish; effects.prepare()?; let mut current = match intent { Some(intent) => intent, @@ -466,7 +476,16 @@ pub fn delete_with( } }; let attempt = mark_and_finish( - home, store, project, stack, intent, deleted_ms, effects, admission, + home, + store, + project, + stack, + intent, + deleted_ms, + Finish { + effects: &mut *effects, + admission, + }, ); let released = effects.release(admission); match attempt { diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs index a02dc621e6..cf6ad16ed8 100644 --- a/control/crates/layerfs-store/src/report.rs +++ b/control/crates/layerfs-store/src/report.rs @@ -28,7 +28,7 @@ pub fn compact_report(report: &CompactReport) -> SdkValue { .residue .artifacts .iter() - .map(|artifact| SdkValue::text(artifact)) + .map(SdkValue::text) .collect(), ), ), diff --git a/control/crates/layerfs-store/src/status.rs b/control/crates/layerfs-store/src/status.rs index 72e277bc7b..664f7d4a87 100644 --- a/control/crates/layerfs-store/src/status.rs +++ b/control/crates/layerfs-store/src/status.rs @@ -16,20 +16,22 @@ use std::path::Path; /// /// While a compaction runs, or after a dead owner left one, `op` is that run's /// projection and `state` is `busy`/`unresolved` (§11); `phases` names the -/// backend's ordered phases. The compaction record is read from -/// `.compact-op` alone: the Store database and its artifacts are never -/// opened for it. +/// backend's ordered phases. A sqlite compaction record is read from +/// `.compact-op` alone, and a PostgreSQL one from the Store-side +/// `gc_op` with its `gc_intent` on one status query (T10.5): neither opens +/// the Store database, changes a schema, nor reads an artifact. pub fn status(home: &Path, name: &str) -> Outcome { let record = record_of(home, name)?; let (holders, _) = lease::read_holders(home, name)?; let mut down = None; let mut more = Vec::new(); let store_ref = record.store_ref()?; - let mut compaction = SdkValue::Null; + let compaction: SdkValue; if let StoreRef::Postgres { .. } = store_ref { more.push(("server", crate::postgresql::connect::status(&record))); - // A PostgreSQL compaction's op lives in the Store's `gc_op`; T9 reads - // it here with its rebuild. This path reads no file. + // T10.5: a PostgreSQL compaction's op is the Store's `gc_op` row read + // with its `gc_intent`; this path opens no Store and reads no file. + compaction = crate::postgresql::connect::compact_status(&record); } else { more.push(("machine", SdkValue::text(store_ref.machine()?))); // the file's size where it lives, starting nothing (P4), and the diff --git a/control/crates/layerfs-store/tests/postgresql_status.rs b/control/crates/layerfs-store/tests/postgresql_status.rs new file mode 100644 index 0000000000..3d0d2e933b --- /dev/null +++ b/control/crates/layerfs-store/tests/postgresql_status.rs @@ -0,0 +1,405 @@ +//! T10.5: the PostgreSQL branch of `store.status`, through the control call. +//! +//! `store.status` of a PostgreSQL Store reads the Store-side `gc_op` row with +//! its compaction `gc_intent` in one status query, projects the operation +//! `{id, name, phase, started, uncertain}`, and separates a live owner +//! (`busy`) from a leftover (`unresolved`); `phases` is PostgreSQL's §8 order +//! `mark`, `build`, `swap`, `cleanup`. A Store with no row — never compacted, +//! or a run that settled and removed its row — is idle, with `op: null` and +//! `phases: null`. +//! +//! This is the control-level end-to-end case T10.5 was missing: the +//! persistence suite proves the reader, and this test drives the call the SDK +//! and CLI actually reach (`layerfs_store::sqlite::family::status`, the +//! kind-neutral `store.status` route) against a real PostgreSQL Store. The +//! rows a compaction leaves are written directly, so the test proves the +//! projection, not a failpoint: a live owner is a real non-idle backend +//! recorded in `pg_stat_activity`. +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` to a +//! libpq connection string or URL (docs/architecture/engine/store/testing.md). +//! The test creates and drops its own database, because `store.status` reads +//! the one schema a 2.0 PostgreSQL Store uses (`layerfs`); a shared schema +//! would collide with another run. Without the variable it reports NOT_RUN. +#![cfg(feature = "postgresql")] + +use layerfs_bridge::control::SdkValue; +use layerfs_history::HistoryCatalogConfig; +use layerfs_home::home; +use layerfs_home::names::StoreName; +use layerfs_persistence::{PostgresHandles, PostgresSelection}; +use layerfs_storage::StoragePolicy; +use layerfs_store::record::StoreRecord; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The §8 phases `store.status` names for a PostgreSQL compaction, in order. +const PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + +const BINDING: &[u8] = b"t105-control-postgresql-status"; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t105-control-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The URL of `database` on the same server as `source`: a URL's path is +/// replaced; a libpq key/value string gains a final `dbname`, which wins. +fn database_url(source: &str, database: &str) -> String { + if source.starts_with("postgres://") || source.starts_with("postgresql://") { + let (base, query) = match source.split_once('?') { + Some((base, query)) => (base, format!("?{query}")), + None => (source, String::new()), + }; + let scheme = base.find("://").expect("a PostgreSQL URL") + 3; + match base[scheme..].find('/').map(|at| scheme + at) { + Some(at) => format!("{}/{database}{query}", &base[..at]), + None => format!("{base}/{database}{query}"), + } + } else { + format!("{source} dbname={database}") + } +} + +/// One disposable database: `store.status` reads the fixed `layerfs` schema, +/// so a test run needs a database nothing else uses. +struct Database { + source: String, + name: String, +} + +impl Database { + fn create(source: &str) -> Self { + let name = format!( + "lfs_t105_ctl_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let mut admin = postgres::Client::connect(source, postgres::NoTls).unwrap(); + admin + .batch_execute(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .unwrap(); + admin + .batch_execute(&format!("CREATE DATABASE {name}")) + .unwrap(); + Self { + source: source.to_owned(), + name, + } + } + + fn url(&self) -> String { + database_url(&self.source, &self.name) + } +} + +impl Drop for Database { + fn drop(&mut self) { + if let Ok(mut admin) = postgres::Client::connect(&self.source, postgres::NoTls) { + let _ = admin.batch_execute(&format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + self.name + )); + } + } +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn op_id(byte: u8) -> String { + format!("op-{}", hex(&[byte; 16])) +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + match value { + SdkValue::List(items) => items + .iter() + .map(|item| { + if let SdkValue::Text(text) = item { + text.clone() + } else { + panic!("not text: {item:?}"); + } + }) + .collect(), + other => panic!("not a list: {other:?}"), + } +} + +/// What a compaction leaves in `gc_op`/`gc_intent`; `pid` and +/// `backend_start` name the owner the §9 rule tests for liveness. +struct OpRow<'a> { + op_id: [u8; 16], + phase: &'a str, + generation: i64, + started_ms: i64, + pid: i64, + backend_start: &'a str, +} + +fn insert(admin: &mut postgres::Client, row: &OpRow<'_>) { + let artifact = format!("lfs_gc_{}", hex(&row.op_id)); + admin + .execute( + "INSERT INTO layerfs.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,\ + started_ms,artifact,pid,backend_start) \ + VALUES (1,$1,$2,$3,$4,$5,$6,$7,$3,$8,$9)", + &[ + &row.generation, + &row.phase, + &artifact, + &11i64, + &12i64, + &vec![9u8; 32], + &row.started_ms, + &row.pid, + &row.backend_start, + ], + ) + .unwrap(); + admin + .execute( + "INSERT INTO layerfs.gc_intent \ + (op_id,kind,scope_kind,owner_runtime,owner_boot,state,uncertain,phase,\ + generation,artifact,updated_ms) \ + VALUES ($1,'compaction','store',$2,$3,'live',FALSE,$4,$5,$6,$7)", + &[ + &row.op_id.to_vec(), + &"runtime-t105", + &"boot-t105", + &row.phase, + &row.generation, + &artifact, + &1_700_000_000_000i64, + ], + ) + .unwrap(); +} + +fn clear(admin: &mut postgres::Client) { + admin + .batch_execute( + "DELETE FROM layerfs.gc_op; \ + DELETE FROM layerfs.gc_intent WHERE kind='compaction'", + ) + .unwrap(); +} + +/// Waits until the recorded owner backend is gone from `pg_stat_activity`, +/// as the persistence fixture does. +fn wait_gone(admin: &mut postgres::Client, pid: i32, start: &str) { + for _ in 0..400 { + let live: bool = admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2)", + &[&pid, &start], + ) + .unwrap() + .get(0); + if !live { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend never ended"); +} + +/// The real control entry point: `store.status` for the Store in `home`. +fn status(home: &Path) -> SdkValue { + let name = StoreName::parse("pgs").unwrap(); + layerfs_store::sqlite::family::status(home, &name).unwrap() +} + +#[test] +fn postgresql_store_status_projects_busy_leftover_and_idle() { + let Some(source) = url() else { + return; + }; + let database = Database::create(&source); + let url = database.url(); + let handles = PostgresHandles::create( + &PostgresSelection::new(&url, "layerfs"), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + + let home = tempdir("home"); + home::ensure(&home).unwrap(); + // `store.connect` records exactly this shape; `store.status` reads it. + StoreRecord::connected("pgs", &url, "layerfs", "off", None) + .write(&home) + .unwrap(); + let mut admin = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + + // Idle: no compaction row, so no op and no phases. + let idle = status(&home); + assert_eq!(idle.get_text("name"), Some("pgs")); + assert_eq!(idle.get_text("kind"), Some("postgresql")); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!( + idle.get("op"), + Some(&SdkValue::Null), + "an idle Store has no op" + ); + assert_eq!( + idle.get("phases"), + Some(&SdkValue::Null), + "an idle Store has no phases" + ); + assert!( + matches!(idle.get("server"), Some(SdkValue::Map(_))), + "the server status is reported beside the op" + ); + + // A live owner: an open, non-idle backend is what the §9 dead-owner rule + // treats as still running the op. + let mut owner = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + owner.batch_execute("BEGIN; SELECT 1").unwrap(); + let (pid, start): (i32, String) = { + let row = owner + .query_one( + "SELECT pg_backend_pid(),backend_start::text FROM pg_stat_activity \ + WHERE pid=pg_backend_pid()", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + let started_ms = 1_700_000_000_000i64; + let live = OpRow { + op_id: [0xab; 16], + phase: "build", + generation: 7, + started_ms, + pid: i64::from(pid), + backend_start: &start, + }; + insert(&mut admin, &live); + + let capture = layerfs_persistence::compact_metrics::CompactMetricsCapture::start(); + let busy = status(&home); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + op, + &SdkValue::map([ + ("id", SdkValue::text(op_id(0xab))), + ("name", SdkValue::text("store.compact")), + ("phase", SdkValue::text("build")), + ("started", SdkValue::Int(started_ms)), + ("uncertain", SdkValue::Bool(true)), + ]), + "the op is exactly the five §11 fields" + ); + assert_eq!(texts(busy.get("phases").unwrap()), PHASES); + assert!( + op.get("state").is_none() && op.get("phases").is_none(), + "state and phases sit beside the op" + ); + + // A gone owner leaves the same row: uncertain, and now unresolved. + owner.close().unwrap(); + wait_gone(&mut admin, pid, &start); + let leftover = status(&home); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("id").unwrap()), op_id(0xab)); + assert_eq!(op.get_text("name"), Some("store.compact")); + assert_eq!(op.get_text("phase"), Some("build")); + assert_eq!(op.get_int("started"), Some(started_ms)); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!(texts(leftover.get("phases").unwrap()), PHASES); + + // The two observed states emit the bounded `compact.status{state}` sample + // once each; idle emits none. + let metrics = capture.finish(); + let sample = |state: &str| { + metrics + .samples() + .iter() + .find(|sample| sample.name == "compact.status" && sample.label("state") == Some(state)) + .map_or(0, |sample| sample.value) + }; + assert_eq!(sample("busy"), 1); + assert_eq!(sample("unresolved"), 1); + assert_eq!(sample("idle"), 0, "an idle status is not a compaction op"); + + // Every recorded phase is projected under its §8 name; a dead owner is + // an unresolved leftover for each. + for (index, phase) in PHASES.iter().enumerate() { + clear(&mut admin); + let byte = u8::try_from(index).unwrap(); + insert( + &mut admin, + &OpRow { + op_id: [byte; 16], + phase, + generation: 7, + started_ms, + pid: 99_999_999, + backend_start: "2000-01-01 00:00:00+00", + }, + ); + let projected = status(&home); + assert_eq!(projected.get_text("state"), Some("unresolved")); + assert_eq!( + projected.get("op").unwrap().get_text("phase"), + Some(*phase), + "{phase}: the row records the phase" + ); + let expected_id = op_id(byte); + assert_eq!( + projected.get("op").unwrap().get_text("id"), + Some(expected_id.as_str()) + ); + assert_eq!(texts(projected.get("phases").unwrap()), PHASES); + } + + // A settled run removes the row: idle again, with no op and no phases. + clear(&mut admin); + let idle = status(&home); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + + drop(admin); + drop(handles); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/core/docs/issues/project-41/gc-acceptance-matrix.md b/core/docs/issues/project-41/gc-acceptance-matrix.md index 70ad3d7a43..e462bf967e 100644 --- a/core/docs/issues/project-41/gc-acceptance-matrix.md +++ b/core/docs/issues/project-41/gc-acceptance-matrix.md @@ -1,14 +1,36 @@ # GC acceptance matrix: #52 evidence plan for the §12 cases > Draft matrix for the frozen `LayerFS-GC-spec.md` §12 (the spec wins; this -> page adds no contract). Main layer `32d78561a`. AC-01..AC-57 are the §12 -> bullets one-to-one, in spec order; AC-58..AC-60 pin sub-steps of §7/§8 that -> §12 folds into broader cases. Each row's expected result restates spec text -> only; a mismatch is a spec bug, not a matrix change. +> page adds no contract). The plan was written at main layer `32d78561a`; +> statuses below are re-checked on main `32ae51ca` (T-G75) and reconciled +> against the reports published on main: T12.3c at `3282f9f6865` regenerated +> [`gc-t12.3a-sqlite-matrix-report.md`](gc-t12.3a-sqlite-matrix-report.md) +> (executed 8 / covered 29 / partial 16 / deferred 7 / SQLite gaps 0), beside +> T12.3a/T12.3b/T9/T10/T-G6a; T-G28 added the real-process SIGKILL arms and +> moved AC-01 to `partial`; T-G60 extended them to the mark, copy, verify and +> rename points; T-G38 and T-G54 supplied the real PostgreSQL +> reply-loss and restricted-role privilege arms and moved AC-44 to `covered`; +> T-G65 killed the real `layerfs store compact` process itself at the mark, +> build, verify and install points and moved AC-01 back to `covered`. +> The final counts are executed 8 / covered 30 / partial 16 / deferred 6; +> T-G75 re-ran the SQLite suites from `32ae51ca` and re-checked all 60 rows. +> AC-01..AC-57 are the §12 bullets one-to-one, in +> spec order; AC-58..AC-60 pin sub-steps of §7/§8 that §12 folds into broader +> cases. Each row's expected result restates spec text only; a mismatch is a +> spec bug, not a matrix change. > -> Status vocabulary: `open` (no §12-conforming evidence recorded yet), -> `pass`, `fail`, `waived`. §13 says nothing has been measured: every E3 row -> starts `open`. Tiers are this page's evidence scale, not contract. +> Status vocabulary: `executed` (a T12.3c runner arm really executes the +> case), `covered` (a named existing test or fixture covers it), `partial` +> (some named arms proven; the evidence cell says what remains), `deferred` (a +> PostgreSQL-only row the SQLite runner leaves to T12.3b by name), `pass` (the +> named arm passed in the recorded run), `pending` (no §12-conforming evidence +> recorded yet; none remain). The dispositions +> are the SQLite half; the PostgreSQL half is T12.3b and its evidence stays +> named below. T-G66 measured the 50 GB `none` and `rebuild` arms (one run each); +> §13's measurement caveat now applies to the 30% dead and 50 GB-live arms, and +> an E3 row records the three-run or property evidence that exists and stays +> `partial` where it does not. +> Tiers are this page's evidence scale, not contract. ## Evidence tiers (E0-E3) @@ -21,68 +43,123 @@ ## Matrix -| Case ID | Contract (spec §) | Backend | Fixture / command | Expected result | Tier | Status | -|---|---|---|---|---|---|---| -| AC-01 | §12 Crash injection; §9 | both | Store with one wholly dead pack; `kill -9` the daemon at mark, build, verify, before and after install; next `store.compact` | Store is wholly old or wholly new; the leftover is judged by §9; generation and digest asserted | E2 | open | -| AC-02 | §12 Backend matrix; §4.1 | both | Run every removal, admission, report and name case on SQLite and PostgreSQL; open a converted 1.0 Store on both | Each case's §12 result holds on both backends; the additive step adds the §4.1 tables on both | E2 | open | -| AC-03 | §12 Crash matrix; §9 | both | Named failpoints: SQLite mark, build, verify, install, WAL switch, artifact removal, op settle; PG mark, build, swap, cleanup, scratch drop; plus lost COMMIT, power loss (fsync/rename) and unresolved op/lease takeover | §9 state, generation, digest and closure hold after resume; only a live owner blocks; an unreachable machine reports unknown, never clean | E2 | open | -| AC-04 | §12 No live object lost; §6.1-§6.2 | both | Property test over random fork, commit, merge, publish, `branch.remove`, `project.delete` and `remove_file` histories; compare closure sets before and after | No live object reaches a collected row; the retained closure is unchanged | E3 | open | -| AC-05 | §12 Independent walk; §9 | both | After compaction, walk the new Store with `diagnostic/scan.rs` and run the full doctor on the fixtures (#155 D-23) | Walk and doctor pass on the compacted Store; no dangling reference | E2 | open | -| AC-06 | §12 Regression; §6.4 | both | After compaction, Save bytes whose `file_digest` row named a collected root | The Save succeeds; no stale digest row blocks it | E2 | open | -| AC-07 | §12 Fences; §10 | both | A connection that bypassed admission; a second compaction; a remote PG daemon holding a mount; a one-shot SDK transaction open at swap time | Bypass gets `StaleStore`; second compaction gets `Busy`; the mount blocks compaction and the matching removal, naming itself; the open transaction makes the swap abandon | E2 | open | -| AC-08 | §12 Admission races; §5.1 | both | Start a mount and a removal of the same Branch together | Never both succeed; the loser gets `MountsActive` or `Busy` | E2 | open | -| AC-09 | §12 Names; §4.4 | both | `NameTaken` after removal (Branch and project); create collision (Branch and project; project create was `Exists`); free name after a rebuild collects the row; `ReservedName` for `main` | Exact codes per §4.4; only a rebuild frees an unreferenced name | E2 | open | -| AC-10 | §12 remove_file; §4.3 | both | `Removed` for Commits at or below the frontier, before and after compaction; the next Commit of a mounted older head records the deletion; a file created later at the path is visible | Codes and visibility per §4.3; 2.0 frees no bytes for the removed path | E2 | open | -| AC-11 | §12 remove_file and derived nodes; §4.3 | both | Non-FF `branch.merge` of two heads at or below the frontier; a post-frontier re-creation; a FF merge control; `branch.publish` of an at-or-below Commit; a merge that cannot apply the whiteout | Path stays hidden, whiteout materialized; post-frontier file visible; FF merge creates no tree; published Layer hidden, its mount whiteouts, next Commit records the deletion; inapplicable whiteout refused `Removed` | E2 | open | -| AC-12 | §12 Removed path in a merge; §4.3 | both | Non-FF merge of two at-or-below heads; merge of an at-or-below head with a post-frontier re-creation | No conflict for the removed path, it is omitted; the re-created file is kept and only an ordinary edit conflict on it is reported | E2 | open | -| AC-13 | §12 Unmerged Branches; §4.3 | both | `remove_file` then a Commit on an unmerged Branch created before the frontier; another Commit after the frontier | The earlier Commit's path is hidden, the later one does not inherit it | E2 | open | -| AC-14 | §12 Layer-headed mount; §4.3 | both | Branch whose head is empty and whose base Layer is at or below the frontier; mount it | The path is whiteouted at mount | E2 | open | -| AC-15 | §12 Visibility only; §6.2 | both | Object reachable only through the removed path; marker hiding reads/listings; mount, merge and publish whiteouts | No bytes freed; the object is retained; reads/listings hide the path; derived trees whiteout it | E2 | open | -| AC-16 | §12 Published Branch; §4.4 | both | Branch published by a retained Layer; Branch named by a retained `commit.branch_id`; `branch.remove` then rebuild | Name stays taken; only an unreferenced Branch's name is freed by a rebuild (not by a `none` pass) | E2 | open | -| AC-17 | §12 Late base; §7 step 4 | both | Delta base in an already-passed pack; a depth-2 chain | Worklist copies both; `pack_id` preserved | E2 | open | -| AC-18 | §12 Digest roots; §6.4 | both | Uncommitted saved file's `file_digest` row; another retained root reaching its content | Row collected unless the retained root reaches the content | E2 | open | -| AC-19 | §12 Path grammar; §4.3 | both | `/a` and `a`; empty, `/`, `//a`, `a/`, `a//b`, `./a`, `a/./b`, dot and dot-dot components, backslash, NUL, >4096 bytes, >256 components; directory subtree; child of a removed non-directory; symlink leaf and intermediate component; hardlink | Accepted paths hide correctly; bad forms are `InvalidInput`; directory hides its subtree; child of removed non-directory `NotFound`; intermediate symlink `InvalidInput`, leaf hidden; hardlink survives | E2 | open | -| AC-20 | §12 Repeated removal; §4.3 | both | Second `remove_file` after a re-creation; repeat with no newer project node; a commit in another project; never-existing path | Re-creation hidden; repeat `UpToDate`; other project neither rewrites nor advances the marker; never-existing path accepted, hides nothing | E2 | open | -| AC-21 | §12 Marker epoch; §5 | both | One-shot write racing `branch.remove` or `project.delete`; merge racing `remove_file` | `Conflict`, no Commit left; retry applies the whiteout | E2 | open | -| AC-22 | §12 Concurrent merge and removal (property); §4.3 | both | Property test: a merge and a `remove_file` started together | The path is never left visible in a derived tree; one side wins, the other sees the marker or gets `Conflict` and retries with the whiteout | E3 | open | -| AC-23 | §12 Marker cleanup; §6.4 | both | Rebuild of a Store with a deleted project and a live project with markers; a `none` pass | No `path_removed` row for the deleted project; live markers kept; `none` writes nothing and retains them | E2 | open | -| AC-24 | §12 Position state; §4.1 | both | Converted Store; `store.compact`; a marker write | Serials backfilled; `next_history_serial` never falls; `project_epoch` copied and bumped only on a marker write | E2 | open | -| AC-25 | §12 Marker survival; §4.4 | both | Retained `branch_retired`, `layer_stack_retired` and `path_removed`; a collected base row | Retained markers survive a rebuild; a collected row takes its marker with it | E2 | open | -| AC-26 | §12 project.delete retry; §4 | both | Marker commits and home removal fails; same-home retry; own-Init identity save failure | Hidden, entry present, `home_entry_removed: false`; retry finishes and returns the stored `deleted_ms`, no second marker; the identity save precedes the marker, a save failure removes nothing | E2 | open | -| AC-27 | §12 Home listing; §4.2 | both | Second home's stale entry; `project.list` and `project.get`; Store calls | Stale entry absent from `project.list`, `project.get` answers `NoSuchProject`; Store calls agree | E2 | open | -| AC-28 | §12 PG cleanup; §8 Phase C | PG | Commit/save during Phase C; crash mid-cleanup; failpoint at the marker/base-row batch boundary | The late write survives the high-water bound; cleanup resumes from `gc_op`; marker and base row deleted in one batch, no resurrection or orphan | E2 | open | -| AC-29 | §12 Admission; §5.1 | both | Mount from another home or machine; `machine_stopped` evidence; a second mount of one project; an uncertain dead op | The mount is visible in the Store-side intent; `machine_stopped` ignored; the second mount proceeds; `Unresolved` observed and settled by the owning call | E2 | open | -| AC-30 | §12 NameTaken; §4.4 | both | `NameTaken` detail for live, retired and deleted holders; forced two-reference fixture (Layer FK and `commit.branch_id`); one-reference and no-reference controls; error matrix | Detail carries holder kind, name and state; two references keep the name, the controls do not; matrix covers `MountsActive`, `Busy`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NameTaken`, `ReservedName`, `Removed`, `InvalidInput` | E2 | open | -| AC-31 | §12 Trace salt; §4.3 | both | Call log with path arguments; `Removed` refusal; rebuild | Log stores ``, trace key is the keyed hash; `Removed` echoes the path; the salt is copied by the rebuild | E2 | open | -| AC-32 | §12 Admission (liveness); §5.1 | both | Live-but-unreachable Sandbox; `machine_stopped`; `gc_intent` from another home; non-owning call | Live-unreachable is `Busy`; `machine_stopped` ignored; the other home's intent visible; `Unresolved` observed by the non-owner and settled by the owner | E2 | open | -| AC-33 | §12 Checkpoint/install; §7 steps 1, 7 | SQLite | Live reader during `wal_checkpoint(TRUNCATE)`; a sidecar or `-mj*` present at install | `busy>0` makes the checkpoint `Busy` with no artifact and no op; a sidecar at install is `Conflict` with no writes | E2 | open | -| AC-34 | §12 Bind failure; §5.1 | both | Committed intent whose bind fails; failed intent delete | Intent deleted in the error path; the failed delete is reported; no `MountsActive` remains | E2 | open | -| AC-35 | §12 PG refcount; §5.2 | PG | Two mounts, one unmount; failed all-or-none acquire | Store and project advisory keys stay after one unmount; the failed acquire unlocks the taken subset | E2 | open | -| AC-36 | §12 Worker/lost reply; §4, §9 | both | Unreachable worker op; lost reply; retry | Op is `unknown`, never clean; the sidecar is kept; the retry resolves retired, live or absent | E2 | open | -| AC-37 | §12 Trace; §4.3 | both | Six path names; refusal details; upgraded Store | The six path names are redacted; refusal details are not traced; the salt is backfilled | E2 | open | -| AC-38 | §12 Artifacts (cross-backend); §9 | both | SQLite artifact set `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; PG `lfs_gc_` schema; a stray schema | SQLite leaves and removes the full set; PG records the name before CREATE, a remote machine can remove it, a stray schema is `Exists` | E2 | open | -| AC-39 | §12 Position state (2.0 backfill); §4.1 | both | Every upgraded Store (2.0 included); `gc_op` settle; serial allocation | One `project_epoch` row per stack; `gc_op` cleared on settle and not copied; `next_history_serial` never lowered | E2 | open | -| AC-40 | §12 Digest agreement; §6.4 | both | Same fixture on SQLite and PostgreSQL; retired head | Same `live_heads_digest`; a retired head changes it | E2 | open | -| AC-41 | §12 PostgreSQL mark; §8 Phase M | PG | Mark transfer; table copy; a base object held only by a dead pack | Only `markBytes` crosses the wire; the copy moves no pack bytes; the base is kept | E2 | open | -| AC-42 | §12 Reserved pack; §8 Phase B | PG | Save reserves a pack/ordinal before the mark and publishes during Phase A (or C); cleanup failpoint | `store_totals` moves; the swap `Conflict`s; the pack stays in the old Store; the failpoint keeps it; no dangling location | E2 | open | -| AC-43 | §12 PG cleanup (resume); §8 Phase C | PG | After the swap at `G+1`; crash mid-cleanup; totals, ACL, trigger, hints | Store usable at `G+1`; cleanup resumes; totals/ACL/trigger/hints asserted; no `inode` on PG | E2 | open | -| AC-44 | §12 PG privileges and reply; §8 | PG | Missing `CREATE`, `TEMP` or `pg_read_all_stats`; lost `COMMIT` reply | `Unsupported`; the lost reply is resolved by proving the backend ended, then reading the incarnation | E2 | open | -| AC-45 | §12 SQLite layouts; §7 step 2 | SQLite | Schema 36 (`Monolithic`), 37 (`GroupRows`), 38 (`GroupRowsIndexed`) and their 42/43/44 acquisition variants rebuild | `Monolithic` copies `pack` body and no `pack_unit`; `GroupRows` copies both; `GroupRowsIndexed` copies `pack_unit` and the schema creates its index; `sqlite_sequence` kept when present; `init_*`/acquisition tables empty | E1 | open | -| AC-46 | §12 Install fence; §10 | SQLite | Connection that bypassed the writer lane opening a transaction after install; in-flight transaction; sealed-state or sidecar change at install | `StaleStore`; the in-flight transaction is excluded by the lane (documented limit); a sealed-state or sidecar change is `Conflict` with no writes | E2 | open | -| AC-47 | §12 Artifacts (SQLite exact); §9 | SQLite | Crash mid-compact; next compact; `none` pass; stray name with no op | Crash leaves the full `.compact` set; the next compact removes all of it; `none` leaves no `.compact`; a stray name is `Exists` | E2 | open | -| AC-48 | §12 Threshold and failures; §3 | both | 9%, 10% and 30% wholly dead; empty Store (0/0); started rebuild whose refined share falls below 10%; `Capacity`; verify failure | 9% is `none`, 10% and 30% are `rebuild`; empty Store is `none` with `zeroReason: empty-store`; the started rebuild completes; `Capacity` and verify failure abort, delete the artifact set and leave the old Store unchanged apart from step 1's checkpoint | E2 | open | -| AC-49 | §12 Mount/compact race; §5, §8 | both | Mount during compact; mount wins the race; Commit during the rebuild | `Busy` with the compact op; compaction returns `MountsActive` when the mount wins; the Commit is not lost (PG Phase A write makes the swap `Conflict`; Phase C survives the high-water bound) | E2 | open | -| AC-50 | §12 Profile and scope; §10 | both | All four calls under a folder scope; MCP admin profile | Refused `out of scope`; allowed only in the admin profile | E2 | open | -| AC-51 | §12 Progress; §11 | both | `store.compact` running; `store.status` | `store.status` shows the op phase: SQLite `mark`, `build`, `verify`, `install`; PG `mark`, `build`, `swap`, `cleanup` | E2 | open | -| AC-52 | §12 Report; §11 | both | `none` and `rebuild` paths; every §11 field | `kind: Compacted`, snake_case keys, every §11 field with its unit and sampling point (`freelist_count * page_size`, old size before rename, history `logical` delta, peak temp, `physicalShrink`, `keptWaste`, `residue`, `estimate`, capped `cap`, `zeroReason`); `markBytes` is a #52 metric, not a result field; bound at least refined; PG `backendReusable` = 0 and PG/none `deadBytesRefined` equals `deadBytesBound` | E2 | open | -| AC-53 | §12 Read surface; §4.3 | both | `file.read`/`stat`/`list`/`glob`/`diff`/`revision`/`export`/cp-from before and after compaction; `edit`/`grep`/`blame`/`rm`/`mv`/`mkdir`/`cp`/`apply`; `branch.diff`/`commit.diff` | §4.3 shapes; visible/hidden/absent diff table; `removed_count` 0 and >0; one whiteout case per one-shot tree creator | E2 | open | -| AC-54 | §12 Restore and listing; §4.2 | both | `backup.restore` before and after a removal; second home's stale entry | Restore succeeds; the stale entry stays out; the §4.2 guard test and allow-list hold | E2 | open | -| AC-55 | §12 Error matrix; §10 | both | Exact code and message per case, including `StaleStore`, `Removed`, `InvalidInput` for `project.remove`, `Exists`, `Unsupported`, `Capacity` | Exact code and message; snake_case details | E2 | open | -| AC-56 | §12 Digest golden vector; §6.4 | both | Fixed-id fixture (out-of-order ids, a retired Branch and stack, differing timestamps); absent head | One literal vector on both backends; absent head is 33 zero bytes, not 32; widths fixed at 17/33 | E3 | open | -| AC-57 | §12 Speed and space; §13 | both | At least 3 runs per arm, median and range, declared caches, generated fixtures; §13 arms: `none` at 9%, `rebuild` at 10% and 30%, 50 GB total/35 GB live, 50 GB live, each removal type, removal-then-compact, `remove_file` 0-freed | Downtime per GB live and per GB freed, bytes written, peak disk, PG `ACCESS EXCLUSIVE` duration; absolute bytes when nothing is freed; build phase vs plain `VACUUM INTO`; mark reported separately; a 50% time/5% space option rejected | E3 | open | -| AC-58 | §7 step 1.5 (refined decision) | SQLite | After the seal: `ATTACH` the old file `immutable=1`, create `.compact-visited`, run the complete §6.1+§6.2 closure, choose `none`/`rebuild` from the refined live set | `none` deletes the scratch and stops with no `.compact` and no residue; `rebuild` keeps the scratch, reuses `visited` in step 4; step 2 creates the new file only on `rebuild` | E2 | open | -| AC-59 | §7 step 1 (seal) | SQLite | `PRAGMA wal_checkpoint(TRUNCATE)`; checked-close the last session; sidecar check under the writer guard; `seal_holding()` | `busy == 0` required; WAL never unlinked; sealed `(size, mtime, db-header change counter)` recorded before release; the guard passes into the seal | E2 | open | -| AC-60 | §8 Phase B step 10 (standby) | PG | Effective `synchronous_commit` at least `remote_write` with `synchronous_standby_names` set; standby missing before Phase M; standby missing inside the swap; standby dies after the check | Wait only when required; `FIRST`/`ANY` standbys must be `state=streaming` with matching `sync_state`; a missing standby abandons before COMMIT with no writes (`Busy`, standby list); a standby dying after the check blocks COMMIT | E2 | open | +| Case ID | Contract (spec §) | Backend | Fixture / command | Expected result | Tier | Status | Evidence | +|---|---|---|---|---|---|---|---| +| AC-01 | §12 Crash injection; §9 | both | Store with one wholly dead pack; `kill -9` the daemon at mark, build, verify, before and after install; next `store.compact` | Store is wholly old or wholly new; the leftover is judged by §9; generation and digest asserted | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_before_seal`..`#crash_after_install` (in-process failpoints); T-G28/T-G60 `sqlite_compact_crash.rs::real_process` `#sigkill_before_seal`, `#sigkill_after_seal` (mark), `#sigkill_during_copy` (copy; the gate is immediately before `VACUUM INTO`, because SQLite's copy cannot be interrupted mid-call), `#sigkill_during_verify` (verify), `#sigkill_during_rename`, `#sigkill_before_install` and `#sigkill_after_install` (install), `#sigkill_after_build` (build) and `#sigkill_during_settle` (settle) - 9 re-executed-test-child SIGKILL arms; T-G65 `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` `#sigkill_before_seal`, `#sigkill_after_seal` (mark), `#sigkill_before_vacuum`, `#sigkill_after_build` (build), `#sigkill_during_verify`, `#sigkill_before_install` (verify), `#sigkill_after_install`, `#sigkill_during_settle` (install) - 8 SIGKILL arms that start the built `layerfs store compact` command, wait through the engine's socket gate and kill that process (`status.signal() == 9`) at a Store with one wholly dead pack, then reopen through the CLI and assert `G`/`G + 1`, the artifact set and killed-session sidecars, the `store.status` phase/`uncertain`/`unresolved`, the next compact settling with empty residue, the retained layer readable and `doctor` clean; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover` (in-process, no `kill -9`) - `covered` for SQLite because the killed process is the production process that performs the compaction (no daemon serves Store calls, #352 F-020); the residual is that the long-lived `layerfs worker` serving a machine's Store is still not exercised, and the PostgreSQL half remains T12.3b's | +| AC-02 | §12 Backend matrix; §4.1 | both | Run every removal, admission, report and name case on SQLite and PostgreSQL; open a converted 1.0 Store on both | Each case's §12 result holds on both backends; the additive step adds the §4.1 tables on both | E2 | partial | T12.3a `sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away`; T12.3b `postgresql_position_columns` 4/4 (converted 1.0 Store arm is SQLite-only) | +| AC-03 | §12 Crash matrix; §9 | both | Named failpoints: SQLite mark, build, verify, install, WAL switch, artifact removal, op settle; PG mark, build, swap, cleanup, scratch drop; plus lost COMMIT, power loss (fsync/rename) and unresolved op/lease takeover | §9 state, generation, digest and closure hold after resume; only a live owner blocks; an unreachable machine reports unknown, never clean | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_after_wal_switch` + mark/build/verify/install failpoints; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover`; lost COMMIT/power loss deferred | +| AC-04 | §12 No live object lost; §6.1-§6.2 | both | Property test over random fork, commit, merge, publish, `branch.remove`, `project.delete` and `remove_file` histories; compare closure sets before and after | No live object reaches a collected row; the retained closure is unchanged | E3 | partial | T12.1 `sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript`; T12.3a `sqlite_compact_matrix.rs#arm_random_store_shapes` (seeded; full random generator deferred) | +| AC-05 | §12 Independent walk; §9 | both | After compaction, walk the new Store with `diagnostic/scan.rs` and run the full doctor on the fixtures (#155 D-23) | Walk and doctor pass on the compacted Store; no dangling reference | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql`; `postgresql_diagnostic` 5/5 | +| AC-06 | §12 Regression; §6.4 | both | After compaction, Save bytes whose `file_digest` row named a collected root | The Save succeeds; no stale digest row blocks it | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_digest_row_after_collection` - a Save whose `file_digest` row named a root the rebuild collected; the row goes with its root, recording the same content again succeeds and points at the new root; PostgreSQL half T12.3b | +| AC-07 | §12 Fences; §10 | both | A connection that bypassed admission; a second compaction; a remote PG daemon holding a mount; a one-shot SDK transaction open at swap time | Bypass gets `StaleStore`; second compaction gets `Busy`; the mount blocks compaction and the matching removal, naming itself; the open transaction makes the swap abandon | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_mount_and_compaction_exclude_each_other`; `sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction` - bypass gives `StaleStore`; a live mount makes compaction `MountsActive` naming it; a live compaction refuses a mount and a second compaction `BusyOp`, leaving no op; the open one-shot SDK transaction arm is PostgreSQL (T12.3b) | +| AC-08 | §12 Admission races; §5.1 | both | Start a mount and a removal of the same Branch together | Never both succeed; the loser gets `MountsActive` or `Busy` | E2 | covered | T12.3a `admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner` (SQLite half; PG arm not recorded) | +| AC-09 | §12 Names; §4.4 | both | `NameTaken` after removal (Branch and project); create collision (Branch and project; project create was `Exists`); free name after a rebuild collects the row; `ReservedName` for `main` | Exact codes per §4.4; only a rebuild frees an unreferenced name | E2 | covered | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; `gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches` (SQLite half) | +| AC-10 | §12 remove_file; §4.3 | both | `Removed` for Commits at or below the frontier, before and after compaction; the next Commit of a mounted older head records the deletion; a file created later at the path is visible | Codes and visibility per §4.3; 2.0 frees no bytes for the removed path | E2 | covered | T12.3a `path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position` (SQLite half) | +| AC-11 | §12 remove_file and derived nodes; §4.3 | both | Non-FF `branch.merge` of two heads at or below the frontier; a post-frontier re-creation; a FF merge control; `branch.publish` of an at-or-below Commit; a merge that cannot apply the whiteout | Path stays hidden, whiteout materialized; post-frontier file visible; FF merge creates no tree; published Layer hidden, its mount whiteouts, next Commit records the deletion; inapplicable whiteout refused `Removed` | E2 | partial | T12.3c `sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent`; `sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation`; `sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden`; `sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount` - non-FF whiteouts, the FF control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute; partial because a merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed | +| AC-12 | §12 Removed path in a merge; §4.3 | both | Non-FF merge of two at-or-below heads; merge of an at-or-below head with a post-frontier re-creation | No conflict for the removed path, it is omitted; the re-created file is kept and only an ordinary edit conflict on it is reported | E2 | covered | T12.3c `sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent`; `sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation`; `sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit` - a removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept and only an ordinary edit conflict on it is reported | +| AC-13 | §12 Unmerged Branches; §4.3 | both | `remove_file` then a Commit on an unmerged Branch created before the frontier; another Commit after the frontier | The earlier Commit's path is hidden, the later one does not inherit it | E2 | covered | T12.3c `sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit` - the earlier Commit on an unmerged Branch created before the frontier is hidden by the marker; a Commit after the frontier does not inherit the path | +| AC-14 | §12 Layer-headed mount; §4.3 | both | Branch whose head is empty and whose base Layer is at or below the frontier; mount it | The path is whiteouted at mount | E2 | covered | T12.3c `sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount`; `sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes` - a Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount, and its next Commit records the deletion | +| AC-15 | §12 Visibility only; §6.2 | both | Object reachable only through the removed path; marker hiding reads/listings; mount, merge and publish whiteouts | No bytes freed; the object is retained; reads/listings hide the path; derived trees whiteout it | E2 | partial | T12.3a `kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs`; T12.1 fixture (read/listing hiding and derived whiteouts not one test) | +| AC-16 | §12 Published Branch; §4.4 | both | Branch published by a retained Layer; Branch named by a retained `commit.branch_id`; `branch.remove` then rebuild | Name stays taken; only an unreferenced Branch's name is freed by a rebuild (not by a `none` pass) | E2 | covered | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; T12.1 fixture (SQLite half) | +| AC-17 | §12 Late base; §7 step 4 | both | Delta base in an already-passed pack; a depth-2 chain | Worklist copies both; `pack_id` preserved | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` (depth-2 chain, `pack_id` kept) | +| AC-18 | §12 Digest roots; §6.4 | both | Uncommitted saved file's `file_digest` row; another retained root reaching its content | Row collected unless the retained root reaches the content | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-19 | §12 Path grammar; §4.3 | both | `/a` and `a`; empty, `/`, `//a`, `a/`, `a//b`, `./a`, `a/./b`, dot and dot-dot components, backslash, NUL, >4096 bytes, >256 components; directory subtree; child of a removed non-directory; symlink leaf and intermediate component; hardlink | Accepted paths hide correctly; bad forms are `InvalidInput`; directory hides its subtree; child of removed non-directory `NotFound`; intermediate symlink `InvalidInput`, leaf hidden; hardlink survives | E2 | partial | T12.3a `gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers`; `path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position` (full form table missing) | +| AC-20 | §12 Repeated removal; §4.3 | both | Second `remove_file` after a re-creation; repeat with no newer project node; a commit in another project; never-existing path | Re-creation hidden; repeat `UpToDate`; other project neither rewrites nor advances the marker; never-existing path accepted, hides nothing | E2 | partial | T12.3a `path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date`; `#an_unknown_project_is_missing` (re-creation/other-project arms partial) | +| AC-21 | §12 Marker epoch; §5 | both | One-shot write racing `branch.remove` or `project.delete`; merge racing `remove_file` | `Conflict`, no Commit left; retry applies the whiteout | E2 | partial | T12.3a `epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back`; T12.3b `postgresql_compact_fixtures.rs#a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write` (true concurrent PG trigger arm missing) | +| AC-22 | §12 Concurrent merge and removal (property); §4.3 | both | Property test: a merge and a `remove_file` started together | The path is never left visible in a derived tree; one side wins, the other sees the marker or gets `Conflict` and retries with the whiteout | E3 | covered | T12.3c `sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible`; `epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back` - a deterministic barrier starts a merge and a `remove_file` together for three rounds; the loser sees the marker or gets `Conflict` and retries with the whiteout, and a derived tree never shows the path; PostgreSQL half T12.3b | +| AC-23 | §12 Marker cleanup; §6.4 | both | Rebuild of a Store with a deleted project and a live project with markers; a `none` pass | No `path_removed` row for the deleted project; live markers kept; `none` writes nothing and retains them | E2 | partial | T12.3a `project_delete.rs#deleting_a_project_hides_it_and_marks_once`; T12.1 fixture (dedicated marker-cleanup arm missing) | +| AC-24 | §12 Position state; §4.1 | both | Converted Store; `store.compact`; a marker write | Serials backfilled; `next_history_serial` never falls; `project_epoch` copied and bumped only on a marker write | E2 | covered | T12.3a `gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order`; T12.3b `postgresql_position_columns` + fixture | +| AC-25 | §12 Marker survival; §4.4 | both | Retained `branch_retired`, `layer_stack_retired` and `path_removed`; a collected base row | Retained markers survive a rebuild; a collected row takes its marker with it | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-26 | §12 project.delete retry; §4 | both | Marker commits and home removal fails; same-home retry; own-Init identity save failure | Hidden, entry present, `home_entry_removed: false`; retry finishes and returns the stored `deleted_ms`, no second marker; the identity save precedes the marker, a save failure removes nothing | E2 | covered | T12.3a `project_delete.rs#retry_classifies_retired_live_and_absent`; `#deleting_a_project_hides_it_and_marks_once` (SQLite half) | +| AC-27 | §12 Home listing; §4.2 | both | Second home's stale entry; `project.list` and `project.get`; Store calls | Stale entry absent from `project.list`, `project.get` answers `NoSuchProject`; Store calls agree | E2 | covered | T12.3a `project_delete.rs#paging_skips_a_hidden_project`; `#every_layer_stack_lookup_carries_the_marker_filter` (SQLite half) | +| AC-28 | §12 PG cleanup; §8 Phase C | PG | Commit/save during Phase C; crash mid-cleanup; failpoint at the marker/base-row batch boundary | The late write survives the high-water bound; cleanup resumes from `gc_op`; marker and base row deleted in one batch, no resurrection or orphan | E2 | deferred | T12.3b `postgresql_compact_fixtures.rs#a_save_after_the_swap_survives_the_resumed_cleanup`; `#every_postgresql_failpoint_leaves_a_resumable_leftover` (marker/base-row batch failpoint still missing) | +| AC-29 | §12 Admission; §5.1 | both | Mount from another home or machine; `machine_stopped` evidence; a second mount of one project; an uncertain dead op | The mount is visible in the Store-side intent; `machine_stopped` ignored; the second mount proceeds; `Unresolved` observed and settled by the owning call | E2 | covered | T12.3a `admission.rs#a_mount_blocks_all_three_removals_until_released`; T12.3b `postgresql_compact.rs` owner-adoption tests (mount-work admission deferred) | +| AC-30 | §12 NameTaken; §4.4 | both | `NameTaken` detail for live, retired and deleted holders; forced two-reference fixture (Layer FK and `commit.branch_id`); one-reference and no-reference controls; error matrix | Detail carries holder kind, name and state; two references keep the name, the controls do not; matrix covers `MountsActive`, `Busy`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NameTaken`, `ReservedName`, `Removed`, `InvalidInput` | E2 | partial | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; `gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect` (PG detail matrix deferred) | +| AC-31 | §12 Trace salt; §4.3 | both | Call log with path arguments; `Removed` refusal; rebuild | Log stores ``, trace key is the keyed hash; `Removed` echoes the path; the salt is copied by the rebuild | E2 | covered | T12.3a `gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log`; `#the_trace_key_is_salted_by_the_store_policy_salt` (PG salt backfill deferred) | +| AC-32 | §12 Admission (liveness); §5.1 | both | Live-but-unreachable Sandbox; `machine_stopped`; `gc_intent` from another home; non-owning call | Live-unreachable is `Busy`; `machine_stopped` ignored; the other home's intent visible; `Unresolved` observed by the non-owner and settled by the owner | E2 | partial | T12.3a `admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved`; T12.3b owner-adoption tests (other-home intent deferred) | +| AC-33 | §12 Checkpoint/install; §7 steps 1, 7 | SQLite | Live reader during `wal_checkpoint(TRUNCATE)`; a sidecar or `-mj*` present at install | `busy>0` makes the checkpoint `Busy` with no artifact and no op; a sidecar at install is `Conflict` with no writes | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_checkpoint_busy`; `#arm_sidecar_at_install` | +| AC-34 | §12 Bind failure; §5.1 | both | Committed intent whose bind fails; failed intent delete | Intent deleted in the error path; the failed delete is reported; no `MountsActive` remains | E2 | partial | T12.3c `sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active` - a committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains; partial because the failed-delete sub-arm is not inducible on SQLite (the schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register) and the explicit `Cleanup` report is covered by the seam code path, not executed; PostgreSQL half T12.3b | +| AC-35 | §12 PG refcount; §5.2 | PG | Two mounts, one unmount; failed all-or-none acquire | Store and project advisory keys stay after one unmount; the failed acquire unlocks the taken subset | E2 | deferred | T12.3b deferred: PG advisory-key refcount is the mount/admission layer; no compaction fixture | +| AC-36 | §12 Worker/lost reply; §4, §9 | both | Unreachable worker op; lost reply; retry | Op is `unknown`, never clean; the sidecar is kept; the retry resolves retired, live or absent | E2 | covered | T12.3c `project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry`; `project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry`; `project_delete.rs#an_absent_stack_finishes_a_stale_entry` - a lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent; PostgreSQL half T12.3b | +| AC-37 | §12 Trace; §4.3 | both | Six path names; refusal details; upgraded Store | The six path names are redacted; refusal details are not traced; the salt is backfilled | E2 | partial | T12.3a `gc_trace.rs#the_redacted_set_covers_every_path_bearing_call`; `#the_trace_key_is_salted_by_the_store_policy_salt` (PG salt backfill deferred) | +| AC-38 | §12 Artifacts (cross-backend); §9 | both | SQLite artifact set `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; PG `lfs_gc_` schema; a stray schema | SQLite leaves and removes the full set; PG records the name before CREATE, a remote machine can remove it, a stray schema is `Exists` | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover` (remote/stray-schema arms deferred) | +| AC-39 | §12 Position state (2.0 backfill); §4.1 | both | Every upgraded Store (2.0 included); `gc_op` settle; serial allocation | One `project_epoch` row per stack; `gc_op` cleared on settle and not copied; `next_history_serial` never lowered | E2 | covered | T12.3a `gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order`; T12.3b `postgresql_position_columns` 4/4 + fixture | +| AC-40 | §12 Digest agreement; §6.4 | both | Same fixture on SQLite and PostgreSQL; retired head | Same `live_heads_digest`; a retired head changes it | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_live_heads_digest_vector_is_the_sqlite_literal` + rebuild fixture | +| AC-41 | §12 PostgreSQL mark; §8 Phase M | PG | Mark transfer; table copy; a base object held only by a dead pack | Only `markBytes` crosses the wire; the copy moves no pack bytes; the base is kept | E2 | deferred | T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` (depth-2 `pack_id` kept; `markBytes` is a #52 metric, not a result field) | +| AC-42 | §12 Reserved pack; §8 Phase B | PG | Save reserves a pack/ordinal before the mark and publishes during Phase A (or C); cleanup failpoint | `store_totals` moves; the swap `Conflict`s; the pack stays in the old Store; the failpoint keeps it; no dangling location | E2 | deferred | T12.3b deferred: the PG round has no pause point between build and swap; the reserved-pack race is not deterministically injectable | +| AC-43 | §12 PG cleanup (resume); §8 Phase C | PG | After the swap at `G+1`; crash mid-cleanup; totals, ACL, trigger, hints | Store usable at `G+1`; cleanup resumes; totals/ACL/trigger/hints asserted; no `inode` on PG | E2 | deferred | T12.3b `#the_rebuild_report_and_markers_hold_on_postgresql`; `#a_save_after_the_swap_survives_the_resumed_cleanup` | +| AC-44 | §12 PG privileges and reply; §8 | PG | Missing `CREATE`, `TEMP` or `pg_read_all_stats`; lost `COMMIT` reply | `Unsupported`; the lost reply is resolved by proving the backend ended, then reading the incarnation | E2 | covered | T-G38 `postgresql_compact.rs#a_killed_backend_before_the_swap_commit_leaves_the_store_at_g` (a real `pg_terminate_backend(pid, backend_start)` parks Phase B's `gc_op` update before COMMIT: the Store is wholly at `G`, the owner is proved ended and the next `store.compact` adopts the `G` leftover); `#a_killed_backend_during_phase_c_resumes_at_g_plus_1` (a real kill in Phase C after the swap COMMIT: wholly at `G + 1`, the next call reads the incarnation and finishes Phase C); `#a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1` (`PostgresAfterSwap` simulated lost reply, owner then really killed, fresh connection resolves). The privilege half is T-G54 `postgresql_compact.rs#a_role_without_create_on_the_database_is_refused_as_unsupported`, `#a_role_without_create_on_the_store_schema_is_refused_as_unsupported`, `#a_role_without_temp_is_refused_as_unsupported` and `#a_role_without_pg_read_all_stats_is_refused_as_unsupported` (each restricted role is `Unsupported` naming its missing privilege, asserted before the intent, the mark and any scratch schema) plus `#a_fully_privileged_role_compacts` (the owner role with all five grants rebuilds at `G + 1`); `access.rs` now also checks both `CREATE`s and `pg_read_all_stats` | +| AC-45 | §12 SQLite layouts; §7 step 2 | SQLite | Schema 36 (`Monolithic`), 37 (`GroupRows`), 38 (`GroupRowsIndexed`) and their 42/43/44 acquisition variants rebuild | `Monolithic` copies `pack` body and no `pack_unit`; `GroupRows` copies both; `GroupRowsIndexed` copies `pack_unit` and the schema creates its index; `sqlite_sequence` kept when present; `init_*`/acquisition tables empty | E1 | covered | T12.3a `sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none`; `#group_rows_indexed_layout_opens_compacts_and_takes_none`; `#acquisition_layout_opens_compacts_and_clears_working_rows` | +| AC-46 | §12 Install fence; §10 | SQLite | Connection that bypassed the writer lane opening a transaction after install; in-flight transaction; sealed-state or sidecar change at install | `StaleStore`; the in-flight transaction is excluded by the lane (documented limit); a sealed-state or sidecar change is `Conflict` with no writes | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_sidecar_at_install`; `sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale`; `#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction` | +| AC-47 | §12 Artifacts (SQLite exact); §9 | SQLite | Crash mid-compact; next compact; `none` pass; stray name with no op | Crash leaves the full `.compact` set; the next compact removes all of it; `none` leaves no `.compact`; a stray name is `Exists` | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_during_copy`; `#crash_during_settle`; `sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted` | +| AC-48 | §12 Threshold and failures; §3 | both | 9%, 10% and 30% wholly dead; empty Store (0/0); started rebuild whose refined share falls below 10%; `Capacity`; verify failure | 9% is `none`, 10% and 30% are `rebuild`; empty Store is `none` with `zeroReason: empty-store`; the started rebuild completes; `Capacity` and verify failure abort, delete the artifact set and leave the old Store unchanged apart from step 1's checkpoint | E2 | covered | T12.3a `sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path`; `#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold`; `#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold`; `#an_empty_store_takes_the_none_path`; `#a_capacity_failure_leaves_the_store_unchanged` (PG boundary arms not separately run) | +| AC-49 | §12 Mount/compact race; §5, §8 | both | Mount during compact; mount wins the race; Commit during the rebuild | `Busy` with the compact op; compaction returns `MountsActive` when the mount wins; the Commit is not lost (PG Phase A write makes the swap `Conflict`; Phase C survives the high-water bound) | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_mount_during_compaction_is_busy`; `sqlite_compact_matrix.rs#arm_commit_during_rebuild`; `admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner` - a mount during compaction is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild is `StaleStore` or lands on `G + 1` and its retry is never lost; PG Phase A/C arms are T12.3b | +| AC-50 | §12 Profile and scope; §10 | both | All four calls under a folder scope; MCP admin profile | Refused `out of scope`; allowed only in the admin profile | E2 | covered | T12.3a `calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover`; `gc_calls.rs#the_four_gc_calls_take_ids_only` (SQLite-side control test; PG not separate) | +| AC-51 | §12 Progress; §11 | both | `store.compact` running; `store.status` | `store.status` shows the op phase: SQLite `mark`, `build`, `verify`, `install`; PG `mark`, `build`, `swap`, `cleanup` | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_status_phases`; T10 `compact_report_status.rs#status_projects_the_sidecar_phase_and_state`; T-G6a `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases`; T10.5 `postgresql_status.rs#every_postgresql_failpoint_projects_its_phase` (PG `store.status` reads `gc_op`) | +| AC-52 | §12 Report; §11 | both | `none` and `rebuild` paths; every §11 field | `kind: Compacted`, snake_case keys, every §11 field with its unit and sampling point (`freelist_count * page_size`, old size before rename, history `logical` delta, peak temp = pre-truncate staged copy + new file + rollback-journal bound + visited, `physicalShrink`, `keptWaste`, `residue`, `estimate`, capped `cap`, `zeroReason`); `markBytes` is a #52 metric, not a result field; bound at least refined; PG `backendReusable` = 0 and PG/none `deadBytesRefined` equals `deadBytesBound` | E2 | covered | T12.1 golden `sqlite_compact_golden.json`; T10 `compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape`; T12.3b `#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-53 | §12 Read surface; §4.3 | both | `file.read`/`stat`/`list`/`glob`/`diff`/`revision`/`export`/cp-from before and after compaction; `edit`/`grep`/`blame`/`rm`/`mv`/`mkdir`/`cp`/`apply`; `branch.diff`/`commit.diff` | §4.3 shapes; visible/hidden/absent diff table; `removed_count` 0 and >0; one whiteout case per one-shot tree creator | E2 | partial | T12.3a `project_export.rs#export_omits_a_path_hidden_by_remove_file`; `gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers` (full read-surface table not executed) | +| AC-54 | §12 Restore and listing; §4.2 | both | `backup.restore` before and after a removal; second home's stale entry | Restore succeeds; the stale entry stays out; the §4.2 guard test and allow-list hold | E2 | partial | T12.3a `restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install`; `restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it` (restore-window arm `Unsupported` on this Sandbox) | +| AC-55 | §12 Error matrix; §10 | both | Exact code and message per case, including `StaleStore`, `Removed`, `InvalidInput` for `project.remove`, `Exists`, `Unsupported`, `Capacity` | Exact code and message; snake_case details | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_error_codes`; `#arm_checkpoint_busy`; `sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted` (full engine/API message matrix partial) | +| AC-56 | §12 Digest golden vector; §6.4 | both | Fixed-id fixture (out-of-order ids, a retired Branch and stack, differing timestamps); absent head | One literal vector on both backends; absent head is 33 zero bytes, not 32; widths fixed at 17/33 | E3 | partial | T12.1 SQLite literal vector; T12.3b `postgresql_compact_fixtures.rs#the_live_heads_digest_vector_is_the_sqlite_literal` (3/3 identical runs) | +| AC-57 | §12 Speed and space; §13 | both | At least 3 runs per arm, median and range, declared caches, generated fixtures; §13 arms: `none` at 9%, `rebuild` at 10% and 30%, 50 GB total/35 GB live, 50 GB live, each removal type, removal-then-compact, `remove_file` 0-freed | Downtime per GB live and per GB freed, bytes written, peak disk, PG `ACCESS EXCLUSIVE` duration; absolute bytes when nothing is freed; build phase vs plain `VACUUM INTO`; mark reported separately; a 50% time/5% space option rejected | E3 | partial | T-G61 `sqlite_compact_capacity.rs#capacity_measure` + `sqlite_compact_capacity_measure.json`: a real 9.987 GB Store, `none` at 8.88% dead (3 runs, median 149.318 s) and `rebuild` at 10.00% dead (3 runs, median 246.371 s; mark 149.089 s, build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink 1.017 GB, peak temp 9.144 GB) plus one WAL/FULL rebuild (284.078 s); T-G66 `sqlite_compact_capacity.rs#capacity_measure` + `sqlite_compact_capacity_measure_50gb.json`: a real 49.999 GB Store (384,850 packs; 347,490 live objects; 50.235 GB physical at `none`, 50.863 GB at `rebuild`), `none` at 8.876% dead (1 run, 1,633.585 s, 0 B freed) and `rebuild` at 10.000% dead (1 run, 5,131.835 s; mark 784.251 s, build 4,254.371 s, verify 85.557 s, install 7.311 s; dead 5.062 GB, shrink 5.100 GB, receipt peak temp 45.784 GB under the old `new + visited` formula, sampled real peak 143.1 GB = ~2.9x the Store, T-G72 corrected receipt bound 147.609 GB = 2.90x the old file and `sqlite_compact_capacity.rs#the_peak_temp_receipt_covers_the_staged_copy_and_the_rollback_journal` bounds the journal + unlinked rewrite temp on a moderate fixture); T12.3a `sqlite_compact_matrix.rs#arm_measurement` + `sqlite_compact_measure.json` (tiny fixture); T-G6a `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases` (PG phase durations); §13's 30% dead / 50 GB-live arms, each removal type, removal-then-compact, the plain `VACUUM INTO` comparison and PG `ACCESS EXCLUSIVE` remain deferred/[E]; model source: [`gc-capacity-model.md`](gc-capacity-model.md) (10 GB three-run [M], 50 GB one-run [M] for `none`/`rebuild`; receipt `peakTempBytes` bounds the staged copy + journal + unlinked rewrite temp (T-G72)) | +| AC-58 | §7 step 1.5 (refined decision) | SQLite | After the seal: `ATTACH` the old file `immutable=1`, create `.compact-visited`, run the complete §6.1+§6.2 closure, choose `none`/`rebuild` from the refined live set | `none` deletes the scratch and stops with no `.compact` and no residue; `rebuild` keeps the scratch, reuses `visited` in step 4; step 2 creates the new file only on `rebuild` | E2 | covered | T12.1 fixture; T12.3a `sqlite_compact_crash.rs#crash_before_copy`; `#crash_during_copy` | +| AC-59 | §7 step 1 (seal) | SQLite | `PRAGMA wal_checkpoint(TRUNCATE)`; checked-close the last session; sidecar check under the writer guard; `seal_holding()` | `busy == 0` required; WAL never unlinked; sealed `(size, mtime, db-header change counter)` recorded before release; the guard passes into the seal | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_checkpoint_busy`; `#arm_seal`; `sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale` | +| AC-60 | §8 Phase B step 10 (standby) | PG | Effective `synchronous_commit` at least `remote_write` with `synchronous_standby_names` set; standby missing before Phase M; standby missing inside the swap; standby dies after the check | Wait only when required; `FIRST`/`ANY` standbys must be `state=streaming` with matching `sync_state`; a missing standby abandons before COMMIT with no writes (`Busy`, standby list); a standby dying after the check blocks COMMIT | E2 | deferred | T12.3b deferred: a real standby needs a second cluster/GUCs; `compact.rs` predicate unit tests pin the E1 decision | +## Evidence index + +| Source | Report | Tests / fixtures | AC ranges | +|---|---|---|---| +| T12.1 golden fixture | [gc-t12-sqlite-compaction-fixtures.md](gc-t12-sqlite-compaction-fixtures.md) | `tests/sqlite_compact_fixtures.rs`, `tests/fixtures/sqlite_compact_golden.json` | AC-04, 05, 17, 18, 25, 38, 40, 47, 48, 52, 56, 58 | +| T12.3c SQLite gap closure | [gc-t12.3a-sqlite-matrix-report.md](gc-t12.3a-sqlite-matrix-report.md) (regenerated by T12.3c; "SQLite gaps: none") | `engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs` (8 executed rows); `engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs`, `sdk_path_merge_removal.rs`, `sdk_mount_bind_failure.rs`; `engine/crates/layerfs-persistence/tests/epoch_fence.rs`; `control/crates/layerfs-control/tests/project_delete_worker.rs` | AC-06, 07, 11, 12, 13, 14, 22, 34, 36, 49 | +| T12.3a SQLite matrix | [gc-t12.3a-sqlite-matrix-report.md](gc-t12.3a-sqlite-matrix-report.md) | `tests/sqlite_compact_matrix.rs` (executed arms), `sqlite_compact.rs`, `sqlite_compact_crash.rs`, `sqlite_compact_fixtures.rs`, `admission.rs`, `path_removal.rs`, `branch_removal.rs`, `project_delete.rs`, `epoch_fence.rs`, `gc_backfill.rs`, `kept_objects.rs`, `gc_trace.rs`, `gc_calls.rs`, `calls_scope.rs`, `project_export.rs`, `restore_install.rs`, `restored_records.rs` | SQLite half of AC-01..AC-60; executed AC-33/46/51/55/59 | +| T12.3b PostgreSQL acceptance | [gc-t12.3b-postgresql-acceptance.md](gc-t12.3b-postgresql-acceptance.md) | `tests/postgresql_compact_fixtures.rs`, `postgresql_compact.rs`, `postgresql_position_columns.rs`, `postgresql_open_check.rs`, `postgresql_diagnostic.rs` | PG half of AC-02, 28-32, 34-44, 56, 60; also AC-01, 03, 05, 17, 18, 21, 24, 25, 49, 51, 52 | +| T9 PostgreSQL rebuild | [gc-t9-postgresql-rebuild.md](gc-t9-postgresql-rebuild.md) | `tests/postgresql_compact.rs` | AC-28, 41, 43, 44, 49, 60; Phases M/A/B/C | +| T10 report and status | [gc-t10-report-status.md](gc-t10-report-status.md) | `control/crates/layerfs-store/tests/compact_report_status.rs`; `sqlite_compact.rs#the_report_projects_every_section_11_field` | AC-51, 52 | +| T-G6a PG metrics | `engine/crates/layerfs-persistence/src/compact_metrics.rs` (`CompactPhase::{PostgresMark,PostgresBuild,PostgresSwap,PostgresCleanup}`) | `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases`, `#a_none_path_records_only_the_pg_mark_phase` | AC-51, 57 | +| T8 SQLite rebuild | [gc-t8-sqlite-rebuild.md](gc-t8-sqlite-rebuild.md) | `tests/sqlite_compact.rs`, `sqlite_compact_crash.rs` | AC-33, 46, 47, 48, 58, 59 | +| T-G28/T-G60 real-process SIGKILL | - (T-G28 on main layer `32a7d0cf`; T-G60 on `task-g60`) | `sqlite_compact_crash.rs::real_process` (9 SIGKILL arms: mark x2, copy, verify, rename, build, before/after install, settle; child re-execution) | AC-01 | +| T-G65 real-process `store.compact` kill | - (T-G65 on `task-g65`) | `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` (8 SIGKILL arms on the built `layerfs store compact` process: mark x2, build x2, verify x2, install x2); gated failpoint in `control/crates/layerfs-store/src/compact.rs` | AC-01 | +| T-G38/T-G54 PG reply loss and privileges | - (main layers `32dc46a2`, `32a2621d`) | `postgresql_compact.rs` (real `pg_terminate_backend` kills, restricted-role fixtures); `backend/postgresql/access.rs` privilege check | AC-44 | +| T-G61/T-G66 SQLite capacity | [gc-capacity-model.md](gc-capacity-model.md) §1 (10 GB three-run [M], 50 GB one-run [M]) | `engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs` (`#capacity_measure`, opt-in/ignored); `tests/fixtures/sqlite_compact_capacity_measure.json`, `tests/fixtures/sqlite_compact_capacity_measure_50gb.json` | AC-57 | +| Gap record | T12.3a/T12.3b/T12.3c `## Gaps` / `SQLite gaps` | none | SQLite gaps 0 after T12.3c (AC-06, 11, 12, 13, 14, 22, 34, 36 closed); AC-44 covered at T-G38/T-G54; AC-35, 42, 60 deferred | + +T12.3c's report is on main as the regenerated +`gc-t12.3a-sqlite-matrix-report.md` linked above; its eight SQLite-gap rows +now stand `executed` / `covered` / `partial` and its SQLite-gap list is empty. +`deferred` rows name the layer that owns them; none is silently waived. + +## Status summary + +At the plan layer all 60 rows were `pending`; the dispositions after T12.3c, +T-G28, T-G38/T-G54 and T-G65 are: + +| Status | Rows | Count | +|---|---|---| +| `executed` | AC-06, 07, 33, 46, 49, 51, 55, 59 | 8 | +| `covered` | AC-01, 03, 05, 08, 09, 10, 12, 13, 14, 16, 17, 18, 22, 24, 25, 26, 27, 29, 31, 36, 38, 39, 40, 44, 45, 47, 48, 50, 52, 58 | 30 | +| `partial` | AC-02, 04, 11, 15, 19, 20, 21, 23, 30, 32, 34, 37, 53, 54, 56, 57 | 16 | +| `deferred` | AC-28, 35, 41, 42, 43, 60 | 6 | +| `pass` | none | 0 | +| `pending` | none | 0 | + +AC-01 moved from `covered` to `partial` at T-G28 because its real-process +SIGKILL arms re-executed the test binary, not the production process. T-G60 +extended those arms to every phase. T-G65 closed the gap: +`api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` starts the built +`layerfs` command with `store compact`, waits through the engine's socket gate +and `SIGKILL`s that process at the mark, build, verify and install points, so +the killed process is the one that performs the compaction (no daemon serves +Store calls, #352 F-020). AC-01 is `covered` again; the residual note is that +the long-lived `layerfs worker` that serves a machine's Store is still not the +killed process. The six `deferred` rows are +PostgreSQL-only: T12.3b executes the PG halves +of AC-28, 41 and 43, while AC-35 (advisory-key refcount), AC-42 +(reserved-pack race) and AC-60 (a real standby) need server control the +harness does not have. AC-44 moved to `covered` at T-G54: T-G38's real +`pg_terminate_backend` kills cover the lost `COMMIT` reply, and T-G54's +restricted-role fixtures prove each missing `CREATE`, `TEMP` or +`pg_read_all_stats` is `Unsupported` before any scratch schema is created. No row is `pending` +and no SQLite evidence gap remains (T12.3c). ## Coverage map (required area -> cases) @@ -103,6 +180,8 @@ ## Notes +- Statuses record evidence, not #52 acceptance; T12.3c, T12.3b, T-G28, T-G60, T-G65 and T-G75 keep acceptance at #52. +- Evidence cells name the test/file in the report, not a new expected result. - `markBytes` is measured in AC-41 and AC-57, never reported as a `store.compact` field (§11). - A case is accepted at E2 only when the named backend's failpoints are exercised; E3 rows additionally need three runs or the property test, per §12 Speed and space. -- Evidence recording (run id, fixture hash, spec layer, artifact paths) goes beside the status once #52 runs each case. +- Evidence recording (run id, fixture hash, spec layer, artifact paths) is the next #52 step; the cells above name tests, not run ids. diff --git a/core/docs/issues/project-41/gc-capacity-model.md b/core/docs/issues/project-41/gc-capacity-model.md new file mode 100644 index 0000000000..f9a9dd2bd2 --- /dev/null +++ b/core/docs/issues/project-41/gc-capacity-model.md @@ -0,0 +1,122 @@ +# GC capacity model: a 50 GB Store (spec section 13) + +> Deferred 50 GB / §13 capacity model for `layerfs-project#41`. Adds no contract; [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) wins, `§N` cites it. +> **[M]** = measured on the cited artifact; **[E]** = extrapolated from the stated assumptions. Nothing marked [E] is a measurement. +> **T-G61** measured a real **10.0 GB** Store (`sqlite_compact_capacity_measure.json`): three `none` runs at 8.88% dead, three `rebuild` runs at 10.00% dead and one WAL/FULL rebuild. **T-G66** then measured the **50 GB** arm on the same host (`sqlite_compact_capacity_measure_50gb.json`): a **49.999 GB** pack-byte Store (384,850 packs, 347,490 live whole-file objects, 33,850 dead = 8.876%, built in 761.2 s), **one** `none` run and **one** `rebuild` run at 10.000% dead. One run per arm is what §13 asks of the 50 GB arm, not a three-run median; the 10 GB three-run medians stay the repeated-run basis. **The 50 GB numbers do not scale linearly from the 10 GB ones**: the 10 GB Store and its rebuild artifacts fitted the host's page cache, the 50 GB one does not, so its `rebuild` is **85.5 min** and its real temp peak on disk is **143.1 GB above the pre-rebuild baseline** (**~2.9×** the Store) — not the [E] ~20 min and 1.9×. **T-G72** replaced the receipt's `new + visited` peak with a documented bound over the run's temp artifacts (pre-truncate staged copy + new file + rollback journal + visited): **147.609 GB** for the same 50 GB `rebuild`, 3.2% above the sampled peak and no longer under-reporting the journal. The 30% dead, 50 GB-live/35 GB-live, each-removal-type and PostgreSQL `ACCESS EXCLUSIVE` arms remain [E]. + +## 1. Measured basis + +| Quantity | Tiny fixture (T12.3a) [M] | Capacity fixture (T-G61), 10 GB [M] | 50 GB arm (T-G66) [M] | +|---|---|---|---| +| Store size | 20 live roots, 20 dead packs | **9.987 GB pack bytes** (76,870 packs; 69,399 live 131,071-byte whole-file objects, 6,770 dead); 10.035 GB physical | **49.999 GB pack bytes** (384,850 packs; 347,490 live 131,071-byte whole-file objects, 33,850 dead); **50.235 GB physical** at the `none` seal, 50.863 GB at the `rebuild` seal | +| Profile | `Disposable` (WAL/OFF) | `Disposable` (WAL/OFF); one `Durable` (WAL/FULL) control | `Disposable` (WAL/OFF); no `Durable` arm (owner profile disabled) | +| `none` wall | n/a | 148.519 / 149.704 / **149.318 s** median; range 148.519–149.704 | **1,633.585 s** (one run) | +| `rebuild` wall | 26.738 / 28.201 / 28.681 ms; median 28.201 ms | 246.371 / 255.548 / 243.306 s; median **246.371 s**; range 243.306–255.548 | **5,131.835 s** (one run) | +| `Durable` rebuild | n/a | **284.078 s** (one run) | not run (owner profile) | +| `deadBytesBound` | 1,100 B | **887,601,160 B** at 8.88% (`none`); **1,011,104,896 B** at 10.00% (`rebuild`) | **4,438,005,800 B** at 8.876% (`none`); **5,062,342,096 B** at 10.000% (`rebuild`) | +| `physicalShrink` | 20,480 B | **1,016.5–1,019.0 MB** (mean 1.017 GB); `none` 0 | **5,099,651,072 B** (5.100 GB); `none` 0 | +| `peakTempBytes` (receipt) | **758,240 B** (T-G72 bound; old formula 258,048 B) | **29,481,103,648 B** (29.481 GB bound; old formula 9.144 GB) | **147,608,626,376 B** (147.609 GB bound; old formula 45.784 GB) | +| Peak on disk, sampled | n/a | not sampled | **143.1 GB above the pre-rebuild baseline** (§3) | +| New file | n/a | 9,139,773,440 B (9.140 GB) | 45,763,203,072 B (45.763 GB) | +| Fixture build | n/a | 150 s | **761.2 s** (65.7 MB/s over 49.999 GB) | +| Host | linux-x86_64, rustc 1.85.1 | 88 vCPU, 135 GB RAM, ext4, 4.0 TB free | 88 vCPU, 125 GB visible RAM, ext4 (`/dev/sda2`), 4.0 TB free; same rustc 1.85.1 | + +Phase durations, 50 GB `rebuild` (one run, WAL/OFF): **mark 784.251 s**, **build 4,254.371 s**, **verify 85.557 s**, **install 7.311 s** (sum 5,131.490 s of the 5,131.835 s wall). `none`: mark 1,633.579 s and nothing else. +Sources: [`sqlite_compact_capacity_measure.json`](../../../../engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json) (10 GB) and [`sqlite_compact_capacity_measure_50gb.json`](../../../../engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json) (50 GB). + +Caches [M]: the 10 GB Store fitted in the host's page cache and no cache drop ran between its runs; the 50 GB arm did not fit. Its `none` mark ran immediately after the fixture build (50 GB of fresh writes still in the page cache) at **35.9 s per live GB**; the `rebuild` mark, after that read pass, ran at **17.2 s per live GB** — the 16.4 s/GB of the 10 GB arm. The mark is still single-threaded payload/identity hashing (~62 MB/s effective); the 2.1× spread is cache state, not the object count. No cache drop ran between the two 50 GB arms either. + +Per-GB rates measured on this host (GB = 10^9 B): + +| Step | 10 GB [M] | 50 GB [M] | Basis | +|---|---|---|---| +| Mark (`none`) | 14.9 s per stored GB / 16.4 per live GB | **35.9 s per live GB** (32.7 per stored GB) | 1,633.579 s over 45.561 GB live; cold page cache, right after the fixture build | +| Mark (`rebuild`) | 16.4 s per live GB | **17.2 s per live GB** | 784.251 s over 45.561 GB live; warm after the `none` read pass | +| Build (`VACUUM INTO` + prune + `VACUUM`) | 10.3 s per new GB | **93.0 s per new GB** | 4,254.371 s over 45.763 GB new; the phase moves ~141 GB (staged copy 50.85 GB, 44.6 GB rollback journal, target rewrite 45.76 GB) at ~33 MB/s | +| Verify | 0.034 s per new GB (0.31 s total) | **1.87 s per new GB** (85.557 s) | at 50 GB it reads the payload back; the 10 GB read was cache-only | +| Install | ~1.9 s fixed | **7.311 s** | rename, fsync, settle | + +The mark dominates at 10 GB; at 50 GB the build phase dominates by 5.4× — the opposite. Do not extrapolate either phase linearly from the 10 GB arm: the 10 GB run was cache-optimistic, and the 50 GB build rate depends on free page cache and device write throughput rather than on CPU. Use the 50 GB [M] rates above for a 50 GB Store and re-measure with §7's harness on the target host. + +## 2. The 50 GB / 9% Store and the rule + +`rebuild iff total > 0 AND dead * 10 >= total` (§3); `dead` = bytes in packs with no live object after the metadata walk (§6.4). GB = 10^9 B. + +| Dead share | Old | Dead | Live | `dead*10 >= total` | Path | Measured? | +|---|---|---|---|---|---|---| +| 9% | 50.0 GB | 4.5 GB | 45.5 GB | 45 >= 50: **no** | `none` | **50 GB [M]**: 49.999 GB pack bytes, 8.876% dead → `none`, 0 B freed | +| 10% | 50.0 GB | 5.0 GB | 45.0 GB | 50 >= 50: **yes** | `rebuild` | **50 GB [M]**: 50.623 GB pack bytes, 10.000% dead → `rebuild`, 5.100 GB freed | +| 30% | 50.0 GB | 15.0 GB | 35.0 GB | 150 >= 50: **yes** | `rebuild` | [E] only; §13's 50 GB total/35 GB live and 50 GB-live arms remain deferred | + +At **9% the measured path is `none`**: mark only, no file, generation unchanged (7 → 7), **0 bytes freed**; the report still gives the dead bound (4.438 GB measured at 50 GB; 0.888 GB at 10 GB). +The measured 50 GB rebuild copied a 45.763 GB new file to free 5.100 GB — **8.97 B copied per B freed** [M], against 9.04 at 10 GB [M] and the frozen 10% floor's ~9.0 [E]. At 30% it is ~2.3 [E]. + +## 3. Temp-space model + +**SQLite (measured).** 50 GB arm: old physical file 50.863 GB; new `.compact` 45.763 GB; `.compact-visited` 20,365,312 B. The rebuild's temp artifacts are the *whole* staged copy, a rollback journal and SQLite's rewrite temp database — not the new file plus visited. `VACUUM INTO` first writes the whole sealed Store to `.compact` (the old page count, 50.863 GB); the prune's `DELETE ... WHERE NOT IN visited` grows `.compact-journal` to **44.6 GB** (44,630,503,928 B observed); the closing `VACUUM` then writes the vacuumed Store to SQLite's unlinked temp database (≈45.763 GB, in the process temp directory) while the staged file is rewritten in place and the rollback journal records the pages it overwrites. A 20 s disk sampler saw the run's real temp peak at **143.1 GB above the pre-rebuild baseline** (sample max 315.9 GB used against 172.8 GB before the rebuild) = staged 50.85 + rewrite temp 45.76 + journal 44.6–45.8 GB; **~2.9×** the 50 GB pack-byte Store [M]. +The old receipt computed `target_bytes + visited_bytes` (`Built::peak_temp_bytes` in `engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs`) *after* the staged file was truncated: 45.784 GB = new + visited, under-reporting the rollback journal and the staged/temp copies (the T-G66 finding). +**T-G72 bounds the whole run instead.** `Built::peak_temp_bytes` is `staged_bytes + journal_bound + target_bytes + visited_bytes`: the staged extent measured right after `VACUUM INTO`, the rollback-journal upper bound (one record per staged page, `page_size + 8` B, plus an 8192 B header allowance), the new file (which also bounds the rewrite's temp database) and the visited scratch. The 50 GB arm is 50,862,854,144 + 50,962,203,848 + 45,763,203,072 + 20,365,312 = **147,608,626,376 B (147.609 GB) = 2.90× the old file**, 4.5 GB (3.2%) above the sampled peak: the bound charges the journal the whole staged page count instead of the pages the transactions actually touch. The total footprint is old + peak = **~198 GB (3.9×)** at 50 GB; the measured sample was old + 143.1 ≈ 194 GB (3.8×) [M]. Upstream issue: [Ephemeral-AI-Lab/layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). +Visited is metadata-scale: 20,365,312 B for 347,490 live objects = **58.6 B per visited object** [M] (10 GB: 57 B). +`physicalShrink` (5.100 GB) exceeds `deadBytesBound` (5.062 GB) by ~0.7%, as at 10 GB (0.6%): shrink is file-level and includes free pages; do not equate the two. + +**PostgreSQL.** Peak = old tables (50 GB, dead included) + scratch schema `lfs_gc_` (LIKE copy of live `pack`/`object_location`) + index/PK + `ANALYZE` + WAL "up to the live bytes" (§8 "Space"); mark temp tables are session-temp and small. +Extrapolated [E; SQLite measured: the T-G72 receipt bound is 2.90× the old file and old + peak ~3.9×, so the SQLite rows below are receipt-bound estimates and every [E] row stays a projection]: + +| Backend | Case | Old | New file / scratch | Visited / index | Journal / WAL | Peak temp | Total footprint | +|---|---|---|---|---|---|---|---| +| SQLite | 9% `none` | 50.0 | 0 | <=0.1 GB | 0 | **<=0.2 GB** | ~50.2 GB | +| SQLite | 10% `rebuild` | 50.0 | ~45.8 new + ~50.9 staged | ~0.02 GB | ~44.6 GB prune / ~45.8 GB rewrite | **~147.6 GB bound** (sampled 143.1) | **~198 GB (3.9× bound; 3.8× sampled)** | +| SQLite | 30% `rebuild` | 50.0 | ~35.2 new + ~50.9 staged | ~0.02–0.1 GB | ~35 GB prune/rewrite [E] | ~137 GB bound [E] (actual ~120 [E]) | **~187 GB [E] (3.7×)** | +| PG | 9% `none` | 50.0 | 0 | 0 | mark only | **~50 GB** | ~50 GB | +| PG | 10% `rebuild` | 50.0 | ~45.0 | ~4–6 GB | <=45 GB | **~99–101 GB, <=146 GB** | same | +| PG | 30% `rebuild` | 50.0 | ~35.0 | ~3–5 GB | <=35 GB | **~88–90 GB, <=125 GB** | same | + +**Headroom [M].** Keep **>=150 GB free** for a 50 GB `rebuild` at ~10% dead (measured 143.1 GB sampled peak, ~2.9× the Store), >=115 GB at <=30% dead [E]; `none` needs a few GB. The T-G61 [E] guidance of 95–110 GB came from the receipt field and is **too low**. +Size `pg_wal` separately: a full `pg_wal` stops the server (§8). + +## 4. Time model + +Measured 50 GB [M] on this host, beside the 10 GB medians. Linearity is **not** assumed: the 50 GB column is measured, not scaled. + +| Dead | Path | Measured old | 10 GB wall (median) [M] | 50 GB wall [M] | s/GB live at 50 GB [M] | Freed at 50 GB | +|---|---|---|---|---|---|---| +| 9% | `none` | 49.999 GB | 149.318 s | **1,633.585 s (~27.2 min)** | 35.9 | 0 | +| 10% | `rebuild` | 50.623 GB | 246.371 s | **5,131.835 s (~85.5 min)** | 112.6 | 5.100 GB | +| 10% | `rebuild`, Durable | - | 284.078 s | not run | - | - | +| 30% | `rebuild` | - | - | **~65–70 min [E]** (mark 17.2 s/GB + build 93 s/GB over ~35.8 GB new) | ~110 [E] | ~15.0 GB | + +The oldest [E] table said 1.8–2.0 min for a 50 GB rebuild; the linear T-G61 [E] said ~20.3 min; the measured 50 GB rebuild is **85.5 min**, 4.2× the linear projection. Two measured causes: (a) the build phase is **9× slower per new GB** than at 10 GB (93.0 against 10.3 s/GB) because the rebuild's ~146 GB working set (old + staged + journal) no longer fits the 125 GB page cache, so the copy is real device I/O; (b) the mark has a 2.1× cold/warm spread (35.9 against 17.2 s/GB live). Plan **~27 min** for a 50 GB `none` and **~85 min** for a 50 GB `rebuild` on a host like this one [M]; re-measure with §7's harness on the target host. +PostgreSQL is not measured by AC-57; its server-side `INSERT ... SELECT` copy, index build and `ANALYZE` extrapolate to **4–10 min**, `ACCESS EXCLUSIVE` only at the swap, **~2 s** [E]. + +## 5. Operation window, mounts and commits + +- **SQLite.** `writer_admission` is held checkpoint→rename, so the exclusive lane is the whole §7 run: **27.2 min for a 50 GB `none`** and **85.5 min for a 50 GB `rebuild`** [M]; the 10 GB values are 2.5 min each [M]. A live mount = `MountsActive`; a mount during compaction = `BusyOp` (AC-07/AC-49); a Commit racing install = `StaleStore` or lands on `G+1`, retry never lost. `none` is mark-only but still holds the lane for its mark. +- **PostgreSQL.** The session advisory key spans M–C; mounts get `BusyOp`, a post-mark commit makes the swap `Conflict` with no writes (§8). `ACCESS EXCLUSIVE` is the swap only (~2 s) [E]; the `G+1` Store is usable during C. Plan 25 minutes quiet for SQLite at 50 GB [M for the phases, E for the quiet-window policy], mostly online for PG [E]. +- **Monitoring.** `store.status` gives `op {id,name,phase,started,uncertain}`, `state` busy/unresolved and phases — SQLite `mark/build/verify/install`, PG `mark/build/swap/cleanup` (§11). The capacity runs' T10.3 phase sums are in the fixtures (`none` = all mark; `rebuild` = mark/build/verify/install). + `compact_metrics` has runs, operations, refusals, settles, retries, status, slo, phase durations and last §11 gauges: process-local, no CLI/SDK renderer, no PG metrics. + Don't loop a direct reader: it makes the checkpoint `Busy` (§7; [`gc-runbook.md`](gc-runbook.md) §1). + +## 6. Risks + +- **Disk full (both).** SQLite `SQLITE_FULL`/`ENOSPC`/`EDQUOT`/`EFBIG` → `Capacity`, artifact deleted, old Store unchanged apart from checkpoint (§7 step 8; AC-55). The measured 50 GB run needed **143.1 GB** of sampled temp peak (~2.9× the 50 GB Store); the T-G72 receipt bound is **147.609 GB** (2.90× the old file), so keep **>=155 GB free** and re-check before a 30% run (>=140 GB there [E]). +- **Peak temp.** `peakTempBytes` now bounds the run's temp artifacts (T-G72): pre-truncate staged copy + new file + rollback-journal bound + visited. The 50 GB receipt is 147.609 GB (2.90× the old file; sampled 143.1 GB); total footprint = old + peak ≈ 198 GB (3.9×). The journal term is conservative: the bound charges one record per staged page while the run journaled 44.6 GB [M]. +- **WAL.** SQLite needs `wal_checkpoint(TRUNCATE) busy == 0` and never unlinks `-wal`/`-shm` (a live reader is `Busy`); PG WAL ≤ live bytes + standby lag. +- **PG standby.** `synchronous_commit ≥ remote_write` with `synchronous_standby_names` needs a streaming sync/quorum standby; missing → `Busy` no writes; one dying after the check blocks COMMIT and `SET LOCAL` cannot bound it (§8; AC-60 deferred). +- **Installed-but-uncertain / races.** Rename/fsync failure or lost COMMIT keeps the new Store at `G+1`, settles `unknown`; never delete the artifact; PG: prove the backend ended (`pid`, `backend_start`, `usename`), then read the incarnation. + `MountsActive`/`BusyOp`/`StaleStore` are expected: quiesce, retry, never kill an owner. The 20,480 B vs 1,100 B ratio and 28.2 ms are tiny-fixture effects; the 10 GB and 50 GB numbers above are the capacity evidence. + +## 7. Runbook thresholds (50 GB Store) + +1. **Eligibility:** only `dead*10 >= total` (>=10% dead, §3); below it `none`, no file, 0 freed — but still a full mark (**27 min at 50 GB** [M]). Record the dead bound and re-check later. +2. **Economics/space [M]:** at 10% dead the measured rebuild copies **8.97 B per B freed** [M] (10 GB: 9.04 [M]) and needs **~148 GB free** (147.609 GB receipt bound, 2.90×) [M]; prefer **>=20–30% dead** (30% ~2.3 B per B [E], ~137 GB bound [E]). +3. **Window/quiesce [M]:** SQLite **~27 min** quiet for a 50 GB `none` and **~85 min** for a 50 GB `rebuild`; the 10 GB values are 2.5 min each. No full doctor; expect `BusyOp`. A `Busy` owner: wait, do not terminate (runbook §2). +4. **Recovery gate:** if `store.status.state` is `unresolved`, rerun `store.compact` to adopt the op; never hand-delete artifacts or `gc_intent`/`gc_dead` (runbook §2–§3). +5. **Backup/alerts/failures:** `backup.create` before the first run ([`gc-rollout.md`](gc-rollout.md) §3; §9); alert on `Capacity`, `unresolved`, `retries`, nonzero `residue`, PG WAL/standby lag. `Capacity` → free space and rerun; uncertain → keep the new Store, settle `unknown`. +6. **Re-measure per host.** `CAPACITY_MEASURE=1 UPDATE_CAPACITY=1 cargo test -p layerfs-persistence --release --test sqlite_compact_capacity -- --ignored --nocapture` with a TMPDIR of **>=3× the target** (the receipt bound is 2.90×; the measured temp peak was 143.1 GB). The 50 GB arm used `CAPACITY_GROUPS=3510 CAPACITY_DEAD_OBJECTS=33850 CAPACITY_RUNS=1 CAPACITY_KEEP=1 CAPACITY_OUT=<...>_50gb.json`; the mark is CPU-bound and the copy I/O-bound, so a host with faster SHA-256 and more page cache can be several times quicker. + +## Sources + +- [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md): §3 threshold, §6.4 dead set, §7 SQLite rebuild/cost/Capacity, §8 PG phases/space/standby, §11 report and `peakTempBytes`, §13 deferred 50 GB arms and estimates. +- [`gc-t12.3a-sqlite-matrix-report.md`](gc-t12.3a-sqlite-matrix-report.md) (AC-57) and `sqlite_compact_measure.json` (the tiny [M] numbers; its `peakTempBytes` is the T-G72 bound, 758,240 B); `sqlite_compact_capacity_measure.json` and `sqlite_compact_capacity.rs#capacity_measure` (the 10 GB [M] numbers); `sqlite_compact_capacity_measure_50gb.json` (the 50 GB [M] numbers, one run per arm). The two capacity JSONs still record the T-G66 old-formula `peakTempBytes` (45.784 GB at 50 GB); the T-G72 bounds in this page are arithmetic over their measured page counts and sizes, not a re-measurement. +- [`gc-t9-postgresql-rebuild.md`](gc-t9-postgresql-rebuild.md) (M/A/B/C, scratch schema, WAL/standby gaps) and [`gc-implementation-report.md`](gc-implementation-report.md) (status/metrics projection). +- [`gc-runbook.md`](gc-runbook.md) and [`gc-rollout.md`](gc-rollout.md) (safe cleanup, rehearsal). diff --git a/core/docs/issues/project-41/gc-implementation-report.md b/core/docs/issues/project-41/gc-implementation-report.md new file mode 100644 index 0000000000..82456cefb3 --- /dev/null +++ b/core/docs/issues/project-41/gc-implementation-report.md @@ -0,0 +1,485 @@ +# GC implementation report: soft removal and Store compaction + +> Final report for `layerfs-project#41`. The frozen +> [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) (2.0.0) wins; `§N` and `AC-nn` +> cite it. This page certifies main layer `32ae51ca` (audited by T-G75 on +> 2026-11-15) in §8; it adds no contract and leaves acceptance sign-off to +> #52. + +## 1. Scope and what shipped +Four ids-only, admin-profile calls on SQLite and PostgreSQL: +`branch.remove`, `project.delete`, `project.remove_file` and `store.compact`. +The first three change visibility and retention only; `store.compact` +reclaims bytes. +**Schema and removals.** The additive open step adds `branch_retired`, +`layer_stack_retired`, `path_removed`, `project_epoch`, `gc_intent`, `gc_op`, +`gc_dead` and `kept_node`, plus `history_meta.next_history_serial` (never +lowered), `commit.history_serial`, `layer.history_serial` and +`store_policy.trace_salt`; it backfills serials deterministically, creates one +zero epoch per stack and raises the next serial above the maximum. `branch` and +`layer_stack` stay append-only; 1.0 Stores convert offline; there is no +down-migration (§4.1). `branch.remove` writes `branch_retired`; +`project.delete` writes `layer_stack_retired` plus a retry sidecar and returns +`home_entry_removed: false` on a home-removal failure; `project.remove_file` +writes `path_removed` at frontier `next_history_serial - 1`. Visibility is +filtered statements plus an allow-list guard, not views, and the three removal +markers are never rewritten (§4.2-§4.4). + +**Admission and mark.** `gc_intent` is one Store-scoped row per operation +(kind, scope, state, owner pid/start/boot, phase, generation, artifact, high +water). SQLite uses two `BEGIN IMMEDIATE` transactions and reports +`MountsActive`, `Busy` or `Unresolved`; PostgreSQL holds `(kind, id)` advisory +keys at Store, project and Branch scope, records the owner in `gc_op`, and ends +a dead idle owner with `pg_terminate_backend(pid, backend_start)`. The epoch +fence re-checks `project_epoch.epoch` before commit (same transaction on +SQLite, a final `FOR SHARE` on PostgreSQL); a changed or missing epoch is +`Conflict` with no Commit (§5). `HistoryProvider::mark_gc` reads the tuple +(`catalog_incarnation`, `next_history_serial`, `next_pack_id`, `next_ordinal`, +`store_totals`, `MAX(stamp)`, history rows), runs the §6.1 history closure and +the §6.2 object closure (`FinalizedObject::admit` plus +`ChainBases::base_of`, one visit level, delta bases), and writes `kept_node`, +the object/pack set, `deadBytesBound`/`deadBytesRefined`, `gc_dead` and the +§6.4 `live_heads_digest` (absent head is 33 zero bytes). `keptWaste` covers +retained-pack waste, delta bases and all `inode_value_block` rows. + +**SQLite rebuild.** `SqliteHandles::compact` holds `writer_admission` from +checkpoint through rename, runs `wal_checkpoint(TRUNCATE)` (busy is `Busy`; the +WAL is never unlinked), seals `(size, mtime, db-header change counter)`, +attaches the old file `immutable=1`, builds `.compact-visited`, runs +`VACUUM INTO .compact`, and in one transaction keeps retained rows, deletes the +complement, sets `G + 1`, recomputes `store_totals` and clears +`binding_hint_block`. Verification checks schema, foreign keys, counts, both +digests, allocator and totals; install rechecks sidecars and `-mj*`, applies +`chmod 0600` and `sync_all`, renames and fsyncs; settle removes the artifact +set and op record. Failpoints `AfterSeal`, `AfterBuild`, `AfterWalSwitch`, +`AfterVerify`, `BeforeInstall`, `AfterInstall` and `CapacityAtBuild` cover the +crash states; the artifact set is +`.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; a stray name is +`Exists`, a `(dev, ino)` change is `StaleStore`, and a sealed-tuple change at +install is `Conflict`. The path is `none` below 10 percent, `rebuild` at or +above it; an empty Store is `none` with `zeroReason: empty-store`; a started +rebuild completes (§7). + +**PostgreSQL rebuild.** `PostgresHandles::compact` runs four phases. M writes +`kept_node`, `gc_op` (phase `mark`, scratch `lfs_gc_`), `gc_dead` and the +dead totals. A creates the scratch schema with LIKE copies of `pack` and +`object_location`, copies retained rows in idempotent batches of 512 packs, +then adds the prefix index, foreign key and `ANALYZE`. B is one short +transaction: `ACCESS EXCLUSIVE NOWAIT` (`55P03` is `Busy`), re-read the mark +tuple and digest (`Conflict` on change), copy owner and ACL, swap the tables, +recreate `pack_immutable`, settle the totals, set `G + 1`, re-check +`synchronous_commit` and the standby predicate, drop the scratch, `COMMIT`. C +deletes only `gc_dead` members, clears `binding_hint_block` and the acquisition +rows, reduces `store_totals` once against the row sums, clears `gc_dead` and +`gc_op`, and releases the intent. The next call adopts a dead idle owner, is +`Busy` on a live owner, and resumes Phase C at `G + 1`; a lost `COMMIT` reply +is resolved by proving the backend ended, then reading the incarnation. +PostgreSQL reports `backendReusable = 0` and `postgresql-in-place`, and has no +`inode` table (§8, §9). + +**Report, status, metrics.** `CompactReport`/`CompactStatus` in +`engine/crates/layerfs-persistence/src/report.rs` and the single serializer in +`control/crates/layerfs-store/src/report.rs` render every §11 field in +snake_case (`dead_bytes_bound`, `kept_waste`, `residue`, `estimate`, and one +`zero_reason` per zero field from +`empty-store|none-path|no-history|no-waste|postgresql-in-place`). `store.status` +adds `op {id, name, phase, started, uncertain}`, `state` (`busy` or +`unresolved`; `uncertain` is true for every remaining record) and the backend +`phases` (SQLite `mark/build/verify/install`; PostgreSQL +`mark/build/swap/cleanup`). `compact_metrics.rs` publishes counters (`runs`, +`operations{outcome}`, `refusals{code}`, `settles{kind}`, `retries{reason}`, +`status{state}`, `slo{objective,result}`), phase durations and last-run §11 +gauges, with labels only from closed enums. + +**API, MCP, CLI, SDK.** The call table is 76 entries (38 work / 38 admin); the +four calls are admin-only and refused under a folder scope with `out of +scope: `. `project.remove` is retired: old clients get `InvalidInput`, +MCP reports an unknown tool, and the CLI exits 2; `project.init` on a held name +is `NameTaken`. MCP names are `branch_remove`, `project_delete`, +`project_remove_file` and `store_compact`; the CLI has `layerfs branch remove`, +`project delete`, `project remove-file` and `store compact`; JS and Python +expose the same four methods. The call log redacts path-bearing arguments +(``) and hashes keys with `trace_salt`. + +| Expected-base CAS upstream | `CommitRequest` carries `expected_head`, `expected_base` and `expected_root`; a moved Branch is `HeadMoved` and nothing is written, so a stale daemon commit cannot clobber a removal | +| Whiteouts | `remove_file` hides at the frontier; merge, publish and mount materialize the whiteout; a merge that cannot is `Removed` with full path elements | +| `gc_intent` / `gc_op` / `gc_dead` | intent is admission and liveness; `gc_op` is the one compaction row with phase, scratch and owner; `gc_dead` is persisted membership, not a predicate | +| Generation | `catalog_incarnation` is `G`; a rebuild installs `G + 1`; leftovers, fences and resume judge by the recorded generation | +| Reclaim unit | the pack; a rebuild copies retained rows into a fresh Store; no in-place pack delete; `rebuild iff total > 0 AND dead * 10 >= total`; a started rebuild completes | +| Visibility-only `remove_file` | 2.0 hides the path and keeps objects reached only through it; a path-aware walk is a follow-up | +| Thresholds | `deadBytesBound` from the metadata walk, `deadBytesRefined` from the payload sweep; PostgreSQL always reports them equal | + +## 3. Test evidence + +T-G75 re-ran the suites from main `32ae51ca` on 2026-11-15 in a Sandbox +Workspace (Branch `task-g75`); the counts below are those runs unless the row +names an earlier run. Earlier lanes were recorded on main `32df79da` or the +Branch the row names. The final lanes are: + +| Suite | Result | Recorded by | +|---|---|---| +| `layerfs-persistence` (full lane) | 271 passed, 0 failed, 3 ignored | T-G75 (T-G24: 264/0/2) | +| `layerfs-store-service` | 214 passed, 0 failed, 15 ignored | T-G75 (T-G24: same) | +| `layerfs-store` | 59 passed, 0 failed, 0 ignored | T-G75 (T-G24: same) | +| `postgresql_compact` | 20 passed (real `pg_terminate_backend` reply-loss arms and the restricted-role privilege arms) | T-G38/T-G54 | +| `postgresql_status` | 3 passed (persistence) + 1 passed (control) | T10.5/T-G53 | +| `postgresql_compact_fixtures` | 5 passed | T12.3b | +| `sqlite_compact` | 20 passed | T-G52; re-run T-G75 | +| `sqlite_compact_crash` | 23 passed (13 in-process + 9 real-process SIGKILL + the re-executed child placeholder) | T-G28/T-G60, re-run T-G52 and T-G75 | +| `sqlite_compact_fixtures` | 1 passed | T-G52; re-run T-G75 | +| `sqlite_compact_matrix` | 1 runner, 8 executed arms | T-G52; re-run T-G75 | +| `sqlite_compact_capacity` | 1 passed, 1 ignored (the opt-in 50 GB arm is not run by default) | re-run T-G75 | +| PostgreSQL doctor (physical) | 16 passed | T-G55 | + +**SQLite lane (T12.3-era baseline; the final counts are the table above).** + +| Binary | Tests | Pins | +|---|---|---| +| `sqlite_compact` | 20 | `none`/`rebuild`, 9/10/30 percent, empty Store, capacity abort, layouts, stray name, report fields, leftover op | +| `sqlite_compact_crash` | 23 | in-process failpoints at seal/build/WAL switch/verify/install/settle and the artifact set, plus 9 real-process SIGKILL arms (mark: BeforeSeal/AfterSeal; copy: BeforeVacuum; verify: DuringVerify; rename: DuringRename; build: AfterBuild; install: BeforeInstall/AfterInstall; settle: DuringSettle) and the re-executed child placeholder | +| `sqlite_compact_fixtures` | 1 | golden receipt with every §11 field, digest vector, three outcomes | +| `sqlite_compact_matrix` | 1 runner, 8 arms | checkpoint busy, sidecar at install, status phases, error codes, seal, random shapes, measurement, digest row after collection | +| `compact_metrics`, `admission`, `path_removal`, `branch_removal`, `project_delete` | 4, 5, 3, 3, 5 | metrics, admission races, frontier, markers | +| `epoch_fence`, `gc_backfill`, `kept_objects`, `kept_node`, layouts, upgrade | 7, 4, 5, 7, 7+9, 2+2 | epoch rollback, backfill, retention, layouts | +| `gc_trace`, `gc_calls`, `calls_scope` (API) | 5, 8, 4 | redaction, four-call shapes, scope refusal | +| `compact_report_status`, `project_delete` (control) | 3, 15+6+2 | wire shape, status, sidecar and frames | +| `branch_remove`, `admission`, `epoch_fence`, `project_delete`, `kernel_guard` (store-service) | 3, 1, 1, 1, 7 | service removals and guard | + +**PostgreSQL lane (T12.3b baseline; the final counts are the table above).** +PostgreSQL 17.12, `cargo test -p layerfs-persistence +--features postgresql`, `--test-threads=1`: 15 test binaries, 73 tests passed +at T12.3b. The run covers `postgresql_compact` (8 at that layer; 20 on +`32df79da`), `postgresql_compact_fixtures` (5, +three consecutive runs), `postgresql_position_columns` (4), +`postgresql_open_check` (3), `postgresql_diagnostic` (5), +`postgresql_acquisition` (4), `postgresql_acquisition_compaction` (4), +`postgresql_identical_ids` (3), `layerstack_removal` (2), +`postgresql_commit_round_trips` (1) and the SQLite controls. The new fixture +proves the §6.4 digest is byte-identical to the SQLite golden and the rebuild +report and markers hold on PG. + +**Harness and CI.** `tools/ci/test_store_ext4.sh` requires +`LAYERFS_TEST_TMPDIR` on ext4, xfs or btrfs (apfs or hfs on macOS), refuses +tmpfs, overlay, FUSE and network mounts before cargo, and points `TMPDIR`, +`CARGO_TARGET_DIR` and `CARGO_HOME` there. `--quick` runs a cross-suite +subset; `--full` runs every Store suite and adds `--features postgresql` when +`LAYERFS_TEST_POSTGRES_URL` is set. `.github/workflows/store-ext4.yml` is the +CI lane: `quick` on push and pull request, `full-postgresql` (a `postgres:17` +service) on schedule and manual dispatch; it is report-only. + +## 4. Acceptance matrix status + +The matrix has 60 cases (47 both, 6 SQLite-only, 7 PostgreSQL-only). Only +#52 records acceptance; these notes record evidence and tier. + +| Runner | Executed | Covered/pass | Partial | Deferred | Gaps | +|---|---|---|---|---|---| +| T12.3a SQLite half (pre-T12.3c) | 5 | 24 | 16 | 7 (PG-only) | 8 | +| T12.3b PostgreSQL half | 0 | 11 | 11 | 8 | 0 | +| T12.3c SQLite gap closure | 8 | 29 | 16 | 7 (PG-only) | 0 | +| T-G28 real-process SIGKILL (AC-01) | 8 | 28 | 17 | 7 (PG-only) | 0 | +| T-G38/T-G54 PG reply loss and privileges (AC-44) | 8 | 29 | 17 | 6 (PG-only) | 0 | +| T-G60 remaining SIGKILL points (AC-01) | 8 | 29 | 17 | 6 (PG-only) | 0 | +| T-G65 real-process `store.compact` kill (AC-01) | 8 | 30 | 16 | 6 (PG-only) | 0 | + +T12.3c closed the eight SQLite gaps: AC-06 (post-compaction Save of a +collected digest), AC-11 (non-FF merge whiteouts and publish at the +frontier), AC-12 (removed path in a merge), AC-13 (unmerged Branch marker +inheritance), AC-14 (layer-headed mount whiteout), AC-22 (concurrent +merge/removal property), AC-34 (bind failure and failed intent delete) and +AC-36 (worker lost reply); its report records "SQLite gaps: none". The +PostgreSQL half's partial and deferred rows are AC-02, AC-29, AC-30, AC-31, +AC-32, AC-34, AC-35, AC-36, AC-37, AC-38, AC-41, AC-42 and AC-60. T-G28 +added the real-process SIGKILL arms of `sqlite_compact_crash.rs`; AC-01 moved +from `covered` to `partial` because the kill targeted a test child, not the +production process. T-G60 extended the real arms to the mark (`BeforeSeal`/ +`AfterSeal`), copy (`BeforeVacuum`), verify and rename points. T-G65 then +started the built `layerfs store compact` command, killed that process at the +mark, build, verify and install points, and asserted `G`/`G + 1`, the +artifact set, the `store.status` projection and the settling run, so AC-01 is +`covered` again (the long-lived `layerfs worker` is still not the killed +process). T-G38's real `pg_terminate_backend` kills and T-G54's +restricted-role fixtures moved AC-44 from `deferred` to `covered`, so the +final dispositions are 8 executed, 30 covered, 16 partial and 6 deferred. +T-G75 re-checked all 60 matrix rows against `32ae51ca` and kept those +dispositions (AC-01 and AC-44 `covered`, AC-57 `partial`). + +## 5. Known gaps and limitations + +These are the gaps that remain on `32ae51ca`; tasks that closed an earlier gap +are recorded in §7 instead. + +- **Clippy clean (fixed by G58/G63/G71).** The gap is closed: G58/G63 fixed + the former 11 pre-existing warnings and G71 fixed the last two; T-G75 + re-ran + `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` + and it exits 0 with zero warnings on the lib and test targets. +- **Branch-delete CLI deployed (fixed 2026-11-15).** The gap is closed: the + 2.0.0 build `4e65ebbae` is live with host CLI `1dc15838…`, bundle `layerfs` + `c4737445…`, `layerfs-daemon` `b8dd0766…` and + `layerfs-sandbox-helper` `3d033ec0…`; `layerfs help branch` ends with + `branch remove `, and a probe against a missing name + returns `NoSuchBranch`. Operators use `layerfs branch remove` directly; MCP + `branch_remove` and the SDK method stay equivalent. The cutover opened the + Store with the one-way GC schema migration: the sqlite file grew + 1,111,203,840 -> 1,111,310,336 B (live hash `d5d65e3b…`) on first open. +- **PG `physicalShrink` is a keyword proxy.** PostgreSQL computes + `physical_shrink` from the `pg_total_relation_size` delta of `pack` and + `object_location` around the swap (an in-place rewrite), not from file-level + bytes; `backendReusable = 0` with `postgresql-in-place` names that absence. + SQLite keeps the file-level measure. +- **Real `kill -9` of the production compaction process (T-G28/T-G60/ + T-G65; AC-01 `covered`).** `sqlite_compact_crash.rs::real_process` spawns a + separate process on the fixture Store and the parent sends `SIGKILL` while + it is stopped at the failpoint; T-G60 extended those arms to the mark + (`BeforeSeal`/`AfterSeal`; the `BeforeSeal` kill keeps the writer + session's WAL sidecars, which the next open replays), copy + (`BeforeVacuum`, immediately before `VACUUM INTO` because SQLite runs that + copy as one uninterruptible call), verify (`DuringVerify`) and rename + (`DuringRename`). Those arms kill a re-executed test child, so T-G65 adds + the production process: + `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` starts the + built `layerfs store compact` command, the engine's gate stops it at the + named failpoint, and the test `SIGKILL`s it there, then reopens through the + CLI and asserts `G`/`G + 1`, the artifact set and sidecars, the + `store.status` phase/`uncertain`/`unresolved`, the settling next run, the + retained layer and `doctor`. No daemon serves Store calls (#352 F-020), so + this is the process that performs the native compaction; the long-lived + `layerfs worker` that serves a machine's Store is still not the killed + process. +- **Capacity model: 10 GB and 50 GB measured (T-G61, T-G66).** + [`gc-capacity-model.md`](gc-capacity-model.md) rests on a real 9.987 GB + Store measurement (`sqlite_compact_capacity.rs#capacity_measure`): `none` at + 8.88% dead (3 runs, median 149.318 s) and `rebuild` at 10.00% dead (3 runs, + median 246.371 s; mark 149.089 s, build 94.317 s, verify 0.312 s, install + 1.916 s; dead 1.011 GB, shrink 1.017 GB, peak temp 9.144 GB, 1.90x total + footprint), plus one WAL/FULL rebuild (284.078 s). T-G66 then measured the + **50 GB** arm (one run per arm, `sqlite_compact_capacity_measure_50gb.json`): + 49.999 GB pack bytes / 50.863 GB physical, `none` at 8.876% dead + (1,633.585 s) and `rebuild` at 10.000% dead (5,131.835 s; mark 784.251 s, + build 4,254.371 s, verify 85.557 s, install 7.311 s; dead 5.062 GB, shrink + 5.100 GB, old-formula receipt peak temp 45.784 GB). The 50 GB numbers are + **not** linear from 10 GB: the build phase is 93.0 s/GB new instead of 10.3, + so the rebuild is 85.5 min instead of the [E] ~20 min, and the sampled real + peak is 143.1 GB (~2.9x the Store). T-G72 replaced the old `new + visited` + receipt with a bound over the staged copy, the new file, the rollback + journal and the visited scratch: **147.609 GB (2.90x the old file)** for the + same 50 GB `rebuild`, 3.2% above the sampled peak, tracked as + [layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + The 30% dead / 50 GB-live arms and the `ACCESS EXCLUSIVE` duration remain + [E]. +- **PostgreSQL `--test-threads`.** T-G14 gave each `postgresql_compact` test + its own catalog binding (`postgresql-compact-`), so its 20 tests pass + with the default test threads (three consecutive runs); the Store-scoped + advisory key is no longer shared. Other PG suites that still share one + schema or session state keep `--test-threads=1` until they get the same + per-test binding. +- **`.git/index` merge rule.** `/.lww` lists `/.git/index` after + `%include .gitignore`, so merging a Branch forked from an older layer keeps + the newer index whole instead of conflicting on the binary file. Only this + generated file is added; no source file is listed. +- **Branch cleanup is soft and bounded (updated 2026-11-15).** `branch.remove` + is live, but removal is a marker write: the name stays taken (a re-fork is + `NameTaken` with holder `state: "retired"`, reads are `NoSuchBranch`) until + `store.compact` collects the rows, and `main` is `ReservedName`. A Branch + with a live mount is `MountsActive`, and a Branch pinned by a retained Layer + or `commit.branch_id` also keeps `NameTaken`; a rebuild needs an idle Store. + The 2026-11-15 cleanup retired 145 Branches (20 empty + 125 superseded) and + deliberately left 162 (unknown or non-empty); T-G75 re-probed the live CLI + on `32ae51ca` (a missing name is `NoSuchBranch`) and counted 174 live + Branches, the 162 that cleanup left plus later gap-fix task Branches. The + remaining work is data judgment, not a missing capability. +- **`project.remove` retired.** Old clients get `InvalidInput`, MCP reports an + unknown tool, and the CLI exits 2; there is no compatibility alias. +- `remove_file` reclaims no bytes in 2.0; a path-aware walk is a follow-up. + PostgreSQL ACL replay is best-effort, and `markBytes` is a #52 metric, not a + report field. Compaction metrics are process-local with no SDK or CLI + renderer, and PostgreSQL compactions have no metrics yet. +- AC-60's real standby arms and the reserved-pack race of AC-42 are deferred; + AC-28/41/43 have T12.3b evidence and AC-35's advisory-key refcount does not. + +## 6. How to run + +``` +# SQLite compaction lane, on a filesystem that carries SQLite locks +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --full +cargo test -p layerfs-persistence --test sqlite_compact +cargo test -p layerfs-persistence --test sqlite_compact_crash +cargo test -p layerfs-persistence --test sqlite_compact_fixtures +cargo test -p layerfs-persistence --test sqlite_compact_matrix +UPDATE_GOLDEN=1 cargo test -p layerfs-persistence --test sqlite_compact_fixtures + +# PostgreSQL, one thread +LAYERFS_TEST_POSTGRES_URL="host=127.0.0.1 port=55432 user=postgres dbname=postgres sslmode=disable" \ + cargo test -p layerfs-persistence --features postgresql --test postgresql_compact -- --test-threads=1 +``` + +The ext4/xfs/btrfs requirement is the kernel guard +(`engine/crates/layerfs-store-service/src/kernel_guard.rs`): tmpfs, overlay, +FUSE and network mounts are refused `Unsupported` because they cannot carry +the WAL locks and the directory flock. Inside a Docker Sandbox the run +directory (`$HOME`, `/layerfs/data/machine/native/run/`) is ext4; +`/tmp` is tmpfs there. The PG value is a libpq keyword string; +`postgresql_diagnostic` needs the URI form. + +## 7. Commit and branch index + +Main layers are publications; heads are the saved Branch tips. `T12.3b` was in +flight at the baseline and is now published on main (`32223b46`, tidy `32927884`). + +| Epic | Branch | Head commit | Main layer | Result | +|---|---|---|---|---| +| T1 schema and position | task-t1.2d-merged, task-t1.7-merged3 | 12eee0da5b, 12fb3019bd | 32b602714d, 32860cbe46 | markers, intents, serials, epoch, trace salt, backfill | +| T2 history contract | task-t2.4-ctors-test-merged3 | 12ebeb2dfb | 32c5500017 | typed `NameTaken`/`Removed`, catalog stubs | +| T3 `branch.remove` | task-t3-final5 | 12dabb4673 | 3256c829a6 | `branch_retired`, `ReservedName`, name lifetime | +| T4 `project.delete` | task-t4.6-merged | 12e8a214b4 | 32aca049c3 | `layer_stack_retired`, sidecar, retry | +| T5 `project.remove_file` | task-t5.8 | 12f58625e0 | 3299d92d08 | `path_removed`, frontier, whiteouts, grammar | +| T6 admission and epoch | task-t6.5-merged4 | 12ebc5d982 | 32e895c4eb | `gc_intent`, codes, epoch fence, PG advisory keys | +| T7 mark and closure | task-t7.2 | 12a41bf33f | 3240699d11 | mark tuple, closures, `kept_node`, digest | +| T8 SQLite rebuild | task-t8.1-merged | 122151d179 | 323465c8b3 | seal, visited, verify, rename install, failpoints | +| T9 PostgreSQL rebuild | task-t9e | 1225694fd7 | 32c9181e3d | phases M/A/B/C, `gc_op` resume, standby | +| T10 report, status, metrics | task-t10.1b, task-t10.3-final | 121262d292, 1236755412 | 32515e1b2a, 3271a46892 | §11 report, status op, `compact_metrics` | +| T11 API, MCP, CLI, SDK | task-t11.3-final | 1271019370 | 3218fea5bc | 76 calls, admin profile, `project.remove` retired | +| T12.1 SQLite fixtures | task-t12.1-merged2 | 12db7c4ea4 | 32444eb25c | golden receipt, §11 fields, digest vector | +| T12.2 SQLite crash matrix | task-t12.2-merged2 | 12433023b7 | 32502588ab | failpoints, artifact set, `StaleStore` | +| T12.3a SQLite matrix | merged3-t12.3a | 1277362ac5 | 327d6a42bd | 5 executed / 24 covered / 16 partial / 8 gaps | +| T12.3b PostgreSQL acceptance | task-t12.3b-final | 1210405949 | 32223b46 | PG-half matrix, 15 binaries / 73 passed (published) | +| T13 spec gaps and runbook | task-t13b, task-g17-publish | 12c0efe9dd, 12dcce8e15 | 32ab564277, 3284fb82 | G1-G6 decisions, operator runbook, testing docs | +| T-G13 default-feature build | task-g13-pub2 | 1299885ee9 | 32971532c5 | PG imports gated behind the feature | +| T-G28 real-process SIGKILL | task-g28-merged | 12106c4dd7 | 32a7d0cf07 | 4 SIGKILL arms; AC-01 `partial` | +| T-G52 SQLite crash re-run | task-g52 | 12c002e409 | 32bf908cf5 | 20/18/1/1, matrix 8 arms | +| T10.5 PG `store.status` | merged-t10.5c | 12f80d29df | 328a16a4c9 | persistence `postgresql_status` 3, control 1 | +| T-G38 PG reply loss | task-g38b-merged | 128162224e | 32dc46a25a | real backend kills and lost `COMMIT` | +| T-G53 PG status control | g53-merged | 12876c4f1f | 322a2c59a1 | control `postgresql_status` +1 | +| T-G54 PG privileges | g54-merged | 12f5197908 | 32a2621dbb | AC-44 `covered`; `postgresql_compact` 20 | +| T-G55 doctor physical | g55-integ | 122880b8b6 | 3285f0d588 | 16 checks | +| T-G57 closeout base | g57-merged | 12babcbaea | 32df79da | T-G57 publication | + +The docs-only review rounds A-M are on `gc-phase-a` ... `gc-phase-m` and the +`gc-schema`/`gc-contract` branches; they changed no product contract. The +post-closeout gap fixes are published as `task-g58-merged2` (G58/G63 clippy), +`task-g63`, `task-g64`, `g61-integ` (capacity harness), `task-g65-merged` and +`task-g65-merged2` (real-process kill), `task-g67` (physical doctor proxy), +`task-g69` (runbook and cleanup), `g70-merged` (report), `task-g71` (final +clippy), `task-g66b` (50 GB measurement), `task-g72` (peak-temp bound) and +`task-g74` (report generator); T-G75 audits those layers. + +## 8. Closeout + +- **Final main layer.** `32ae51ca` (root `d0b744cb`), the layer this audit + certifies; it is the T-G74 publication (commit `128899cd74`). The chain from + the previous closeout is T-G57 `32df79da` -> T12.8 `3255cfb1fa` -> T-G60 + `32284e2d4c` -> T-G59 `327b250e45` -> T-G58 `329abfa4b4` -> T-G63 + `32e51aa657` -> T-G64 `32feb7df87` -> G61 `32977da52f` -> T-G67 + `327892e625` -> T-G69 `32cfd1e67a` -> T-G65 `3299d19429` -> `32957696ef` + -> G70 `32bb6d7557` -> G71 `32f89ff7b8` -> T-G66b `32c363ad8a` -> T-G72 + `327aa16ab0` -> T-G74 `32ae51ca`. This closeout publishes as main's next + layer. +- **Shipped.** Four admin-only, ids-only calls on SQLite and PostgreSQL: + `branch.remove`, `project.delete`, `project.remove_file` and + `store.compact`; the §4.1 additive schema and deterministic backfill; the + `gc_intent`/`gc_op` admission intent, the epoch fence and the §6.1/§6.2 + mark with the §6.4 digest; the SQLite seal/visited/verify/rename rebuild + with the 10% `none`/`rebuild` threshold and named failpoints; the + PostgreSQL Phase M/A/B/C rebuild with `gc_op` resume, ACL and the standby + predicate; the §11 `CompactReport`/`CompactStatus`, `store.status` and + `compact_metrics` (SQLite phases plus `pg_mark`/`pg_build`/`pg_swap`/ + `pg_cleanup`); the 76-call API/MCP/CLI/SDK table with `project.remove` + retired; and the ext4 harness plus report-only CI lane. +- **Final test evidence on `32ae51ca` (T-G75 re-run).** 271 passed / 0 failed + / 3 ignored (`layerfs-persistence`; T-G24 recorded 264/0/2), 214/0/15 + (`layerfs-store-service`) and 59/0/0 (`layerfs-store`), 544 passed / 0 + failed / 18 ignored in all. The five SQLite compaction targets re-ran to + `sqlite_compact` 20, `sqlite_compact_crash` 23, `sqlite_compact_fixtures` + 1, `sqlite_compact_matrix` 1 runner (8 arms) and `sqlite_compact_capacity` + 1 passed + 1 ignored (the opt-in 50 GB arm). The PostgreSQL lanes were not + re-run here (this Sandbox has no server): 20 `postgresql_compact` (T-G54), + 3+1 `postgresql_status` (T10.5/T-G53), 5 `postgresql_compact_fixtures` + (T12.3b) and 16 doctor-physical checks (T-G55) stay as recorded. The full + table is in §3. +- **Clippy clean (G58/G63/G71).** G58/G63 fixed the former 11 pre-existing + warnings and G71 fixed the last two; T-G75 re-ran + `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` + to exit 0 with zero warnings on the lib and test targets. +- **Matrix after T12.3c, T-G28/T-G60, T-G38/T-G54 and T-G65.** 60 cases: 8 + `executed`, 30 `covered`, 16 `partial`, 6 `deferred` (all + PostgreSQL-only), 0 SQLite evidence gaps, 0 `pending`; T-G75 re-checked all + 60 rows on `32ae51ca` (AC-01 and AC-44 `covered`, AC-57 `partial`). +- **Remaining partial (16).** AC-02, 04, 11, 15, 19, 20, 21, 23, 30, 32, + 34, 37, 53, 54, 56, 57. Why: the SQLite half is proven but the PostgreSQL + half is T12.3b's or not named (AC-02/30/32/34), the full form/read-surface + table is not one test (AC-19/53), the named arm is not inducible or does not + exist (AC-11's inapplicable-whiteout branch, AC-34's failed intent delete), + or the E3 scale/three-run arms remain (AC-04, 56, 57). +- **Remaining deferred (6, PostgreSQL-only).** AC-28, 35, 41, 42, 43, 60. + T12.3b executes the PG halves of AC-28/41/43; AC-35 (advisory-key refcount) + and AC-42 (reserved-pack race) have no deterministic pause point, while + AC-60 (a real standby) needs server control the harness does not have. +- **Capacity model.** [`gc-capacity-model.md`](gc-capacity-model.md) records a + real 9.987 GB measurement ([M]: `none` 149.318 s, `rebuild` 246.371 s median, + phase sums and the T-G72 peak bound in + `sqlite_compact_capacity_measure.json`) and the real 49.999 GB measurement + ([M], one run per arm, phase sums and 5.100 GB shrink in + `sqlite_compact_capacity_measure_50gb.json`; 85.5 min rebuild, sampled peak + 143.1 GB). T-G72 replaced the old `new + visited` receipt with a documented + bound over the staged copy, the new file, the rollback journal and the + visited scratch: **147.609 GB (2.90x the old file)** for the 50 GB + `rebuild`, 3.2% above the sampled peak, tracked as + [layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + AC-57 stays `partial` for the section 13 30% dead / 50 GB-live arms and the + PostgreSQL arms. +- **Branch-removal cutover and cleanup (2026-11-15, main `327892e6`).** + `branch.remove` is live end to end. Build identity: version `2.0.0`, + commit `4e65ebbae`; host CLI `1dc15838…`, bundle `layerfs` `c4737445…`, + `layerfs-daemon` `b8dd0766…`, `layerfs-sandbox-helper` `3d033ec0…`. The + Store was opened once by the new build (one-way GC schema migration, + 1,111,203,840 -> 1,111,310,336 B, live hash `d5d65e3b…`). Probe: + `layerfs branch remove … __deploy_probe_missing__` is `NoSuchBranch`; + `help branch` lists `remove`. Semantics are unchanged (§4.2): the marker + hides the Branch, `branch remove … main` is `ReservedName`, and the name + stays taken until `store.compact` rebuilds. Retirement: 307 -> 287 (20 + empty) -> 162 (125 superseded), 145 total, 0 refusals; 162 remained by + choice (unknown or non-empty, deliberately untouched). T-G75 re-probed the + live CLI from `32ae51ca`: `help branch` lists `remove`, a missing name is + `NoSuchBranch`, and the Store holds 174 live Branches (the 162 that cleanup + left plus later gap-fix task Branches). Backup: + `/home/dell/lfs-work/backup-20261115-180517` (earlier + `/home/dell/lfs-work/backup-20261115-180149`); rollback was not needed. + `layerfs doctor` is `{"ok":true,"findings":[]}`, and no Sandbox/Workspace of + the cutover or cleanup remains. +- **Validation checklist (T12.8, re-verified T-G75).** Every artifact below + exists on `32ae51ca`; rows 12-13 record the 2026-11-15 cutover and are + re-verified on main `32ae51ca`: + + | # | Artifact | Path | Check | + |---|---|---|---| + | 1 | implementation report | `core/docs/issues/project-41/gc-implementation-report.md` | present; updated by T-G75 | + | 2 | acceptance matrix | `core/docs/issues/project-41/gc-acceptance-matrix.md` | present; 60 rows, 8/30/16/6 | + | 3 | capacity model | `core/docs/issues/project-41/gc-capacity-model.md` | present; 10 GB (3 runs) and 50 GB (1 run) measured [M]; T-G72 peak bound 147.609 GB (2.90x); 30% dead / 50 GB-live and PG arms [E] | + | 4 | 76-call coverage | `docs/architecture/api/calls/coverage.md` | generated block: 76 calls, 10 families, no front-end problems | + | 5 | ext4 harness | `tools/ci/test_store_ext4.sh` | present, executable | + | 6 | CI lane | `.github/workflows/store-ext4.yml` | present; `quick` and `full-postgresql` | + | 7 | PG phase metric | `engine/crates/layerfs-persistence/src/compact_metrics.rs` | `PostgresMark => "pg_mark"` | + | 8 | real SIGKILL arms | `engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs` | `sigkill_before_seal`, `sigkill_after_seal`, `sigkill_during_copy`, `sigkill_during_verify`, `sigkill_during_rename`, `sigkill_after_build`, `sigkill_before_install`, `sigkill_after_install`, `sigkill_during_settle` | + | 9 | privilege check | `engine/crates/layerfs-persistence/src/backend/postgresql/access.rs` | `compaction_missing` checks both `CREATE`s and `pg_read_all_stats` | + | 10 | PG conninfo helper | `engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs` | present | + | 11 | PG status control | `control/crates/layerfs-store/tests/postgresql_status.rs` | present; 1 test | + | 12 | `branch.remove` live | host CLI `layerfs branch remove` | live on 2.0.0/`4e65ebbae`, re-probed T-G75; `help branch` lists `remove`; probe on a missing name returns `NoSuchBranch` | + | 13 | doctor clean after cutover | `layerfs doctor` | `{"ok":true,"findings":[]}`; 145 Branches retired (20 empty + 125 superseded), 162 remained then and 174 are live on `32ae51ca`; removal is soft, so a name returns only after `store compact` | + | 14 | clippy clean | `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` | T-G75: exit 0, zero warnings | + | 15 | final suite re-run | `cargo test -p layerfs-persistence -p layerfs-store-service -p layerfs-store --no-fail-fast` | T-G75: 544 passed / 0 failed / 18 ignored; the five SQLite compaction targets keep 20/23/1(+1 ignored)/1/1 | + + `git diff --check` is clean on the closeout Branch. +- **How to run.** + `LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick` + (or `--full`, adding `--features postgresql` when `LAYERFS_TEST_POSTGRES_URL` + is set) needs ext4/xfs/btrfs. `cargo test -p layerfs-persistence --test + sqlite_compact_matrix` runs the T12.3c arms; `UPDATE_REPORT=1 cargo test -p + layerfs-persistence --test sqlite_compact_matrix` regenerates the report. + The opt-in capacity arm is + `CAPACITY_MEASURE=1 UPDATE_CAPACITY=1 cargo test -p layerfs-persistence + --release --test sqlite_compact_capacity -- --ignored --nocapture` with a + TMPDIR of >=2.2x the target; it writes + `tests/fixtures/sqlite_compact_capacity_measure.json`. + `.github/workflows/store-ext4.yml` is the report-only CI lane. diff --git a/core/docs/issues/project-41/gc-runbook.md b/core/docs/issues/project-41/gc-runbook.md index b976397931..2a238425c6 100644 --- a/core/docs/issues/project-41/gc-runbook.md +++ b/core/docs/issues/project-41/gc-runbook.md @@ -88,7 +88,10 @@ installed → it records the outcome. A dead owner's op is adopted; only a live `commit.branch_id` keeps `NameTaken` (§4.4); `remove_file` reversal does not unhide at-or-below versions; `rebuild` install is irreversible (§9). Content recovery is `backup.create`/`backup.restore` from before compaction (SQLite; the server's backup for PG); - no undo copy is kept (§9, #155 D-24). + no undo copy is kept (§9, #155 D-24). The 2026-11-15 branch cutover keeps its + pre-change home and Store in `/home/dell/lfs-work/backup-20261115-180517` + (earlier `backup-20261115-180149`); restore that backup rather than editing + markers (see §7). - `project.remove` is gone; an old client gets `InvalidInput` (§4, §10). ## 5. Known gaps at this baseline @@ -112,3 +115,67 @@ installed → it records the outcome. A dead owner's op is adopted; only a live 4. Compare artifact/schema with the op; leave anything a record names to the next compact (`G` removes, `G+1` records). 5. Never hand-edit `gc_intent`, `gc_dead`, markers or `project_epoch`; never unlink `-wal`/`-shm`; never drop a schema `gc_op` names. 6. To re-use a name after a removal, re-create after the frontier (or after a rebuild frees it); don't delete markers. Escalate if old content must be restored. + +## 7. Branch retirement (`branch.remove`, live since 2026-11-15) + +`branch.remove` is live on the 2.0.0 build `4e65ebbae` (host CLI `1dc15838…`, +bundle `layerfs` `c4737445…`, daemon `b8dd0766…`, helper `3d033ec0…`); +`layerfs help branch` lists it, and a probe on a missing name returns +`NoSuchBranch` (exit 125). The call is a marker write, not a delete: it hides +the Branch, keeps its rows and its name until a later `store.compact` rebuild +collects the marker, and never touches `main` (`ReservedName`). + +**Preflight.** + +1. `layerfs sandbox list ` is empty, or every listed Sandbox belongs + to a task you have agreed to exclude; a mounted Branch is `MountsActive`. +2. `layerfs doctor` is `{"ok":true,"findings":[]}`; `machine status docker` is + `running`. +3. Take a backup; the cutover used + `/home/dell/lfs-work/backup-20261115-180517` (bundle, host, daemon, helper, + `bundle.json`, `worker.json` and the Store volume; the earlier one is + `backup-20261115-180149`). Keep it until the retired names are no longer + needed. +4. Page the inventory: `layerfs branch list --page-size 100 + --page-pos `; the last page has `has_more: false`. Record the + names and the count before and after. +5. Build the retire list: an empty Branch (`head: null`), or an explicitly + reviewed superseded Branch (typically a `-merged`/`-integ`/`-final`/ + `-publish` variant) whose content is already on `main` or no longer needed; + `layerfs branch diff main --paths-only` shows the paths. + Never include `main`, a running task's Branch, or a Branch with a live + mount. + +**Procedure.** + +- For each name: `layerfs branch remove `. One call per + Branch; there is no batch, force or dry-run form. `Busy` is another + operation holding the Store scope (wait, call once more); `MountsActive` is + a live mount; `NoSuchBranch` means already gone; `ReservedName` is `main`. + Record every refusal. +- Verify: the retired name reads `NoSuchBranch`, a re-fork is `NameTaken` with + the holder's `state: "retired"`, `branch list` no longer shows it, and the + `main` head is unchanged. +- **Soft name retention.** The name is not reusable until a rebuild collects + the `branch_retired` rows; plan around that and never hand-delete markers + (§4). `store.compact` below is the only supported reclaim. + +**Reclaim (optional, later).** `layerfs store compact ` only when the +Store is idle (no mounts, daemon or other op). A rebuild copies retained rows +into a fresh Store and installs `G + 1`; only then are retired names and their +rows collected. A `none` outcome (<10 % wholly dead) keeps everything, and the +retired names stay taken. Follow the compaction procedure in §6 and the +rollback limits in §4. + +**Rollback.** Restore the whole home and Store from the pre-cutover backup +above (or `backup.restore` from a `backup.create` taken during the window); +markers cannot be undone by dropping tables (§4). The 2026-11-15 cutover was +not rolled back: no call failed, and `main` and `Cargo.toml` were byte-identical +before and after. + +**2026-11-15 result.** 145 Branches retired (20 empty at cutover + 125 +superseded), 0 refusals; 162 remained by choice (unknown or non-empty, +deliberately untouched). `main` stayed `32977da5…` (root `56dbe645…`), +`layerfs doctor` stayed clean, and no Sandbox/Workspace of the cleanup was +left running. The new build's one-way GC schema migration grew the sqlite +Store 1,111,203,840 -> 1,111,310,336 B on first open. diff --git a/core/docs/issues/project-41/gc-t10-report-status.md b/core/docs/issues/project-41/gc-t10-report-status.md index 5125a0e9d9..e8646ff64d 100644 --- a/core/docs/issues/project-41/gc-t10-report-status.md +++ b/core/docs/issues/project-41/gc-t10-report-status.md @@ -74,7 +74,7 @@ backend-only field, so PostgreSQL reports its zeros through the same keys wire change. `store.status` reads the Store-side `gc_op` at the same point in `status.rs`; nothing is read from a file there. -## Metrics and SLO signals (T10.3) +## Metrics and SLO signals (T10.3, PostgreSQL T10.4) `engine/crates/layerfs-persistence/src/compact_metrics.rs` publishes the compaction observations from the real `store.compact` and `store.status` @@ -85,8 +85,9 @@ paths: `compact.retries{reason}`, `compact.status{state}` and `compact.slo{objective,result}` (`residue_free`, `generation`); - duration aggregates: `compact.phase.duration_ns{phase,statistic}` for - `mark`/`build`/`verify`/`install` and `compact.duration_ns{statistic}`, each - with `count`, `sum` and `max`; + `mark`/`build`/`verify`/`install` on SQLite and + `pg_mark`/`pg_build`/`pg_swap`/`pg_cleanup` on PostgreSQL, and + `compact.duration_ns{statistic}`, each with `count`, `sum` and `max`; - the last successful run's §11 gauges: `compact.dead_bytes{kind}`, `compact.kept_waste{part}`, `compact.kept_waste_bytes`, `compact.residue_bytes` and `compact.generation{which}`. @@ -94,12 +95,26 @@ paths: Labels come only from closed enums: outcome, phase, refusal code, settle kind, retry reason, status state, SLO objective/result, statistic and report key. A Store name, path, operation id or artifact name is never a label, so metric -cardinality is fixed. A `none` run times only `mark`; a rebuild times all four -phases; a failure or unknown run records its outcome and total but publishes -no report gauge; a run that settles an earlier leftover adds `leftover` to +cardinality is fixed. A SQLite `none` run times only `mark` and a PostgreSQL +`none` run only `pg_mark`; a rebuild times all four phases its backend names; +a failure or unknown run records its outcome and total but publishes no report +gauge; a run that settles an earlier leftover adds `leftover` to `compact.settles` and `compact.retries`. A `Busy`/`Unresolved` refusal that -stops the call before a run is counted once by the control path, and a refusal -the measured run itself produces is counted by that run. +stops a SQLite call before a run is counted once by the control path, and a +refusal a measured run itself produces (either backend) is counted by that +run. + +PostgreSQL §8 emits the same observation from +`backend/postgresql/compact.rs`: its `RunMetrics` times Phase M/A/B/C as +`pg_mark`/`pg_build`/`pg_swap`/`pg_cleanup`. Those are distinct closed values, +not aliases of the SQLite phases, so a process total never blends a SQLite +phase with a PostgreSQL phase of another meaning. It records the same §11 +gauges from the PostgreSQL receipt (where `dead_bytes_refined == +dead_bytes_bound`, `backend_reusable = 0` and `residue` is empty) and maps +`Busy`/`BusyOp` to `compact.refusals{code="busy"}` and `Unresolved` to +`code="unresolved"`. A resumed `G + 1` leftover records Phase C, the receipt's +original `generation {before, after}` and the `leftover` settle/retry; a `G` +leftover is settled before the new round. The registry follows the telemetry crate's rules (fixed low-cardinality labels, recording never changes a result). `store.compact` has no new public @@ -109,12 +124,21 @@ metrics call: `snapshot()` reads the process totals and ## Tests - `engine/crates/layerfs-persistence/tests/compact_metrics.rs`: a successful - rebuild (all four phase durations, outcome, dead/kept-waste/residue/generation - gauges and both SLO checks), the `none` path (only `mark` timed, generation - kept), a stopped `AfterBuild` run followed by the leftover settle - (`uncertain` outcome, then `rebuild` + `retries{leftover}` + - `settles{leftover}`), and a definite failure with no report gauge. Every - capture also asserts closed label keys and values. + SQLite rebuild (all four phase durations, outcome, + dead/kept-waste/residue/generation gauges and both SLO checks), the `none` + path (only `mark` timed, generation kept), a stopped `AfterBuild` run + followed by the leftover settle (`uncertain` outcome, then `rebuild` + + `retries{leftover}` + `settles{leftover}`), and a definite failure with no + report gauge. +- `engine/crates/layerfs-persistence/tests/postgresql_compact.rs`: the same + capture on a real PostgreSQL Store: a successful rebuild (the four `pg_*` + phases, outcome and the same report gauges and SLO checks), the `none` path + (only `pg_mark`), a Busy swap refusal (`failed` + + `refusals{code="busy"}`, phases M/A/B, no report gauge) and an uncertain + leftover intent refusal (`failed` + `refusals{code="unresolved"}`, no phase, + no report gauge). +- `engine/crates/layerfs-persistence/tests/support/metrics.rs`: the one + closed label-key/value assertion both capture tests use. - `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: `the_report_projects_every_section_11_field`, `the_empty_store_report_explains_its_zeros`, @@ -127,13 +151,30 @@ metrics call: `snapshot()` reads the process totals and `delta_bases.references` hex), and `store.status` projects an idle Store, a live owner's record and a leftover one (`op`, `state`, `phases`), and the `compact.status{state}` metric of each read. +- `engine/crates/layerfs-persistence/tests/postgresql_status.rs` (T10.5): on a + real PostgreSQL Store, an idle Store projects null; a live non-idle owner + projects `busy` with `uncertain`; a gone owner projects `unresolved`; each + `mark`/`build`/`swap` failpoint projects its recorded phase from the scratch + name after `gc_intent` is released; and a completed run leaves no row. +- `control/crates/layerfs-store/tests/postgresql_status.rs` (T10.5): the same + projection through the control `store.status` call on a real PostgreSQL + Store: no row projects `idle` with `op`/`phases` null; a live + `pg_stat_activity` owner projects `busy`; the gone owner projects + `unresolved`; each of `mark`/`build`/`swap`/`cleanup` is projected from the + row; the bounded `compact.status{state}` samples are emitted once each; and + a Store whose row was removed is idle again. ## Gaps - The compaction metrics are emitted into the process registry only; no SDK call or CLI command renders `snapshot()` yet. -- T9's PostgreSQL rebuild has no metrics of its own yet. -- T9's PostgreSQL rebuild and its `gc_op` status read are not implemented; - the trait and the status comment are the only PostgreSQL pieces. +- PostgreSQL compaction metrics cover the run and its admission/refusal paths. + T10.5 supplies the PostgreSQL `store.status` source: `status.rs` reads the + Store-side `gc_op` with its `gc_intent` on one status query (no Store open) + and projects `busy`/`unresolved` and the `mark`/`build`/`swap`/`cleanup` + phases, so `compact.status{state}` now has both sources. +- No control-level test joins `store.compact` and `store.status` in one run: + the control case writes the rows a compaction leaves and checks the + projection, while the persistence suites run the compactions themselves. - The `{value, cap}` capped-field shape is not produced: the SQLite receipt has no cap today. diff --git a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md new file mode 100644 index 0000000000..f3b4ce8c9a --- /dev/null +++ b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md @@ -0,0 +1,203 @@ +# T12.3a: SQLite acceptance-matrix runner report + +> Generated by `cargo test -p layerfs-persistence --test sqlite_compact_matrix`. +> The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) section 12 and +> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) AC-01..AC-60 win; this +> page records evidence and adds no contract. Regenerate with +> `UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix`. + +## Scope and counts + +- Rows: 60 (AC-01..AC-60); backend split: both 47, SQLite-only 6, PG-only 7. +- Executed here: 8; covered by a named existing test/fixture: 29; partial: 16; deferred to T12.3b: 7; SQLite gaps: 0. +- **T-G28/T-G60 follow-up.** T-G28 later added the real-process SIGKILL arms + and T-G60 extended them to mark, copy, verify and rename, so + `sqlite_compact_crash` now has 23 tests (9 real-process SIGKILL arms and the + child placeholder), not the 13 recorded below. Those arms still kill a + re-executed test child, so T-G65 added + `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs`: 8 SIGKILL + arms on the built `layerfs store compact` process at mark, build, verify and + install, so AC-01 is `covered` (the matrix now reads covered 30 / partial + 16). This page keeps the T12.3c run's original counts. +- A `covered`/`partial` row names a file and test function that this runner + asserts exists; an `executed` row names an arm this binary really runs. +- A cross-backend row records its SQLite half only; every PostgreSQL half + and the PostgreSQL-only rows are deferred to T12.3b by name. +- **T-G61 capacity follow-up.** A separate opt-in test, + `sqlite_compact_capacity.rs#capacity_measure` (ignored by default), measured a + real **9.987 GB** Store: `none` at 8.88% dead (three runs, median 149.318 s) + and `rebuild` at 10.00% dead (three runs, median 246.371 s; mark 149.089 s, + build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink + 1.017 GB, peak temp 9.144 GB), plus one WAL/FULL rebuild (284.078 s). The + numbers are in `tests/fixtures/sqlite_compact_capacity_measure.json`; the + 50 GB arms and the 30% arm remain [E]. + +## Matrix (SQLite half) + +| Case | Backend | Matrix tier | Disposition | Evidence | Note | +|---|---|---|---|---|---| +| AC-01 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_after_verify; sqlite_compact_crash.rs#crash_after_install; sqlite_compact_crash.rs#crash_during_settle | The SQLite section 9 crash matrix through named failpoints; a real kill -9 of a separate daemon process is T12.2's in-process stop points, not run here. | +| AC-02 | both | E2 | partial | sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | The SQLite arm runs through the suite; the converted 1.0 Store arm and the PostgreSQL backend matrix are deferred to T12.3b. | +| AC-03 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_after_wal_switch; sqlite_compact_crash.rs#crash_before_install; sqlite_compact_crash.rs#crash_after_install | SQLite mark/build/verify/install and WAL-switch stop points; lost COMMIT and power-loss arms belong to the PostgreSQL/fsync halves. | +| AC-04 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Deterministic closure fixture plus a seeded six-history reachability property arm; the full random fork/commit/merge/publish/removal generator is not built here. | +| AC-05 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Full doctor (including the diagnostic scan) on the compacted Store, also in the property arm. | +| AC-06 | both | E2 | executed | this runner #arm_digest_row_after_collection | A Save's `file_digest` row named a root the rebuild collected; the row goes with its root and recording the same content again succeeds. The PostgreSQL half is T12.3b. | +| AC-07 | both | E2 | executed | this runner #arm_mount_and_compaction_exclude_each_other; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | SQLite half executed: bypass gives `StaleStore`, a mount blocks compaction (`MountsActive`, naming the holder) and a live compaction refuses a mount and a second compaction (`BusyOp`). The open one-shot SDK transaction arm is PostgreSQL, T12.3b. | +| AC-08 | both | E2 | covered | admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | One winner; the loser sees MountsActive, and a removed Branch is Missing. | +| AC-09 | both | E2 | covered | branch_removal.rs#a_removed_branch_is_hidden_and_the_epoch_bumps_once; branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches | Reserved main, absent Branch and holder details are asserted; project-side name collisions are in AC-30. | +| AC-10 | both | E2 | covered | path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Removed/UpToDate codes and the frontier; 2.0 frees no bytes for the removed path. | +| AC-11 | both | E2 | partial | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden; sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount | Non-FF whiteouts, the fast-forward control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute. A merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed. | +| AC-12 | both | E2 | covered | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit | A removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept, and only an ordinary edit conflict on it is reported. | +| AC-13 | both | E2 | covered | sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit | The earlier Commit on an unmerged Branch is hidden by the marker; a Commit after the frontier does not inherit the path. | +| AC-14 | both | E2 | covered | sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount; sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes | A Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount; its next Commit records the deletion. | +| AC-15 | both | E2 | partial | kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retention of the only-reaching-path object is proven; marker hiding in reads/listings and derived whiteouts are not in one named test. | +| AC-16 | both | E2 | covered | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | T12.1 pins a retained retired Branch name and a collected unreferenced one; the branch.remove-then-rebuild arm is exercised through those markers. | +| AC-17 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#a_delta_base_is_enqueued_and_reported | Depth-2 chain copied, pack ids preserved. | +| AC-18 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Collected digest row gone, retained one kept. | +| AC-19 | both | E2 | partial | gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers; path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Path-grammar refusals and the marker frontier are named; the full form table (symlink/hardlink, >4096 bytes, >256 components) is not one test. | +| AC-20 | both | E2 | partial | path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date; path_removal.rs#an_unknown_project_is_missing | UpToDate and never-existing paths are named; the re-creation and other-project arms are partial. | +| AC-21 | both | E2 | partial | epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back; epoch_fence.rs#a_concurrent_commit_and_removal_have_one_winner | One-shot writes racing removals roll back as Conflict; the merge-racing-remove_file arm is not named. | +| AC-22 | both | E3 | covered | sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible; epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back | A deterministic barrier starts a merge and a `remove_file` together for three rounds: the loser sees the marker or gets `Conflict` and retries with the whiteout, and a tree derived afterwards never shows the path. The PostgreSQL half is T12.3b. | +| AC-23 | both | E2 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Deleted-project rows and markers are collected by the rebuild; a dedicated path_removed cleanup arm is not named. | +| AC-24 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order; gc_backfill.rs#existing_serials_are_kept_and_new_ones_start_above_the_maximum | Serials backfilled, next_history_serial never falls. | +| AC-25 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retained branch_retired/layer_stack_retired markers survive; collected rows take their markers. | +| AC-26 | both | E2 | covered | project_delete.rs#retry_classifies_retired_live_and_absent; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Hidden, entry present, retry returns the stored deleted_ms. | +| AC-27 | both | E2 | covered | project_delete.rs#paging_skips_a_hidden_project; project_delete.rs#every_layer_stack_lookup_carries_the_marker_filter | Stale home entries stay out of list/lookup. | +| AC-28 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup and its failpoints: T12.3b. | +| AC-29 | both | E2 | covered | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved; admission.rs#a_mount_blocks_all_three_removals_until_released | Owner liveness and Unresolved settlement; machine_stopped and the other-home intent are partial. | +| AC-30 | both | E2 | partial | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect | Holder detail and refusal codes are named; the full exact-message matrix (AC-55) is partial. | +| AC-31 | both | E2 | covered | gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Byte-count path arguments and the keyed trace hash are asserted. | +| AC-32 | both | E2 | partial | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved | Live/uncertain ownership is named; live-but-unreachable, machine_stopped and another home's intent are partial. | +| AC-33 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_sidecar_at_install | A live reader makes wal_checkpoint(TRUNCATE) Busy with no artifact and no op; a -mj* super-journal at install is Conflict with no writes. | +| AC-34 | both | E2 | partial | sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active | A committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains. The failed-delete sub-arm is not inducible on SQLite: the storage schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register too; the explicit `Cleanup` report is covered by the seam's code path, not executed here. The PostgreSQL half is T12.3b. | +| AC-35 | PG | E2 | deferred (T12.3b) | - | PostgreSQL advisory-key refcount: T12.3b. | +| AC-36 | both | E2 | covered | project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry; project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry; project_delete.rs#an_absent_stack_finishes_a_stale_entry | A lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent. The PostgreSQL half is T12.3b. | +| AC-37 | both | E2 | partial | gc_trace.rs#the_redacted_set_covers_every_path_bearing_call; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Redaction and the keyed hash are named; salt backfill on an upgraded Store is partial. | +| AC-38 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_during_settle | The SQLite artifact set is removed exactly; the PostgreSQL schema/name arms are T12.3b. | +| AC-39 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order | One project_epoch row per stack, gc_op not copied, next_history_serial never lowered. | +| AC-40 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#the_digest_is_stable_and_moves_with_a_head | SQLite live_heads_digest is identical before/after and moves with a head; PostgreSQL agreement is deferred. | +| AC-41 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase M mark transfer: T12.3b. | +| AC-42 | PG | E2 | deferred (T12.3b) | - | PostgreSQL reserved pack/Phase B: T12.3b. | +| AC-43 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup resume: T12.3b. | +| AC-44 | PG | E2 | deferred (T12.3b) | - | PostgreSQL privileges and lost COMMIT reply: T12.3b. | +| AC-45 | SQLite | E1 | covered | sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none; sqlite_compact.rs#group_rows_indexed_layout_opens_compacts_and_takes_none; sqlite_compact.rs#acquisition_layout_opens_compacts_and_clears_working_rows; sqlite_group_rows.rs#explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order | Monolithic/GroupRows/GroupRowsIndexed and the acquisition variants compact; working rows are cleared. | +| AC-46 | SQLite | E2 | executed | this runner #arm_sidecar_at_install; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | Sealed-state/sidecar change at install gives Conflict with no writes; a replaced Store gives StaleStore. The in-flight-transaction exclusion is a documented lane limit, not probed. | +| AC-47 | SQLite | E2 | covered | sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_during_settle; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Crash leaves the full set, the next compact removes it, none leaves no artifact, a stray name is Exists. | +| AC-48 | both | E2 | covered | sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path; sqlite_compact.rs#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold; sqlite_compact.rs#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold; sqlite_compact.rs#an_empty_store_takes_the_none_path; sqlite_compact.rs#a_capacity_failure_leaves_the_store_unchanged | 9/10/30 percent, empty-store zeroReason, capacity abort; verify-failure and re-refine arms are partial. | +| AC-49 | both | E2 | executed | this runner #arm_mount_during_compaction_is_busy; this runner #arm_commit_during_rebuild; admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | SQLite half executed: a mount during compact is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild gets `StaleStore` or lands, and its retry is never lost. The PG Phase A/Phase C arms are T12.3b. | +| AC-50 | both | E2 | covered | calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover; gc_calls.rs#the_four_gc_calls_take_ids_only | Folder scope refuses the four calls; the admin profile is the only allowance. | +| AC-51 | both | E2 | executed | this runner #arm_status_phases; compact_report_status.rs#status_projects_the_sidecar_phase_and_state; sqlite_compact_crash.rs#crash_before_seal | The op sidecar records mark/build/verify/install (runner); T10 projects it through store.status, and the named control test asserts the phase, state and phases list. | +| AC-52 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json; compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape | Every section 11 field pinned with units and sampling point (golden) and rendered by the T10 wire-shape test; markBytes is a metric, not a result field. | +| AC-53 | both | E2 | partial | project_export.rs#export_omits_a_path_hidden_by_remove_file; gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers | One hidden-path export case is named; the full read-surface table (read/stat/list/glob/diff/revision/edit/grep/blame/cp/apply) is not executed. | +| AC-54 | both | E2 | partial | restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install; restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it | Restore install guards and restored records are named; restore before/after a removal and the stale-entry listing arm are partial. The restore-window arm needs a lock-capable filesystem (ext4/xfs/btrfs), so on this Sandbox it reports Unsupported while the other two cases pass. | +| AC-55 | both | E2 | executed | this runner #arm_error_codes; this runner #arm_checkpoint_busy; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted | Exact SQLite texts asserted: Busy, Conflict("Store sidecar at install"), Exists("unrecorded compaction artifact"), Capacity("disk space"); the full engine/API message matrix stays partial. | +| AC-56 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json | The literal SQLite vector is pinned (absent head 33 zero bytes); the shared PostgreSQL literal and the three-run E3 shape are deferred. | +| AC-57 | both | E3 | partial | this runner #arm_measurement; sqlite_compact_measure.json; T-G61 sqlite_compact_capacity.rs#capacity_measure; sqlite_compact_capacity_measure.json | The tiny three-run arm is median 28.2 ms. T-G61 measured a real 9.987 GB Store with 3 runs per arm: `none` at 8.88% dead (median 149.318 s) and `rebuild` at 10.00% dead (median 246.371 s, mark 149.089 s / build 94.317 s / verify 0.312 s / install 1.916 s), plus one WAL/FULL rebuild at 284.078 s; profile, host and toolchain are declared in the fixture, and the warm page cache is declared below. The section 13 50 GB arms, the 30% arm, each removal type, the plain `VACUUM INTO` comparison and PostgreSQL timings remain deferred/[E]. | +| AC-58 | SQLite | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_before_copy; sqlite_compact_crash.rs#crash_during_copy | none deletes the scratch and leaves no .compact; rebuild keeps the visited scratch through the build; step 2 writes only on rebuild. | +| AC-59 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_seal; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale | busy != 0 refuses Busy and the WAL is never unlinked; a successful seal records (size, mtime, counter) as fenced by the StaleStore arm. | +| AC-60 | PG | E2 | deferred (T12.3b) | - | PostgreSQL synchronous standby handling: T12.3b. | + +## Executed in this runner + +- AC-33 / AC-59: `arm_checkpoint_busy` - a live reader makes + `wal_checkpoint(TRUNCATE)` report busy, the compaction refuses `Busy`, + leaves no `.compact`, `.compact-visited` or `.compact-op`, and the WAL is + never unlinked; the next run rebuilds at `G -> G+1`. +- AC-33 / AC-46: `arm_sidecar_at_install` - a `-mj*` super-journal placed + at install is `Conflict("Store sidecar at install")` with no artifact, + no op and an unchanged generation. +- AC-51: `arm_status_phases` - stop points at mark, build, verify and install + record the same phase in `.compact-op`; every phase is observed. +- AC-55: `arm_error_codes` - exact `Exists("unrecorded compaction artifact")` and + `Capacity("disk space")` texts, with no artifact after the capacity abort. +- AC-59: `arm_seal` - a successful seal leaves no sidecar, reports the main-file + bytes and a SQLite version, and the file reopens. +- AC-04 / AC-05: `arm_random_store_shapes` - a seeded property arm over six + reachability shapes: every live object stays readable, every wholly dead one + is collected, the retained-closure digest is unchanged and the doctor is clean. +- AC-06: `arm_digest_row_after_collection` - a `file_digest` row named a + root the rebuild collected, so the row goes; recording the same content + again succeeds and points at the new root. +- AC-07 / AC-49: `arm_mount_and_compaction_exclude_each_other` - a live mount + makes compaction `MountsActive` naming it, and a live compaction makes a + mount and a second compaction `BusyOp`; the refused runs leave no op. +- AC-49: `arm_mount_during_compaction_is_busy` - the stopped run's live + compaction intent refuses a mount with `BusyOp`, and the next run adopts + the leftover and rebuilds. +- AC-49: `arm_commit_during_rebuild` - a Commit started inside the install + fence is `StaleStore` or lands on `G + 1`; a retry is never lost. +- AC-57: `arm_measurement` - three runs of the same rebuild with median and + range in `engine/crates/layerfs-persistence/tests/fixtures/ + sqlite_compact_measure.json` (refresh with `UPDATE_MEASURE=1`). The separate + T-G61 `sqlite_compact_capacity.rs#capacity_measure` measures a real 9.987 GB + Store (three `none` + three `rebuild` runs, one WAL/FULL control) into + `sqlite_compact_capacity_measure.json`; it is opt-in because the fixture is + ~10 GB and the ignored test is not part of a normal suite. + +## Deferred to T12.3b (PostgreSQL) + +- AC-28 - PostgreSQL Phase C cleanup and its failpoints: T12.3b. +- AC-35 - PostgreSQL advisory-key refcount: T12.3b. +- AC-41 - PostgreSQL Phase M mark transfer: T12.3b. +- AC-42 - PostgreSQL reserved pack/Phase B: T12.3b. +- AC-43 - PostgreSQL Phase C cleanup resume: T12.3b. +- AC-44 - PostgreSQL privileges and lost COMMIT reply: T12.3b. +- AC-60 - PostgreSQL synchronous standby handling: T12.3b. +- The PostgreSQL half of every `both` row (digest agreement, artifact + names, Phase M/B/C, standby, lost reply, 50 GB arms). + +## SQLite gaps (not claimed by this runner) + +- none; every SQLite gap this runner can execute is executed or covered. + +## E3 arms + +- AC-04: the T12.1 closure fixture (deterministic) plus the seeded + reachability property arm; the full random history generator is deferred. +- AC-22: a deterministic barrier starts a merge and a `remove_file` together + for three rounds; the loser retries and a derived tree never shows the path. +- AC-56: the SQLite literal vector is pinned by the T12.1 golden; the shared + PostgreSQL literal is deferred. +- AC-57: three-run measurement arm executed on this Sandbox (small fixture), + plus T-G61's real 9.987 GB capacity fixture with three runs per arm + (`none` 149.318 s median, `rebuild` 246.371 s median, phase sums recorded). + Its 50 GB projection, the 30% arm, PG `ACCESS EXCLUSIVE` duration and the + plain `VACUUM INTO` comparison are deferred/[E]. + +## Status and report projection + +T10 (merged from main after this branch forked) projects `.compact-op` +through `store.status`: `op {id, name, phase, started, uncertain}`, the +busy/unresolved `state` and the backend `phases`. The named control tests +in AC-51 and AC-52 assert that projection and the snake_case section 11 +render; the runner still asserts the recorded SQLite phases directly, so +AC-51 does not depend on the projection alone. + +## Run environment + +- T-G61 capacity run (separate opt-in test): Docker Sandbox `g61sbx` + (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 overlay, 4.0 TB free), rustc + 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF) plus one + `Durable` control, `CARGO_TARGET_DIR=/var/tmp/target-g61`, fixture under + `/var/tmp/g61big`; the test ran 1678 s. Caches: the ~10 GB Store fits in the + 135 GB page cache and no cache drop ran between the three runs of an arm; + the mark is CPU-bound at ~62 MB/s, not cache-bound. +- Docker Sandbox `t123csbx` (Linux x86_64, tmpfs `/tmp`, overlayfs + `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable` + (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`. +- Verified here: `sqlite_compact_matrix` (1 test); the persistence suites + `sqlite_compact` (20), `sqlite_compact_crash` (13), + `sqlite_compact_fixtures` (1), `admission` (5), `path_removal` (3), + `branch_removal` (3), `project_delete` (5), `epoch_fence` (7), + `gc_backfill` (4), `kept_objects` (5); the Store-service path suites + `sdk_path_merge_removal` (5), `sdk_mount_bind_failure` (1), + `sdk_path_whiteouts` (7), `sdk_path_hidden_semantics` (2), + `sdk_path_removal` (2); and `layerfs-control --features postgresql + project_delete_worker` (5), `layerfs-store --features postgresql + project_delete` (15) and `compact_report_status` (3). + All of those pass on the merged branch. +- The control `restore_install` arm that opens a Store needs ext4/xfs/btrfs; + this Sandbox has no such direct filesystem, so that one case reports + `Unsupported` while its other two cases pass. It is an environment limit, + not a matrix result; the cited API tests are validated by source name only + in this run. diff --git a/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md b/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md new file mode 100644 index 0000000000..ddefe61a52 --- /dev/null +++ b/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md @@ -0,0 +1,150 @@ +# T12.3b: the PostgreSQL half of the GC acceptance matrix + +> Acceptance note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §8/§9/§11/§12, the companion of +> [`gc-t12-sqlite-compaction-fixtures.md`](gc-t12-sqlite-compaction-fixtures.md) +> (SQLite half) and [`gc-t9-postgresql-rebuild.md`](gc-t9-postgresql-rebuild.md) +> (the PostgreSQL rebuild). It adds no contract: the spec and the +> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) win; `§N` and `AC-nn` +> cite them. The matrix statuses stay `open` (only #52 records acceptance); +> this note records the PG-half evidence and its tier. +> +> **T-G28/T-G60 follow-up.** `sqlite_compact_crash` now has 23 tests (9 +> real-process SIGKILL arms at mark, copy, verify, rename, build, install and +> settle, plus the child placeholder), not the 13 passed recorded below, and +> the matrix statuses are no longer `open`: AC-01 is `partial` and the other +> rows are `executed`/`covered`/`partial`/`deferred`. This page keeps the +> T12.3b baseline counts. + +## Environment + +- PostgreSQL **17.12** (`postgresql-17_17.12-1.pgdg12+1`, Debian 12), server on + `127.0.0.1:55432`, one schema per test dropped at test end, `postgres` + superuser, `sslmode=disable`. +- `LAYERFS_TEST_POSTGRES_URL` was set for every run below. The suites that pass + the URL straight to the driver take the keyword form + (`host=127.0.0.1 port=55432 user=postgres dbname=postgres sslmode=disable`). + The suites that append startup options (`postgresql_diagnostic`) need the URI + form (`postgresql://postgres@127.0.0.1:55432/postgres?sslmode=disable`); + with the keyword form those three tests fail at connect, which is a harness + URL-shape issue, not a Store result. +- Toolchain: Rust 1.85.1, `cargo test -p layerfs-persistence --features + postgresql`, `-j 8`, `CARGO_TARGET_DIR=/tmp/target-merged`. +- Branch `task-t12.3b` forked main layer `32c9181e`; it was merged with main + before publishing, and every result below was re-run on the merged tree + (main `3284fb82` at the merge), so the commands and counts are the merged + branch's, not the fork base's. + +## The fixture + +`engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs` is +the T12.1 fixture rebuilt on PostgreSQL through the public APIs only. It is a +new test target (5 tests); it changes no production code. + +| Test | What it pins | Matrix rows | +|---|---|---| +| `the_live_heads_digest_vector_is_the_sqlite_literal` | The §6.4 vector on PG is **byte-identical to the T12.1 SQLite golden**: the Commit id, `absentHeadDigest` (33 zero bytes) and `retiredBranchDigest` are the literals embedded in `fixtures/sqlite_compact_golden.json`. | AC-40, AC-56 (PG half) | +| `the_rebuild_report_and_markers_hold_on_postgresql` | The full T12.1 rebuild fixture on PG: `live_heads_digest` equals the SQLite golden **rebuild** literal; `deadBytesBound == deadBytesRefined`; `backendReusable = 0` with `postgresql-in-place`; `physicalShrink`, `peakTempBytes`, `keptWaste.{deadInRetainedPacksBytes, inodeBlocksBytes, deltaBases}`; `logicalDiscarded == 160`; empty residue; `G -> G+1`; depth-2 chain kept with `pack_id` and pack length unchanged; retained pack keeps its unreached group; markers/digest rows follow the retained set; full doctor; epoch copied, serial never lowered. | AC-02, AC-05, AC-17, AC-18, AC-25, AC-40, AC-41, AC-43, AC-52 | +| `a_save_after_the_swap_survives_the_resumed_cleanup` | `PostgresAfterSwap` aborts the round at `G+1`; a Save lands after the swap; the resumed Phase C deletes only the persisted dead set, the late write (object, pack, totals) survives the high-water bound, and the Store is usable at `G+1`. | AC-28, AC-43, AC-49 (Phase C arm) | +| `every_postgresql_failpoint_leaves_a_resumable_leftover` | All four named §8 failpoints (`mark`, `build`, `swap`, `cleanup`) leave the documented state (generation, dead pack, scratch schema, op phase, owner pid) and the next call adopts/resumes it; the heads digest is unchanged and the doctor is clean. | AC-01 (in-process arm), AC-03 (PG failpoints), AC-28, AC-43 | +| `a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write` | Commit/Layer leave `project_epoch` alone; a Branch removal and a `remove_path` marker bump it once each; a repeated marker is `UpToDate` and does not bump; compact copies the epoch and never lowers `next_history_serial`; a removal-first stale Commit and a missing epoch row are both `Conflict("project epoch")` and write nothing. | AC-21, AC-24, AC-39 | + +The digest equality is not a value comparison on "both backends ran" alone: the +fixture ids, roots and Commit records are content-addressed, so the PostgreSQL +mark hashing the SQLite literal proves the §6.4 vector agrees byte for byte. +Run three times, the five tests passed 3/3 with the same literals (the matrix +lists AC-56 at E3; the deterministic two-backend literal is E2, three +identical runs are recorded here for the E3 arm). + +## Matrix: requested PostgreSQL-half rows + +Counts: **19 rows — 5 passed, 6 partial, 8 deferred** (AC-33 is a SQLite-only +row and is not part of the PG half). + +| Case | Result | Tier | Evidence / leftover | +|---|---|---|---| +| AC-02 | partial | E1 | The additive step on PG (position columns + serial backfill) passes `postgresql_position_columns` 4/4; the report/removal parity is covered below. A converted **1.0** Store is SQLite-only: PostgreSQL is a 2.0 format with no 1.0 reader, so that arm is not runnable on this backend. | +| AC-28 | **pass** | E1 | `a_save_after_the_swap_survives_the_resumed_cleanup`: late Save survives the resumed Phase C; op/generation asserted. Deferred sub-arm: the marker/base-row batch boundary failpoint (no injected point inside `cleanup`; Postgres only has the four round failpoints). | +| AC-29 | partial | E1 | `postgresql_compact` 8/8: a live idle owner is terminated by `(pid, backend_start)` and adopted; the caller's own uncertain round is adopted without self-termination. Mount-work admission belongs to the Store/Workspace layer and is not in this harness (deferred). | +| AC-30 | deferred | E0 | The `NameTaken` detail/error matrix is backend-neutral history work; no PG-specific fixture was added here. | +| AC-31 | deferred | E0 | Call-log/trace-salt redaction is the control-layer path, not the persistence harness. | +| AC-32 | partial | E1 | Same T9 owner-adoption evidence as AC-29 (uncertain op settled by the owning call). "Another home's intent visible" is control-layer, deferred. | +| AC-34 | deferred | E0 | A failed intent bind/delete needs the admission registry around a mount/init call, not this harness. | +| AC-35 | deferred | E0 | The project/Store advisory-key refcount exercise is the mount layer (`pg_advisory` keys), outside the persistence fixture. | +| AC-36 | partial | E1 | Lost swap reply (`postgresql_compact`), lost cleanup reply and lost mark/build replies (`every_postgresql_failpoint_leaves_a_resumable_leftover`) all resolve as documented. An unreachable worker/remote Store is deferred (needs the worker transport). | +| AC-37 | deferred | E0 | Trace redaction is the control-layer call log. | +| AC-38 | partial | E1 | The scratch name `lfs_gc_` is recorded before CREATE and removed at the swap/settle: the four-failpoint test asserts `scratch_exists` per phase. Deferred: a remote machine removing the scratch schema and a stray schema refusing with `Exists`. | +| AC-39 | **pass** | E1 | `postgresql_position_columns` 4/4 (one `project_epoch` row per stack, serials backfilled once, never lowered) plus the new fixture: the rebuild copies the epoch, `gc_op`/`gc_dead` are cleared on settle and not copied. | +| AC-40 | **pass** | E2 | `the_live_heads_digest_vector_is_the_sqlite_literal` + `the_rebuild_report_and_markers_hold_on_postgresql` vs the T12.1 SQLite golden; the retired head moves the digest on both. | +| AC-41 | partial | E1 | The depth-2 chain is kept only as delta bases, both reported in identity order; every retained pack keeps its `pack_id` and its `length` after the swap. Deferred: `markBytes` as a wire metric — it is a #52 metric and is not a `store.compact` result field (§11 matrix note), so there is nothing to assert in the receipt. | +| AC-42 | deferred | E0 | A Save reserved before the mark and published during Phase A must make the swap `Conflict`. The PG round has no pause point between build and swap (`stop()` returns immediately), so a publication cannot be interleaved deterministically; a wall-clock race would not be a fixed fixture. | +| AC-43 | **pass** | E2 | After the swap and the resumed cleanup: the four `store_totals` equal their row sums, `pack`/`object_location` keep owner and the `pack_immutable` trigger bound to the schema function (and the trigger still refuses an update), `binding_hint_block` is cleared, there is no `inode` table on PG, the Store opens at `G+1` and passes the full doctor. | +| AC-44 | deferred | E0 | Missing `CREATE`/`TEMP`/`pg_read_all_stats` needs roles the suite may create and a server it may reconfigure; a truly lost `COMMIT` reply needs a killed backend. The `PostgresAfterSwap` failpoint is the deterministic substitute and proves resume, not reply loss. | +| AC-56 | **pass** | E2 (3/3 runs) | The PG digests are the T12.1 SQLite literals: absent head (`6864c4…`), retired branch (`dafcdd…`), and the rebuild vector (`d3a3ea…`); the frozen Commit id `121c28…` is identical on both backends. | +| AC-60 | deferred | E0 (predicate E1) | A real synchronous standby needs a second cluster and `synchronous_standby_names`/`synchronous_commit` changes on a dedicated server; a shared suite URL cannot own those GUCs. `compact.rs` unit tests pin the predicate (no names / below `remote_write` → no wait; names → a streaming sync/quorum standby with a matching `sync_state`). | + +## Matrix: additional PostgreSQL rows recorded by this note + +Counts: **11 rows — 6 passed, 5 partial, 0 deferred**. + +| Case | Result | Tier | Evidence / leftover | +|---|---|---|---| +| AC-01 | partial | E1 | The four in-process PG aborts leave a wholly old (`mark`, `build`) or wholly new (`swap`, `cleanup`) Store and the next call settles. A `kill -9` of a daemon at the same points is not injectable in this harness. | +| AC-03 | partial | E1 | Named PG failpoints `mark`, `build`, `swap`, `cleanup` are all exercised with state, generation, op phase, scratch and digest asserted. Lost `COMMIT`, power loss and lease takeover are deferred (AC-44 note). | +| AC-05 | **pass** | E2 | Full doctor on the compacted PG Store in both rebuild tests; `postgresql_diagnostic` 5/5 (transport, cancellation, denied signature). | +| AC-17 | **pass** | E1 | Depth-2 PREFIX chain copied; `pack_id` preserved for both bases and the target. | +| AC-18 | **pass** | E1 | The collected `file_digest` root is gone, the retained one survives. | +| AC-21 | partial | E1 | Removal-first refusal (`Missing`/`Conflict`, no Commit) and the missing-epoch-row `Conflict` are deterministic on PG; the true concurrent one-shot-write arm (a trigger injecting an epoch change mid-transaction, as the SQLite `epoch_fence` tests do) is not implemented for PG here. | +| AC-24 | **pass** | E1 | Marker write bumps `project_epoch` once; `UpToDate` does not; compact copies it; serial never falls. | +| AC-25 | **pass** | E1 | A retained `branch_retired` row survives a rebuild; a collected Branch takes its marker with it. | +| AC-49 | partial | E1 | Phase C survives the high-water bound (late Save); the mount-wins-the-race arm needs the mount/lease layer, deferred. | +| AC-51 | partial | E1 | The op row records the §8 phase (`swap` asserted after the failpoint, `mark`/`build` in the matrix test). Projecting it through `store.status` needs a Store-home fixture; the T10 serializer test is SQLite-side today. | +| AC-52 | **pass** | E2 | The PG report has every §11 component: `deadBytesBound == deadBytesRefined`, `backendReusable = 0` + `postgresql-in-place`, `physicalShrink > 0`, `peakTempBytes > 0`, `keptWaste` three parts, empty residue, `logicalDiscarded` = history delta, `estimate = true`, zero reasons named. | + +## Commands and results + +Server: PostgreSQL 17.12 on `127.0.0.1:55432`; `LAYERFS_TEST_POSTGRES_URL` set as +above; `-j 8`; `--test-threads=1`. + +| Command (`cargo test -p layerfs-persistence --features postgresql --test …`) | Result | +|---|---| +| `postgresql_compact_fixtures` (new) | **5 passed**; 3 consecutive runs, 5 passed each | +| `postgresql_compact` | 8 passed | +| `postgresql_position_columns` | 4 passed | +| `postgresql_open_check` | 3 passed | +| `postgresql_commit_round_trips` | On the fork base: 1 failed, `13 != 11`; current main already fixes the expectation (see below), the merged tree passes 1/1 | +| `postgresql_diagnostic` (URI URL) | 5 passed | +| `postgresql_acquisition_compaction` | 4 passed | +| `postgresql_acquisition` | 4 passed | +| `postgresql_identical_ids` | 3 passed | +| `layerstack_removal` | 2 passed (one is the PostgreSQL project-removal arm) | +| `sqlite_compact_fixtures` | 1 passed (the golden still matches) | +| `sqlite_compact_crash` | 13 passed | +| `sqlite_compact` | 20 passed | + +## Baseline note: the stale round-trip count is already fixed on main + +`postgresql_commit_round_trips.rs` expected 11 round trips for `lane + head + +commit_branch` on the fork base; the path takes 13. The two extra statements are +T6.5's writer fence (`gc_epoch_read` at the start of the write transaction and +`gc_epoch_share` … `FOR SHARE` as its last statement, `src/history/epoch.rs`). +Current main already carries the updated expectation (13), so this branch does +not change that file; it only records the finding. + +## Gaps (not contract changes) + +- **AC-42** (a Save reserved before the mark racing the swap) and **AC-60** + (real standby, missing standby before/inside the swap, dying after the check) + need a hook the deterministic round does not have or a dedicated server whose + GUCs the suite owns. +- **AC-44** privileges and a killed backend's lost `COMMIT` need server roles + and processes the suite cannot own; the swap failpoint is the recorded + substitute. +- **AC-28**'s marker/base-row batch boundary needs a failpoint inside Phase C; + only the four round failpoints exist. +- The mount/lease admission rows (**AC-29/32/35/49**) are Store-layer, not + compaction-layer; only their compaction-op liveness arm is here. +- **AC-30/31/34/37** (error detail, trace, intent bind, redaction) are + control-layer rows and are untouched. +- `markBytes` (AC-41) is not a `store.compact` field, so the PG half cannot + assert it from the receipt. diff --git a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md index cc8461befe..ca544b11a7 100644 --- a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md +++ b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md @@ -88,6 +88,16 @@ without killing the process. the collected history rows' `logical` bytes; `parallel` note: `estimate` is always true until #52 measures the arms. +`peakTempBytes` is the §11 peak temporary space as a documented upper bound +(T-G72): `staged_bytes + journal_bound + target_bytes + visited_bytes`. The +staged copy holds the sealed Store's whole extent until the rewrite truncates +it (`VACUUM INTO`); the rewrite holds SQLite's unlinked temp database (the new +file) beside it and journals the pages it overwrites; the journal bound is one +rollback-journal record per staged page (`page_size + 8` B) plus 8192 B of +header. The 50 GB arm's bound is 147.609 GB against a 143.1 GB sampled temp +peak; the old `new + visited` receipt (45.784 GB) dropped the 44.6 GB +rollback journal (T-G66 finding, T-G72 fix). Upstream issue: [Ephemeral-AI-Lab/layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + ## Tests `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: @@ -106,6 +116,12 @@ always true until #52 measures the arms. next call judges it and settles; - generation increases once per installed rebuild. +T-G72 (`tests/sqlite_compact_capacity.rs`): a moderate fixture samples the +staged copy, the rollback journal and SQLite's unlinked rewrite temp database +from a second thread and asserts the receipt bounds the new file plus the +journal and the whole sampled footprint; `compact.rs`'s unit test pins the +50 GB arithmetic. + Runs: `cargo test -p layerfs-persistence --offline` (all binaries pass), `cargo check --workspace --offline` passes. `kernel_guard` in `layerfs-store-service` fails in this container because `/tmp` is tmpfs and @@ -132,7 +148,10 @@ the guard refuses direct Store access; it is unrelated to this change. and a live intent that outlived its op record refused every later GC call `BusyOp`. The op record is kept only when the release itself fails, as the next run's recovery breadcrumb (§9). -- Peak temp is the new file plus the visited scratch; journal/WAL sizes - after the checkpoint are zero and are not added. +- ~~Peak temp is the new file plus the visited scratch; journal/WAL sizes + after the checkpoint are zero and are not added.~~ T-G72: `peakTempBytes` + bounds the pre-truncate staged copy, the new file (the rewrite's temp + database), the prune/rewrite rollback journal and the visited scratch. It is + an estimate and over-reports the sampled peak by a few percent. - The mark runs before the seal under the same lane (the spec's step 1.5 runs it after the seal); the sealed tuple re-read makes the window explicit. diff --git a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md index 1d9265c448..041dd5db97 100644 --- a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md +++ b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md @@ -94,6 +94,12 @@ postgresql_compact` — 8 passed on PostgreSQL 17.12 (Debian, server on `127.0.0.1:55432`); `--lib` 4 passed; the SQLite suite (`--test sqlite_compact`) passed (17 when T9 branched, 20 on the rebased main with the T12 fixtures); `sqlite_compact_crash` passed 13. +> **T-G28/T-G60 follow-up.** `sqlite_compact_crash` grew to 23 tests with the +> real-process SIGKILL arms (9 arms after T-G60), so the 13 passed here is the +> T9 baseline. The +> original `Gaps` bullet below still says those rows remain `open`; the current +> matrix has them `partial`/`deferred`. This page keeps the T9 counts. + ## Gaps (not contract changes) - **AC-41/AC-42/AC-43/AC-44/AC-60 remain `open`** in the acceptance matrix: diff --git a/docs/architecture/api/calls/coverage.md b/docs/architecture/api/calls/coverage.md index a6cfa5dd53..53f4a26fcb 100644 --- a/docs/architecture/api/calls/coverage.md +++ b/docs/architecture/api/calls/coverage.md @@ -42,7 +42,7 @@ check and proves a missing, renamed or extra call is reported. -61 calls in 9 families, counted from the Node table. Calls present: Node 61, Python 61, Python `aio` 61, CLI 61, MCP 61 (34 in `work`, 27 admin only). +76 calls in 10 families, counted from the Node table. Calls present: Node 76, Python 76, Python `aio` 76, CLI 76, MCP 76 (38 in `work`, 38 admin only). Argument names that are Python keywords: none. @@ -53,24 +53,30 @@ Front-end problems: none. | `project.init` | project, folder | timeout, on_progress() | yes | yes | yes | yes (--progress) | admin | | `project.list` | ?store | page_size, page_pos | yes | yes | yes | yes | yes | | `project.get` | project | - | yes | yes | yes | yes | yes | +| `project.stats` | project, ?branch, ?commit | - | yes | yes | yes | yes | yes | | `project.export` | project, destination | branch, on_progress() | yes | yes | yes | yes (--progress) | yes | -| `project.remove` | project | - | yes | yes | yes | yes | admin | -| `branch.list` | project | page_size, page_pos | yes | yes | yes | yes | yes | +| `project.delete` | project | - | yes | yes | yes | yes | admin | +| `project.remove_file` | project, path | - | yes | yes | yes | yes | admin | +| `branch.list` | project | page_size, page_pos, page | yes | yes | yes | yes | yes | | `branch.get` | project, branch | - | yes | yes | yes | yes | yes | +| `branch.commits` | project, branch | page, page_size, at_commit, include_storage | yes | yes | yes | yes | yes | +| `branch.get_commit` | project, branch, commit | - | yes | yes | yes | yes | yes | | `branch.fork` | project, name, from_branch, ?from_commit | - | yes | yes | yes | yes | yes | | `branch.merge` | project, name, a, b | message, units | yes | yes | yes | yes | yes | | `branch.publish` | project, branch, ?commit | - | yes | yes | yes | yes | yes | -| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | +| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor, page, from_commit, to_commit, path_prefix, include_attribution, continuation | yes | yes | yes | yes | yes | +| `branch.remove` | project, branch | - | yes | yes | yes | yes | admin | | `commit.list` | project, branch | page_size, page_pos | yes | yes | yes | yes | yes | | `commit.get` | project, branch, commit | - | yes | yes | yes | yes | yes | | `commit.diff` | project, from_commit, to_commit | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | | `file.read` | project, branch, path, ?commit | start_line, end_line | yes | yes | yes | yes | yes | | `file.stat` | project, branch, path, ?commit | - | yes | yes | yes | yes | yes | -| `file.list` | project, branch, path, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.glob` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.grep` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.blame` | project, branch, path, ?commit | from_line | yes | yes | yes | yes | yes | -| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit | yes | yes | yes | yes | yes | +| `file.list` | project, branch, path, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.glob` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.grep` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.revision` | project, branch, path | at_commit, traversal, page, page_size, continuation | yes | yes | yes | yes | yes | +| `file.blame` | project, branch, path, ?commit | from_line, line_count | yes | yes | yes | yes | yes | +| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit, include_attribution, allow_missing, page, page_size | yes | yes | yes | yes | yes | | `file.write` | project, branch, path, content | message, mode | yes | yes | yes | yes (stdin/--from) | yes | | `file.edit` | project, branch, path | old, new, message | yes | yes | yes | yes | yes | | `file.rm` | project, branch, path | message | yes | yes | yes | yes | yes | @@ -78,7 +84,7 @@ Front-end problems: none. | `file.mv` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.cp` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.apply` | project, branch, changes | message | yes | yes | yes | yes (stdin/--from) | yes | -| `sandbox.create` | project, name, ?machine | image | yes | yes | yes | yes | admin | +| `sandbox.create` | project, name, ?machine | image, on_progress() | yes | yes | yes | yes (--progress) | admin | | `sandbox.list` | ?project | page_size, page_pos | yes | yes | yes | yes | yes | | `sandbox.get` | name | page_size, page_pos | yes | yes | yes | yes | yes | | `sandbox.close` | name | force | yes | yes | yes | yes | admin | @@ -91,15 +97,19 @@ Front-end problems: none. | `machine.list` | - | - | yes | yes | yes | yes | admin | | `machine.platforms` | - | - | yes | yes | yes | yes | admin | | `machine.get` | name | - | yes | yes | yes | yes | admin | +| `machine.images` | name | - | yes | yes | yes | yes | admin | +| `machine.load` | name, image | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.prune` | name, ?image | unused | yes | yes | yes | yes | admin | | `machine.status` | name | - | yes | yes | yes | yes | admin | -| `machine.start` | name | - | yes | yes | yes | yes | admin | -| `machine.stop` | name | - | yes | yes | yes | yes | admin | -| `machine.create` | name, platform | cpus, memory, disk, image | yes | yes | yes | yes | admin | +| `machine.start` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.stop` | name | force | yes | yes | yes | yes | admin | +| `machine.create` | name, platform | cpus, memory, disk, image, os_image, on_progress() | yes | yes | yes | yes (--progress) | admin | | `machine.remove` | name | - | yes | yes | yes | yes | admin | | `machine.update` | name | cpus, memory, image, docker, wsl, daemon | yes | yes | yes | yes | admin | | `store.list` | - | - | yes | yes | yes | yes | admin | | `store.get` | name | - | yes | yes | yes | yes | admin | | `store.status` | name | - | yes | yes | yes | yes | admin | +| `store.stats` | name | - | yes | yes | yes | yes | admin | | `store.create` | name | - | yes | yes | yes | yes | admin | | `store.connect` | name, url | password, synchronous_commit | yes | yes | yes | yes | admin | | `store.attach` | name, machine | url, password | yes | yes | yes | yes | admin | @@ -107,12 +117,15 @@ Front-end problems: none. | `store.push` | name, machine | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.pull` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.remove` | name | - | yes | yes | yes | yes | admin | +| `store.compact` | name | - | yes | yes | yes | yes | admin | | `store.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.connect` | name, url | key, region, part_size | yes | yes | yes | yes | admin | | `backup.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.list` | ?bucket | - | yes | yes | yes | yes | admin | | `backup.create` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | | `backup.restore` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `logs.list` | - | op, project, client, tag, outcome, since, cursor, limit | yes | yes | yes | yes | admin | +| `logs.reset` | - | - | yes | yes | yes | yes | admin | diff --git a/docs/architecture/api/sdk/store.md b/docs/architecture/api/sdk/store.md index d2a45bb3fc..f29787e1cd 100644 --- a/docs/architecture/api/sdk/store.md +++ b/docs/architecture/api/sdk/store.md @@ -130,7 +130,14 @@ leases are ignored, not removed password reference is resolved here and the server dialed once; a PostgreSQL Store is never `down` (an unreachable server shows in `server`) ([`status.rs`](../../../../control/crates/layerfs-store/src/status.rs)). It is -the call an `Unresolved` refusal names. Errors: `NotFound`; `InvalidInput` (a +the call an `Unresolved` refusal names. When a compaction runs on a sqlite +Store, `op` is read from `.compact-op`; on a PostgreSQL Store it is +the Store-side `gc_op` row with its `gc_intent`, read by one status query +with no Store session and no artifact read. Either way `state` is `busy` +while the owner lives and `unresolved` once it is gone, `uncertain` stays +the record's own fact, and `phases` names the backend's order (`mark`, +`build`, `verify`, `install` on sqlite; `mark`, `build`, `swap`, `cleanup` +on PostgreSQL). Errors: `NotFound`; `InvalidInput` (a record of unknown kind); `Unavailable` (a 1.1.0 Store whose machine is not known; `layerfs import-env` fixes it). diff --git a/docs/architecture/engine/store/postgresql.md b/docs/architecture/engine/store/postgresql.md index 688c129bbd..806926d1e8 100644 --- a/docs/architecture/engine/store/postgresql.md +++ b/docs/architecture/engine/store/postgresql.md @@ -70,7 +70,7 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`. | `Light` check | A Sandbox daemon's writer, Workspace and control readers, and `attach_read_only`; a hand-changed column fails the statement that uses it instead | [`bootstrap.rs`](../../../../engine/crates/layerfs-store-service/src/bootstrap.rs), [`owner.rs`](../../../../engine/crates/layerfs-daemon/src/application/owner.rs) | | `binding(selection)` | The binding key a schema records, read in one read-only session, so a home that did not create it can open it | [`store/postgresql.rs`](../../../../engine/crates/layerfs-persistence/src/store/postgresql.rs) | | `attach_read_only(selection)` | One read-only session under the recorded binding, returned for the caller to compare with its own | same | -| `postgresql_access(url, schema)` | Server version, one round trip's latency, `current_user`, whether the schema exists, and each privilege the role lacks: the schema's `USAGE`, the database's `TEMP` (merges stage in temporary tables), each table's `SELECT`/`INSERT`/`UPDATE`/`DELETE` and each sequence's `USAGE` | [`access.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/access.rs) | +| `postgresql_access(url, schema)` | Server version, one round trip's latency, `current_user`, whether the schema exists, and each privilege the role lacks: the schema's `USAGE` and `CREATE`, the database's `CREATE` (compaction creates the scratch schema) and `TEMP` (merges stage in temporary tables; compaction reads its visited set from them), `pg_read_all_stats` (compaction reads the recorded owner's `pg_stat_activity` row and calls `pg_terminate_backend`), each table's `SELECT`/`INSERT`/`UPDATE`/`DELETE` and each sequence's `USAGE` | [`access.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/access.rs) | | `postgresql_status(url, schemas)` | Server version, one round trip's latency and each named schema's size | [`status.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/status.rs) | `access` and `status` open one connection for the call and close it, run @@ -78,6 +78,11 @@ read-only queries only and never retry. An authentication or unknown database error is `Denied`; any other connection failure `Unreachable`. The session keeps the frozen policy and server version the open check read, so `PackPersistence::policy` costs no round trip afterwards. +`compaction_missing(session)` applies the same five Store-wide checks to the +compaction session in one read-only round trip, before the intent, the mark +and the `lfs_gc_` scratch schema; the first privilege the role lacks is +`Unsupported`, naming it (AC-44). It is reachable from a session because a +compaction holds one already and never retains the URL. `PostgresSelection`'s `Debug` redacts the URL. ## Session diff --git a/docs/architecture/engine/store/testing.md b/docs/architecture/engine/store/testing.md index f57473ef28..d7c6ee0d9e 100644 --- a/docs/architecture/engine/store/testing.md +++ b/docs/architecture/engine/store/testing.md @@ -11,6 +11,7 @@ the harness that enforces it before cargo spends a build. | Harness | [`tools/ci/test_store_ext4.sh`](../../../../tools/ci/test_store_ext4.sh) | | Suites | `layerfs-persistence`, `layerfs-store-service`, `layerfs-store`, and the `store_*` targets of `layerfs-api` | | Guard | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | +| CI | [`.github/workflows/store-ext4.yml`](../../../../.github/workflows/store-ext4.yml) | | Related | [Store](README.md), [SQLite](sqlite.md), [PostgreSQL](postgresql.md), [Diagnostics](diagnostics.md) | ## Why the tests need ext4 @@ -132,17 +133,146 @@ LAYERFS_TEST_POSTGRES_URL='host=127.0.0.1 port=55433 user=postgres password=… LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh ``` +### Server and extensions + Use PostgreSQL 17 or newer. The Store schema itself needs 16 or newer for `STORAGE PLAIN` ([PostgreSQL](postgresql.md)), and the PostgreSQL suites are qualified on 17.x, so 17 is the supported version for tests. -Each PostgreSQL test creates and drops its own schema: point the variable at a +The full-mode doctor calls `amcheck`'s `verify_heapam` and `bt_index_check` for +every heap and ready B-tree relation, and it never installs an extension itself +([Diagnostics](diagnostics.md)). Create the contrib extensions in the test +database before a full audit: + +```sql +CREATE EXTENSION IF NOT EXISTS amcheck; +CREATE EXTENSION IF NOT EXISTS pgstattuple; +``` + +```bash +psql "$LAYERFS_TEST_POSTGRES_URL" -c 'CREATE EXTENSION IF NOT EXISTS amcheck;' +``` + +`amcheck` is the required one: without it a full audit reports each relation as +`store.database.relations=Unknown`, because no supported function exists, and +`store.database.relations.summary` is `Unknown` too. That is missing coverage, +not corruption, but the PostgreSQL compaction tests assert that a rebuilt Store +audits cleanly with no `Fail` or `Unknown` observation. On a fresh cluster that +never created the extension, the five `clean()` assertions in +`postgresql_compact.rs` fail with exactly that finding; the +`assert_doctor_clean` assertions in `postgresql_compact_fixtures.rs` depend on +it too. `pgstattuple` is the companion contrib module; the current diagnostic +calls only `amcheck`, but the CI lane creates both so the scratch server +matches the qualified setup. + +The push lane is SQLite-only and does not need any of this. When +`LAYERFS_TEST_POSTGRES_URL` is set, the harness asks `psql` whether the +extensions exist and warns with the `CREATE EXTENSION` command when `amcheck` +is missing; an absent `psql` or an unreachable server downgrades that to a +message and never fails the run. `LAYERFS_TEST_EXPECT_AMCHECK` (any value) +makes `postgresql_diagnostic` assert `store.database.relations.summary` is +`Pass`; the `store-ext4` full job sets it after creating the extensions. + +### Credentials and TLS + +The PostgreSQL test fixtures and the doctor's admin client connect with +`postgres::Client::connect(…, NoTls)`, not through the Store's TLS policy. The +connection string must therefore carry its own password and must not require a +TLS handshake: use `sslmode=disable` on loopback or a private route, and +`password=…` unless the server uses trust authentication. `sslmode=require` or +`verify-full` is fine for the Store sessions, but the test fixtures reject it. +Do not put a password in a job log; pass the whole connection string through +the environment, as the workflow does with a scratch trust-authenticated +service. + +### The physical doctor's extra servers + +`postgresql_doctor_physical` needs two servers beyond the primary: + +| Variable | What it must be | +| --- | --- | +| `LAYERFS_TEST_POSTGRES_MISSING_URL` | a second database (or server) where `amcheck` is **not** installed; the test asserts that the missing extension is `Unknown` and that the doctor did not install it | +| `LAYERFS_TEST_POSTGRES_REPLICA_URL` | a physical standby of the primary, in recovery (`pg_is_in_recovery()` is true), with the test's schema replicated within 5 s | + +Create the amcheck-free database on the same scratch server, for example: + +```bash +psql "$LAYERFS_TEST_POSTGRES_URL" -c 'CREATE DATABASE layerfs_no_amcheck' +LAYERFS_TEST_POSTGRES_MISSING_URL='host=127.0.0.1 port=5432 user=postgres dbname=layerfs_no_amcheck sslmode=disable' +``` + +Both variables are optional: with the primary URL set, a test that needs one +and finds it unset prints `NOT_RUN: LAYERFS_TEST_POSTGRES_MISSING_URL is unset` +(or the replica variable) and returns. The `store-ext4` full job has no physical +standby and sets neither, so those extra cases print `NOT_RUN` and the target +stays green. Run the target separately with both for the physical coverage. + +Each PostgreSQL test creates and drops its own schema: point the variables at a scratch server, never at a schema that holds a Store you care about, and make sure the server is reachable from the process that runs the tests (inside a Sandbox, `127.0.0.1` is the Sandbox). Without the variable the PostgreSQL test files print `NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset` and check nothing; the SQLite Store suites still run and still need the supported filesystem. +### Compaction tests and per-test bindings + +The T9 PostgreSQL compaction suite (`postgresql_compact.rs`) gives every test +its own catalog binding, `postgresql-compact-`. PostgreSQL advisory locks +are server-wide, not schema-scoped, so two tests that shared a binding would +exclude each other while running in parallel even though each owns its schema. +The per-test binding fix is already in place; keep it when adding a case. Those +tests need only `LAYERFS_TEST_POSTGRES_URL`, `amcheck` and the supported +`LAYERFS_TEST_TMPDIR`, not the physical doctor's two extra URLs. + +## CI + +[`.github/workflows/store-ext4.yml`](../../../../.github/workflows/store-ext4.yml) +runs this harness in the repository's report-only CI. It adds no test +selection: every target it runs is already in the push or nightly lane that +[`tools/ci_tests.py`](../../../../tools/ci_tests.py) chooses; it runs them where +their guard admits a Store. Like the other lanes it is not a required check, an +agent does not wait for it, and a green run is not verification of a change +([AGENTS.md](../../../../AGENTS.md) §4). + +| Job | Trigger | Command | PostgreSQL | +| --- | --- | --- | --- | +| `store ext4 (--quick)` | push to `main` and pull requests touching `core/**`, `engine/**`, `control/**`, `api/crates/**` (not `layerfs-node`), `Cargo.*`, `.config/**`, `tools/ci/**`, `.cargo/**`; a markdown-only change does not run it | `tools/ci/test_store_ext4.sh --quick` | off | +| `store ext4 (--full, PostgreSQL 17)` | `schedule` (04:47 UTC) and manual dispatch | `tools/ci/test_store_ext4.sh --full` | a `postgres:17` service container | + +Both jobs set `LAYERFS_TEST_TMPDIR=/mnt/layerfs-store-tests`. On the +`ubuntu-24.04` runner `/mnt` is a separate ext4 data disk and `/tmp` is tmpfs; +the prepare step creates the directory, hands it to the runner user (it belongs +to `root` on the image) and asserts the filesystem with `findmnt`, so a runner +image without an ext4 `/mnt` fails in that step instead of as a guard refusal +inside a suite. The harness then sets `TMPDIR`, `CARGO_TARGET_DIR` and +`CARGO_HOME` itself, as it does locally. + +The scheduled job sets `LAYERFS_TEST_POSTGRES_URL` to +`host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=disable` +against that scratch service container, with trust authentication, so every +suite runs its PostgreSQL cases. Each PostgreSQL test creates and drops its own +schema, nothing outlives the run, and the runner reaches the container's +published port on `127.0.0.1`. The push job leaves the variable unset, so its +PostgreSQL cases print `NOT_RUN` and only the SQLite suites run. + +Before the suites, the full job installs `postgresql-client` and creates +`amcheck` and `pgstattuple` in the service container's database, then sets +`LAYERFS_TEST_EXPECT_AMCHECK=1` so `postgresql_diagnostic` asserts the +full-mode relation summary is `Pass`. A server that did not take the extension +fails in that step with its name, instead of as five `Unknown` findings inside +the compaction suite. The job also runs `postgresql_doctor_physical`. It needs +`LAYERFS_TEST_POSTGRES_MISSING_URL` and `LAYERFS_TEST_POSTGRES_REPLICA_URL` for +its extra cases; the full job provides neither, so those cases print `NOT_RUN` +and the target stays green. Run it separately with both for that coverage +(above). + +The harness moves `CARGO_HOME` and `CARGO_TARGET_DIR` onto +`LAYERFS_TEST_TMPDIR`, so these jobs do not restore the checkout-scoped +`Swatinem/rust-cache` the other workflows use: it would restore a `target/` +that the harness never reads. They install `mold` as the fast lane's `test` job +does, because they link the same test profile from a cold cache, and carry a +60-minute (`--quick`) or 120-minute (`--full`) job timeout. + ## Troubleshooting | Symptom | Cause | Fix | @@ -151,6 +281,9 @@ the SQLite Store suites still run and still need the supported filesystem. | `… is on 'tmpfs', which cannot carry a Store.` | `/tmp` (or another tmpfs/overlay) was chosen | pick ext4, xfs or btrfs on Linux, apfs or hfs on macOS | | a suite fails with `filesystem 0x… at … does not carry SQLite locks` | `TMPDIR` or `CARGO_TARGET_DIR` escaped the harness (exported by hand) | let the harness set both; do not override them to a scratch directory | | `NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset` | no PostgreSQL server | expected; set the variable to run those cases | +| `store.database.relations=Unknown` and a compaction test fails on `clean` | `amcheck` was never created in the test database | `CREATE EXTENSION IF NOT EXISTS amcheck;`, then re-run; the harness warns when `psql` can see it is missing | +| `NOT_RUN: LAYERFS_TEST_POSTGRES_MISSING_URL is unset` (or `…_REPLICA_URL`) from `postgresql_doctor_physical` | the physical target's extra cases need an amcheck-free database and a physical standby; without them those cases skip | expected in the full sweep; set both (see [the physical doctor's extra servers](#the-physical-doctors-extra-servers)) and run that target separately for the physical coverage | +| `error: TLS support is disabled` or psql prompts for a password | the fixtures use `NoTls` and read the conninfo only | put `password=…` in the connection string and use `sslmode=disable` on a private route | | `fatal: unable to access 'https://github.com/…/fuser.git'` or a TLS error | the pinned `fuser` git dependency could not be fetched | the harness sets `CARGO_NET_GIT_FETCH_WITH_CLI=true` and `GIT_SSL_NO_VERIFY=true`; check egress to `github.com` | | `has mode 0755 … fix with chmod 700` | a left-over Store directory with group/other access | remove the named directory; the guard never changes permissions | | `Updating crates.io index` stays silent for minutes | the sandbox network can stall cargo's multiplexed sparse-index connection | re-run with `CARGO_HTTP_MULTIPLEXING=false`; the download cache resumes where it stopped | diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs index 417d654060..488b0fee19 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs @@ -1,11 +1,13 @@ //! What a role may do in a Store's schema, for `store.attach`'s check of a -//! machine's connection: the server's version, one round trip's latency, -//! the role, and each privilege a Sandbox's writes need that it lacks. One -//! connection opened for the call and closed after it, read-only queries -//! only, no retry. -use crate::backend::postgresql::session::connection::Connector; +//! machine's connection and for a PostgreSQL compaction's preflight: the +//! server's version, one round trip's latency, the role, and each privilege +//! the Store's writes and compaction need that it lacks. One connection +//! opened for the call and closed after it, read-only queries only, no retry. +use crate::backend::postgresql::session::connection::{Connector, Session}; use crate::backend::postgresql::status::PostgresStatusError; +use crate::backend::records::{BackendError, Param}; use crate::PostgresSelection; +use layerfs_history::error::HistoryResult; use std::time::{Duration, Instant}; /// What the server answered for a role and a schema. @@ -20,18 +22,26 @@ pub struct PostgresAccess { /// Whether the database has the schema. pub schema: bool, /// Each privilege the role lacks, as ` on `, in a - /// fixed order: the schema's `USAGE`, the database's `TEMP`, then each - /// table's `SELECT`, `INSERT`, `UPDATE`, `DELETE` and each sequence's - /// `USAGE`. Empty when the role has them all (or there is no schema). + /// fixed order: the schema's `USAGE` and `CREATE`, the database's + /// `CREATE` and `TEMP`, `pg_read_all_stats`, then each table's `SELECT`, + /// `INSERT`, `UPDATE`, `DELETE` and each sequence's `USAGE`. Empty when + /// the role has them all (or there is no schema). pub missing: Vec, } -/// The role, the schema's `USAGE` (NULL without the schema) and the -/// database's `TEMP` (a merge stages in temporary tables). +/// The role, the schema's `USAGE` and `CREATE` (NULL without the schema), the +/// database's `CREATE` (a compaction creates the `lfs_gc_` scratch +/// schema) and `TEMP` (a merge stages in temporary tables, and a compaction +/// reads its visited set from them), and `pg_read_all_stats` (a compaction +/// reads another backend's `pg_stat_activity` row). const ROLE: &str = "SELECT current_user::text, \ CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = $1::text) \ THEN has_schema_privilege($1::text, 'USAGE') END, \ - has_database_privilege(current_database(), 'TEMP')"; + CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = $1::text) \ + THEN has_schema_privilege($1::text, 'CREATE') END, \ + has_database_privilege(current_database(), 'CREATE'), \ + has_database_privilege(current_database(), 'TEMP'), \ + pg_has_role('pg_read_all_stats', 'MEMBER')"; /// Each table and sequence privilege of the schema the role lacks. const MISSING: &str = "SELECT p.privilege || ' on ' || $1::text || '.' || c.relname \ @@ -44,6 +54,49 @@ const MISSING: &str = "SELECT p.privilege || ' on ' || $1::text || '.' || c.reln ELSE has_table_privilege(c.oid, p.privilege) END \ ORDER BY c.relkind DESC, c.relname, p.at"; +/// The five Store-wide privileges a compaction needs before it names or +/// creates its scratch schema, in the order [`compaction_missing`] reports a +/// missing one: the schema's `USAGE` and `CREATE`, the database's `CREATE` +/// and `TEMP`, and `pg_read_all_stats`. `CREATE` on the schema is the +/// `ALTER TABLE ... SET SCHEMA` of Phase B; `CREATE` on the database is the +/// scratch schema itself; `pg_read_all_stats` is the recorded owner's +/// `pg_stat_activity` row and its `pg_terminate_backend` proof. +const COMPACTION: &str = "SELECT \ + has_schema_privilege($1::text, 'USAGE')::int, \ + has_schema_privilege($1::text, 'CREATE')::int, \ + has_database_privilege(current_database(), 'CREATE')::int, \ + has_database_privilege(current_database(), 'TEMP')::int, \ + pg_has_role('pg_read_all_stats', 'MEMBER')::int"; + +/// The privilege named for each [`COMPACTION`] column, in its order. +const COMPACTION_NAMES: [&str; 5] = [ + "USAGE on the Store schema", + "CREATE on the Store schema", + "CREATE on the database", + "TEMP on the database", + "pg_read_all_stats", +]; + +/// The first Store-wide privilege `session`'s role lacks for a PostgreSQL +/// compaction, or `None` when it has them all: one read-only round trip, +/// taken before the mark and before the `lfs_gc_` scratch schema is +/// named or created. The caller reports the returned name as `Unsupported` +/// (AC-44). +pub(crate) fn compaction_missing(session: &Session) -> HistoryResult> { + session.run(false, |tx| { + let rows = tx.query(COMPACTION, vec![Param::Text(tx.schema.to_owned())])?; + let Some(row) = rows.first() else { + return Err(BackendError::Integrity.into()); + }; + for (at, privilege) in COMPACTION_NAMES.iter().enumerate() { + if row.get::(at)? == 0 { + return Ok(Some(*privilege)); + } + } + Ok(None) + }) +} + /// Reads what the role `url` logs in as may do in `schema`. pub fn access(url: &str, schema: &str) -> Result { let connector = Connector::new(&PostgresSelection::new(url, "public")) @@ -67,14 +120,26 @@ pub fn access(url: &str, schema: &str) -> Result = row.try_get(1).map_err(failed)?; - let temp: bool = row.try_get(2).map_err(failed)?; + let create_schema: Option = row.try_get(2).map_err(failed)?; + let create_database: bool = row.try_get(3).map_err(failed)?; + let temp: bool = row.try_get(4).map_err(failed)?; + let read_all_stats: bool = row.try_get(5).map_err(failed)?; let mut missing = Vec::new(); if usage == Some(false) { missing.push(format!("USAGE on schema {schema}")); } + if create_schema == Some(false) { + missing.push(format!("CREATE on schema {schema}")); + } + if !create_database { + missing.push("CREATE on the database".to_owned()); + } if !temp { missing.push("TEMP on the database".to_owned()); } + if !read_all_stats { + missing.push("pg_read_all_stats".to_owned()); + } if usage == Some(true) { for row in client.query(MISSING, &[&schema]).map_err(failed)? { missing.push(row.try_get(0).map_err(failed)?); diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs index a359f6cd6f..0ae5ebc265 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs @@ -11,6 +11,7 @@ //! crash (or a lost COMMIT reply) is judged by the next `store.compact`: //! generation `G` removes the artifact, `G + 1` resumes cleanup. +use super::access::compaction_missing; use super::session::{ connection::Session, transaction::{Mode, Transaction}, @@ -20,13 +21,16 @@ use crate::backend::sqlite::compact::{ CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, CompactResidue, CompactZeroReason, }; +use crate::compact_metrics::{ + self, CompactOutcome, CompactPhase, CompactRefusal, CompactRunObservation, +}; use crate::history::GcTuple; use crate::{MarkResult, PostgresHandles}; use layerfs_history::error::{HistoryError, HistoryResult}; use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; use layerfs_history::HistoryCatalog; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; /// The §3 rebuild threshold: rebuild when dead * 10 >= stored. const THRESHOLD: u64 = 10; @@ -68,6 +72,113 @@ struct Swapped { dead: DeadTotals, } +/// The phase clock and §11 values one measured PostgreSQL compaction observes. +/// +/// Phase M times the intent, the mark and the wholly-dead-pack decision, Phase +/// A the scratch copy, Phase B the swap, and Phase C the cleanup — also when a +/// later call resumes an installed `G + 1` leftover. The `pg_` phase values +/// keep these distinct from the SQLite §7 phases in the process totals. +struct RunMetrics { + started: Instant, + phase: Option<(CompactPhase, Instant)>, + phase_nanos: Vec<(CompactPhase, u64)>, + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + kept_waste: CompactKeptWaste, + residue_bytes: u64, + leftover_settled: bool, +} + +impl RunMetrics { + fn start() -> Self { + Self { + started: Instant::now(), + phase: None, + phase_nanos: Vec::new(), + generation_before: 0, + generation_after: 0, + dead_bytes_bound: 0, + dead_bytes_refined: 0, + kept_waste: CompactKeptWaste::default(), + residue_bytes: 0, + leftover_settled: false, + } + } + + /// Ends the current phase and starts `phase`; a phase is inclusive wall + /// time, never an exclusive CPU decomposition. + fn begin_phase(&mut self, phase: CompactPhase) { + self.end_phase(); + self.phase = Some((phase, Instant::now())); + } + + fn end_phase(&mut self) { + if let Some((phase, started)) = self.phase.take() { + self.phase_nanos.push(( + phase, + u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), + )); + } + } + + /// Records this run once, whatever its result. The outcome and refusal + /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted + /// without changing the error the caller receives. + fn finish(mut self, result: &HistoryResult) { + self.end_phase(); + let total_nanos = u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let (outcome, refusal) = match result { + Ok(receipt) => { + // A resumed `G + 1` round reports the original run's + // generation, so the receipt is the authority here. + self.generation_before = receipt.generation_before; + self.generation_after = receipt.generation_after; + self.dead_bytes_bound = receipt.dead_bytes_bound; + self.dead_bytes_refined = receipt.dead_bytes_refined; + self.kept_waste = receipt.kept_waste.clone(); + self.residue_bytes = receipt.residue.bytes; + let outcome = match receipt.path { + CompactPath::Rebuild => CompactOutcome::Rebuild, + CompactPath::None => CompactOutcome::None, + }; + (outcome, None) + } + Err(error) => { + if !error.unknown() { + self.generation_after = self.generation_before; + } + ( + if error.unknown() { + CompactOutcome::Uncertain + } else { + CompactOutcome::Failed + }, + match error { + HistoryError::Busy | HistoryError::BusyOp(_) => Some(CompactRefusal::Busy), + HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), + _ => None, + }, + ) + } + }; + compact_metrics::record_run(&CompactRunObservation { + outcome, + generation_before: self.generation_before, + generation_after: self.generation_after, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + kept_waste: self.kept_waste, + residue_bytes: self.residue_bytes, + phase_nanos: self.phase_nanos, + total_nanos, + leftover_settled: self.leftover_settled, + refusal, + }); + } +} + fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -699,6 +810,7 @@ fn settle_leftover( handles: &PostgresHandles, session: &Arc, schema: &str, + metrics: &mut RunMetrics, ) -> HistoryResult> { let Some(op) = read_op(session, schema)? else { return Ok(None); @@ -720,6 +832,7 @@ fn settle_leftover( } } } + metrics.begin_phase(CompactPhase::PostgresCleanup); if let Some(artifact) = &op.artifact { drop_scratch(session, artifact)?; } @@ -730,13 +843,18 @@ fn settle_leftover( ))?; Ok(()) })?; + metrics.end_phase(); + metrics.leftover_settled = true; return Ok(None); } if current != op.generation + 1 { return Err(HistoryError::Integrity("compaction op generation")); } + metrics.begin_phase(CompactPhase::PostgresCleanup); let op_id = compaction_intent(session, schema)?; cleanup(handles, session, schema, op.dead, op.totals_applied, op_id)?; + metrics.end_phase(); + metrics.leftover_settled = true; Ok(Some(report( CompactPath::Rebuild, u64::try_from(op.generation).unwrap_or(0), @@ -1153,7 +1271,9 @@ fn round( session: &Arc, schema: &str, request: &CompactRequest, + metrics: &mut RunMetrics, ) -> HistoryResult { + metrics.begin_phase(CompactPhase::PostgresMark); let (mark, tuple) = handles.history.mark_gc()?; load_visited(session, schema, &mark)?; let stored = u64::try_from(total_stored(session, schema)?).unwrap_or(0); @@ -1176,6 +1296,7 @@ fn round( }; let rebuild = stored > 0 && dead_bytes.saturating_mul(THRESHOLD) >= stored; if !rebuild { + metrics.end_phase(); clear_round(session, schema)?; let _ = handles.history.gc_release(&request.op_id); return Ok(report( @@ -1194,6 +1315,7 @@ fn round( mark.history().len() as u64, )); } + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterMark, @@ -1201,7 +1323,9 @@ fn round( schema, "mark", )?; + metrics.begin_phase(CompactPhase::PostgresBuild); let mut built = build(session, schema, &scratch, &mark, &tuple)?; + metrics.end_phase(); built.dead.logical = mark_dead.logical; stop( request, @@ -1210,7 +1334,9 @@ fn round( schema, "build", )?; + metrics.begin_phase(CompactPhase::PostgresSwap); let swapped = swap(handles, session, schema, &tuple, &mark, &built)?; + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterSwap, @@ -1219,6 +1345,7 @@ fn round( "swap", )?; let logical_discarded = u64::try_from(swapped.dead.logical).unwrap_or(0); + metrics.begin_phase(CompactPhase::PostgresCleanup); cleanup( handles, session, @@ -1227,6 +1354,7 @@ fn round( false, Some(request.op_id), )?; + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterCleanup, @@ -1351,10 +1479,29 @@ impl PostgresHandles { /// deletes only the persisted dead set. The caller keeps its session; the /// Store is usable at `G + 1` from the swap on. pub fn compact(&self, request: &CompactRequest) -> HistoryResult { + let mut metrics = RunMetrics::start(); + let result = self.compact_measured(request, &mut metrics); + metrics.finish(&result); + result + } + + /// The measured body of [`Self::compact`]; `metrics` observes it. + fn compact_measured( + &self, + request: &CompactRequest, + metrics: &mut RunMetrics, + ) -> HistoryResult { let session = self.storage.session.clone(); let schema = session.settings.schema.clone(); - let _ = read_generation(&session, &schema)?; - if let Some(receipt) = settle_leftover(self, &session, &schema)? { + // AC-44: a restricted role is refused before the intent, the mark and + // the scratch schema; the missing privilege is named, never a raw + // `42501` from whichever statement happened to run first. + if let Some(missing) = compaction_missing(&session)? { + return Err(HistoryError::Unsupported(missing)); + } + let generation = read_generation(&session, &schema)?; + metrics.generation_before = u64::try_from(generation).unwrap_or(0); + if let Some(receipt) = settle_leftover(self, &session, &schema, metrics)? { return Ok(receipt); } let intent = GcIntent::operation( @@ -1372,7 +1519,7 @@ impl PostgresHandles { let result = (|| { let rank = effective_sync_rank(&session)?; standby_precheck(&session, rank)?; - round(self, &session, &schema, request) + round(self, &session, &schema, request, metrics) })(); if result.is_err() { let _ = self.history.gc_release(&request.op_id); diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs index c9b7d7a7a2..69eeb53eb6 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs @@ -2,7 +2,12 @@ //! its version, one round trip's latency and the size of each named schema //! (tables and materialized views with their indexes and TOAST), on one //! connection opened for the call and closed after it, with no retry. +//! +//! It also reads a Store's compaction op — the `gc_op` row with its +//! `gc_intent` — for `store.status` (T10.5): one query, with no Store open, +//! schema change, artifact read or scratch-schema read. use crate::backend::postgresql::session::connection::Connector; +use crate::report::{CompactStatus, POSTGRES_COMPACT_PHASES}; use crate::PostgresSelection; use std::time::{Duration, Instant}; @@ -80,3 +85,182 @@ pub fn status(url: &str, schemas: &[String]) -> Result, + /// The owning backend's `backend_start`, when the row records one. + pub owner_backend_start: Option, + /// The owning process runtime `gc_intent` recorded, when it left one. + pub owner_runtime: Option, + /// The owning process boot id `gc_intent` recorded, when it left one. + pub owner_boot: Option, + /// The `gc_intent` row's own state (`live`, `uncertain` or `done`), when + /// it is still there; a crash leaves `live`, so this is evidence, not + /// liveness. + pub intent_state: Option, + /// The mark's history serial, when recorded. + pub mark_serial: Option, + /// The mark's next pack id, when recorded. + pub mark_pack_id: Option, + /// The mark's live-heads digest, when recorded. + pub mark_digest: Option>, + /// True while the row remains: the run did not settle, so its outcome is + /// unknown. `store.status` reports `busy` while the owner lives and + /// `unresolved` once it is gone. + pub uncertain: bool, + /// True when the recorded owner backend is still alive and not `idle`, + /// the same §9 test a resolving compaction applies. + pub alive: bool, +} + +impl PostgresCompactOpStatus { + /// The `store.status` projection of this operation: `busy` while the + /// owner lives, `unresolved` once it is gone. `uncertain` stays true for + /// every remaining row, live or leftover; `state` separates the two. The + /// phases are PostgreSQL's §8 ones: `mark`, `build`, `swap`, `cleanup`. + #[must_use] + pub fn status(&self) -> CompactStatus { + CompactStatus { + id: format!("op-{}", hex(&self.op_id)), + name: "store.compact", + phase: self.phase.clone(), + started: self.started_ms, + uncertain: self.uncertain, + state: if self.alive { "busy" } else { "unresolved" }, + phases: &POSTGRES_COMPACT_PHASES, + } + } +} + +/// `gc_op` plus its compaction `gc_intent` and the owner's live state, in one +/// query on one status connection. Returns `None` when the Store has no +/// compaction row (idle), or when a Store written before the GC tables has no +/// `gc_op` at all. No Store open, schema mutation, artifact or scratch read +/// happens here. +pub fn compact_status( + url: &str, + schema: &str, +) -> Result, PostgresStatusError> { + let connector = Connector::new(&PostgresSelection::new(url, schema)) + .map_err(|_| PostgresStatusError::Unreachable("the URL does not parse".to_owned()))?; + let mut client = connector + .connect_raw() + .map_err(|error| match error.code() { + Some(code) if code.code().starts_with("28") || code.code() == "3D000" => { + PostgresStatusError::Denied(error.to_string()) + } + _ => PostgresStatusError::Unreachable(error.to_string()), + })?; + let table = quote_ident(schema); + let sql = format!( + "SELECT o.generation,o.phase,o.started_ms,o.pid,o.backend_start,o.artifact,\ + o.mark_serial,o.mark_pack_id,o.mark_digest,\ + i.op_id,i.owner_runtime,i.owner_boot,i.state,\ + EXISTS (SELECT 1 FROM pg_stat_activity a \ + WHERE a.datname=current_database() AND a.pid=o.pid \ + AND a.backend_start::text=o.backend_start \ + AND a.state IS DISTINCT FROM 'idle') \ + FROM {table}.gc_op o \ + LEFT JOIN LATERAL (SELECT op_id,owner_runtime,owner_boot,state \ + FROM {table}.gc_intent WHERE kind='compaction' \ + ORDER BY updated_ms LIMIT 1) i ON TRUE \ + WHERE o.id=1" + ); + let rows = match client.query(&sql, &[]) { + Ok(rows) => rows, + // A Store opened before the GC tables has no row to read: idle. + Err(error) + if matches!( + error.code().map(postgres::error::SqlState::code), + Some("42P01" | "3F000") + ) => + { + let _ = client.close(); + return Ok(None); + } + Err(error) => return Err(PostgresStatusError::Failed(error.to_string())), + }; + let Some(row) = rows.first() else { + let _ = client.close(); + return Ok(None); + }; + let failed = |error: postgres::Error| PostgresStatusError::Failed(error.to_string()); + let intent: Option> = row.try_get(9).map_err(failed)?; + let artifact: Option = row.try_get(5).map_err(failed)?; + let op = PostgresCompactOpStatus { + op_id: op_identity(intent, artifact.as_deref()), + generation: row.try_get(0).map_err(failed)?, + phase: row.try_get(1).map_err(failed)?, + started_ms: row.try_get(2).map_err(failed)?, + owner_pid: row.try_get(3).map_err(failed)?, + owner_backend_start: row.try_get(4).map_err(failed)?, + owner_runtime: row.try_get(10).map_err(failed)?, + owner_boot: row.try_get(11).map_err(failed)?, + intent_state: row.try_get(12).map_err(failed)?, + mark_serial: row.try_get(6).map_err(failed)?, + mark_pack_id: row.try_get(7).map_err(failed)?, + mark_digest: row.try_get(8).map_err(failed)?, + // A remaining row is an unsettled run: the record's own fact. It is + // never inferred from the owner's liveness. + uncertain: true, + alive: row.try_get(13).map_err(failed)?, + }; + let _ = client.close(); + Ok(Some(op)) +} + +/// The `lfs_gc_` scratch name `gc_op.artifact` records carries the +/// operation identity, so a row whose `gc_intent` was already released still +/// projects a stable `op.id`. +fn op_identity(intent: Option>, artifact: Option<&str>) -> [u8; 16] { + if let Some(bytes) = intent { + if let Ok(op) = <[u8; 16]>::try_from(bytes.as_slice()) { + return op; + } + } + artifact + .and_then(|name| name.strip_prefix("lfs_gc_")) + .and_then(unhex) + .unwrap_or([0; 16]) +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn unhex(text: &str) -> Option<[u8; 16]> { + if text.len() != 32 { + return None; + } + let mut out = [0u8; 16]; + for (index, byte) in out.iter_mut().enumerate() { + *byte = u8::from_str_radix(text.get(index * 2..index * 2 + 2)?, 16).ok()?; + } + Some(out) +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs index 5a02a14703..e3f07fe462 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs @@ -45,6 +45,11 @@ const OP_SUFFIX: &str = ".compact-op"; const COMPACT_SUFFIX: &str = ".compact"; /// The §6.3 visited scratch database. const VISITED_SUFFIX: &str = ".compact-visited"; +/// Upper bound on the rollback-journal header: SQLite pads it to the +/// filesystem's sector size and writes a header for each transaction. Two +/// sectors cover the prune and rewrite headers. +const JOURNAL_HEADER_BOUND: u64 = 8192; + /// The full SQLite artifact set of §9, without the variable `-mj*` journals. const ARTIFACT_SUFFIXES: [&str; 7] = [ COMPACT_SUFFIX, @@ -126,6 +131,12 @@ pub enum CompactFailpoint { AfterSeal, /// Inside the build, before the visited scratch or the copy is written. BeforeCopy, + /// Inside the build, immediately before `VACUUM INTO` starts the copy. + /// SQLite runs that copy as one uninterruptible call, so a real-process + /// `SIGKILL` cannot land inside it; the real-kill arm stops here with the + /// visited scratch present and the staged file not yet created. The + /// in-process `DuringCopy` arm keeps the torn-target state. + BeforeVacuum, /// Inside the build, after `VACUUM INTO` produced `.compact` and /// before the copy is pruned: the state a crash during the copy leaves. DuringCopy, @@ -164,6 +175,7 @@ impl CompactFailpoint { match self { Self::BeforeSeal | Self::BeforeSealRead | Self::AfterSeal => "mark", Self::BeforeCopy + | Self::BeforeVacuum | Self::DuringCopy | Self::AfterBuild | Self::CapacityAtBuild @@ -739,6 +751,31 @@ fn live_page_metrics(session: &Arc) -> HistoryResult<(u64, u64)> { .map_err(Into::into) } +/// The rollback-journal upper bound of one transaction on a `pages`-page file: +/// one record per modified page (`page_size + 8` bytes: page number, page image +/// and checksum) plus the header allowance. A page is journaled at most once +/// per transaction, so the staged copy's page count bounds both the prune and +/// the rewrite journal, and the bound never under-reports the observed file. +const fn journal_bound_bytes(pages: u64, page_size: u64) -> u64 { + pages + .saturating_mul(page_size.saturating_add(8)) + .saturating_add(JOURNAL_HEADER_BOUND) +} + +/// The staged file's page count and page size, before the prune frees pages. +fn file_page_extent(connection: &Connection) -> HistoryResult<(u64, u64)> { + let pages: i64 = connection + .query_row("PRAGMA page_count", params![], |row| row.get(0)) + .map_err(db_error)?; + let page: i64 = connection + .query_row("PRAGMA page_size", params![], |row| row.get(0)) + .map_err(db_error)?; + Ok(( + u64::try_from(pages).map_err(|_| HistoryError::Integrity("page count"))?, + u64::try_from(page).map_err(|_| HistoryError::Integrity("page size"))?, + )) +} + fn file_page_metrics(connection: &Connection) -> HistoryResult<(u64, u64)> { let free: i64 = connection .query_row("PRAGMA freelist_count", params![], |row| row.get(0)) @@ -855,6 +892,10 @@ CREATE TABLE live_pack (pack_id INTEGER PRIMARY KEY) STRICT, WITHOUT ROWID;", struct Built { target: PathBuf, target_bytes: u64, + /// The staged copy's extent right after `VACUUM INTO`, before the prune. + staged_bytes: u64, + /// The rollback-journal upper bound the staged page count implies. + journal_bound_bytes: u64, visited_bytes: u64, logical: u64, canonical: u64, @@ -864,8 +905,24 @@ struct Built { } impl Built { + /// §11 peak temporary bytes: the run's whole artifact footprint below the + /// rename, as a documented upper bound rather than a sampled peak. + /// + /// The staged copy is the whole sealed Store (`VACUUM INTO`), so it keeps + /// the source extent until the rewrite's final truncate. The rewrite then + /// writes the new Store into SQLite's temp database (the process temp + /// directory, unlinked) while rewriting the staged file in place, and both + /// the prune and the rewrite journal the pages they overwrite. The journal + /// term bounds either journal by one record per staged page + /// (`page_size + 8` bytes each: page number, image and checksum) plus the + /// header allowance, so it never under-reports the observed journal. The + /// sum over-reports the measured 50 GB peak by a few percent; §13 already + /// marks the report an estimate. fn peak_temp_bytes(&self) -> u64 { - self.target_bytes.saturating_add(self.visited_bytes) + self.staged_bytes + .saturating_add(self.journal_bound_bytes) + .saturating_add(self.target_bytes) + .saturating_add(self.visited_bytes) } } @@ -885,19 +942,32 @@ fn table_present(connection: &Connection, name: &str) -> HistoryResult { Ok(found == 1) } +/// The sealed Store and the mark a staged build is measured against. +struct BuildInput<'a> { + facts: &'a SealedFacts, + mark: &'a MarkResult, + tuple: &'a MarkTuple, + packs: &'a BTreeMap, + layout: crate::SqlitePackLayout, + synchronous: i64, +} + /// The §7 steps 2–5: build the staged Store, copy only retained rows, compact /// it, and switch it to WAL. fn build( - facts: &SealedFacts, - mark: &MarkResult, - tuple: &MarkTuple, - packs: &BTreeMap, - layout: crate::SqlitePackLayout, - synchronous: i64, + input: BuildInput<'_>, capacity: bool, request: &CompactRequest, op: &RunOp, ) -> HistoryResult { + let BuildInput { + facts, + mark, + tuple, + packs, + layout, + synchronous, + } = input; let store = &facts.path; let target = compact_path(store); let visited = visited_path(store); @@ -916,6 +986,9 @@ fn build( return Err(io_error(std::io::Error::from_raw_os_error(28))); } + // A real-process crash arm stops here: the visited scratch is written and + // `VACUUM INTO` has not started. See `BeforeVacuum`. + stop_at(request, CompactFailpoint::BeforeVacuum, op, store)?; let source = open_immutable(store)?; let target_text = target .to_str() @@ -928,6 +1001,9 @@ fn build( .map_err(|(_, error)| db_error(error))?; // The copy produced the staged file but has not pruned or verified it. stop_at(request, CompactFailpoint::DuringCopy, op, store)?; + // The staged copy holds the source's whole extent until the final + // `VACUUM` truncates it; §11 counts it in the peak temp. + let staged_bytes = std::fs::metadata(&target).map_err(io_error)?.len(); let connection = Connection::open_with_flags( &target, @@ -942,6 +1018,10 @@ fn build( "PRAGMA journal_mode=DELETE; PRAGMA synchronous={synchronous};" )) .map_err(db_error)?; + // The prune and the rewrite journal the pages they overwrite; one record + // per staged page bounds either journal before the staged file shrinks. + let (staged_pages, page_size) = file_page_extent(&connection)?; + let journal_bound_bytes = journal_bound_bytes(staged_pages, page_size); // The copied published-pack rows may be deleted in the artifact; the live // immutability triggers are restored before the file is verified. // `sqlite_sequence` is the acquisition operation counter: the copy keeps @@ -1160,6 +1240,8 @@ metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", Ok(Built { target, target_bytes, + staged_bytes, + journal_bound_bytes, visited_bytes, logical: u64::try_from(totals.0).unwrap_or(0), canonical: u64::try_from(totals.1).unwrap_or(0), @@ -1169,20 +1251,33 @@ metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", }) } +/// The staged Store and the mark a verification reads it against. +struct VerifyInput<'a> { + store: &'a Path, + built: &'a Built, + mark: &'a MarkResult, + tuple: &'a MarkTuple, + layout: crate::SqlitePackLayout, + acquisition: crate::SqliteAcquisitionSchema, +} + /// The §7 step 6 verification through an `immutable=1` reader: schema identity, /// foreign keys, row counts against the mark, both live-heads digests, the /// allocator tuple and the recomputed totals. `quick_check` and the full doctor /// stay separate commands. fn verify( - store: &Path, - built: &Built, - mark: &MarkResult, - tuple: &MarkTuple, - layout: crate::SqlitePackLayout, - acquisition: crate::SqliteAcquisitionSchema, + input: VerifyInput<'_>, request: &CompactRequest, op: &RunOp, ) -> HistoryResult<()> { + let VerifyInput { + store, + built, + mark, + tuple, + layout, + acquisition, + } = input; let connection = open_immutable(&built.target)?; schema::check( &connection, @@ -1522,6 +1617,30 @@ impl Drop for WriterGuard { } } +/// Set by the real-process crash harness (T12.2's second arm): when present, +/// a reached failpoint connects to this Unix socket, writes one readiness +/// byte and blocks until the harness SIGKILLs the process. Production callers +/// never set it, so a failpoint keeps its in-process behaviour. +const CRASH_GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; + +/// Holds a reached failpoint until the test harness kills this process. The +/// variable being absent is a no-op: the in-process stop applies. +#[cfg(unix)] +fn crash_gate() -> HistoryResult<()> { + use std::io::{Read, Write}; + use std::os::unix::net::UnixStream; + let Some(path) = std::env::var_os(CRASH_GATE_ENV) else { + return Ok(()); + }; + let mut gate = UnixStream::connect(path).map_err(io_error)?; + gate.write_all(b"R").map_err(io_error)?; + gate.flush().map_err(io_error)?; + // The harness never releases the gate: it SIGKILLs this process here. + let mut release = [0u8; 1]; + let _ = gate.read(&mut release); + Ok(()) +} + /// A test-only crash stop: records the phase and leaves the state a crash /// would leave. fn stop_at( @@ -1536,6 +1655,8 @@ fn stop_at( let mut stopped = op.clone(); stopped.phase = point.phase().to_owned(); stopped.write(store)?; + #[cfg(unix)] + crash_gate()?; Err(HistoryError::UnknownOutcome) } @@ -1733,7 +1854,12 @@ impl SqliteHandles { request.owner_boot.as_deref(), ) .with_phase(Some("mark")); - history.gc_register(&intent)?; + if let Err(error) = history.gc_register(&intent) { + // A definite admission refusal started nothing: the run's op + // record must not outlive it as a false leftover (§9). + let _ = std::fs::remove_file(op_path(&store)); + return Err(error); + } // §8: the registration, the mark and the totals read are one // admitted sequence. A definite failure here releases the intent // before the artifact set and the op record go, so the Store is @@ -1812,9 +1938,7 @@ impl SqliteHandles { // a live intent would refuse every later GC operation. The held // writer lane still excludes writers through the install, and a // crash leaves the op record for the next run to settle. - if let Err(error) = history.gc_release(&op_id) { - return Err(error); - } + history.gc_release(&op_id)?; drop(history); drop(storage); let mut guard = WriterGuard::new(lane.into_writer()); @@ -1861,12 +1985,14 @@ impl SqliteHandles { op.write(&store)?; let capacity = request.failpoint == Some(CompactFailpoint::CapacityAtBuild); let built = match build( - &facts, - &mark, - &tuple, - &object_packs, - layout, - synchronous, + BuildInput { + facts: &facts, + mark: &mark, + tuple: &tuple, + packs: &object_packs, + layout, + synchronous, + }, capacity, request, &op, @@ -1891,12 +2017,14 @@ impl SqliteHandles { return Err(error); } if let Err(error) = verify( - &store, - &built, - &mark, - &tuple, - layout, - acquisition_schema, + VerifyInput { + store: &store, + built: &built, + mark: &mark, + tuple: &tuple, + layout, + acquisition: acquisition_schema, + }, request, &op, ) { @@ -2067,7 +2195,7 @@ impl RunMetrics { #[cfg(test)] mod tests { - use super::io_error; + use super::{journal_bound_bytes, io_error, Built}; use layerfs_history::HistoryError; /// §7 step 3 on a bounded filesystem: `ENOSPC`, `EDQUOT` and `EFBIG` are @@ -2089,4 +2217,44 @@ mod tests { let error = io_error(std::io::Error::from_raw_os_error(5)); assert!(matches!(error, HistoryError::Integrity(_)), "{error:?}"); } + + /// §11 `peakTempBytes` on the 50 GB arm (T-G66): the staged copy holds the + /// source extent, the rewrite holds that copy beside the new Store, and the + /// rollback journal is bounded by the staged page count. The receipt must + /// cover the measured 143.1 GB pre-rebuild peak and the 44.6 GB journal the + /// old `new + visited` formula dropped. + #[test] + fn peak_temp_covers_the_staged_copy_and_the_rollback_journal() { + let page_size = 4096; + // 50,862,854,144 B staged copy = the sealed Store's page count. + let staged_pages = 12_417_689; + let staged_bytes = staged_pages * page_size; + let target_bytes = 45_763_203_072; + let visited_bytes = 20_365_312; + let journal = journal_bound_bytes(staged_pages, page_size); + assert_eq!(journal, 50_962_203_848); + let built = Built { + target: std::path::PathBuf::new(), + target_bytes, + staged_bytes, + journal_bound_bytes: journal, + visited_bytes, + logical: 0, + canonical: 0, + stored: 0, + metadata: 0, + free_bytes: 0, + }; + let peak = built.peak_temp_bytes(); + assert_eq!(peak, 147_608_626_376); + assert!(peak >= target_bytes + journal + visited_bytes); + assert!( + peak >= target_bytes + 44_630_503_928 + visited_bytes, + "the bound must cover the 50 GB arm's observed 44.6 GB journal" + ); + assert!( + peak >= 143_100_000_000, + "the bound must cover the measured 143.1 GB sampled 50 GB peak" + ); + } } diff --git a/engine/crates/layerfs-persistence/src/compact_metrics.rs b/engine/crates/layerfs-persistence/src/compact_metrics.rs index c46246c63c..96b42bde6c 100644 --- a/engine/crates/layerfs-persistence/src/compact_metrics.rs +++ b/engine/crates/layerfs-persistence/src/compact_metrics.rs @@ -1,4 +1,5 @@ -//! Bounded metrics and SLO signals for SQLite Store compaction. +//! Bounded metrics and SLO signals for SQLite (§7) and PostgreSQL (§8) +//! Store compaction. //! //! Every metric a real `store.compact` or `store.status` path records goes //! through this module. Label values come from closed enums ([`CompactOutcome`], @@ -17,7 +18,7 @@ use std::cell::RefCell; use std::collections::BTreeMap; use std::sync::{Mutex, OnceLock}; -/// The outcome of one SQLite compaction call. +/// The outcome of one Store compaction call. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] pub enum CompactOutcome { /// A rebuild installed at `G -> G + 1`. @@ -43,7 +44,13 @@ impl CompactOutcome { } } -/// One SQLite phase a run may time (§11: `mark`, `build`, `verify`, `install`). +/// One phase a run may time. +/// +/// SQLite (§7) reports `mark`, `build`, `verify` and `install`. PostgreSQL +/// (§8) reports its own Phase M (`pg_mark`), Phase A (`pg_build`), Phase B +/// (`pg_swap`) and Phase C (`pg_cleanup`). The two backends' values are +/// distinct, so a process total never blends a SQLite phase with a PostgreSQL +/// phase of a different meaning, and every value stays a closed enum string. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] pub enum CompactPhase { /// The §7 step 1 mark and the wholly-dead-pack decision. @@ -54,6 +61,17 @@ pub enum CompactPhase { Verify, /// The install and its settle. Install, + /// PostgreSQL §8 Phase M: the intent, the T7.2 mark and the persisted + /// `gc_dead` set. + PostgresMark, + /// PostgreSQL §8 Phase A: copy the retained payload into the scratch + /// schema. + PostgresBuild, + /// PostgreSQL §8 Phase B: swap `pack`/`object_location` at `G -> G + 1`. + PostgresSwap, + /// PostgreSQL §8 Phase C: delete the persisted dead set and settle the + /// totals, also for a resumed `G + 1` leftover. + PostgresCleanup, } impl CompactPhase { @@ -65,6 +83,10 @@ impl CompactPhase { Self::Build => "build", Self::Verify => "verify", Self::Install => "install", + Self::PostgresMark => "pg_mark", + Self::PostgresBuild => "pg_build", + Self::PostgresSwap => "pg_swap", + Self::PostgresCleanup => "pg_cleanup", } } } diff --git a/engine/crates/layerfs-persistence/src/history/admission.rs b/engine/crates/layerfs-persistence/src/history/admission.rs index f45a48eef0..53e0996e97 100644 --- a/engine/crates/layerfs-persistence/src/history/admission.rs +++ b/engine/crates/layerfs-persistence/src/history/admission.rs @@ -404,19 +404,30 @@ fn get_in(tx: &Transaction<'_>, op_id: &[u8; 16]) -> HistoryResult)>, + shared: bool, +} + #[cfg(feature = "postgresql")] fn keys_for_release( tx: &Transaction<'_>, catalog: CatalogId, intent: &GcIntent, -) -> HistoryResult<(Vec<(u8, Vec)>, bool)> { +) -> HistoryResult { let mut intent = intent.clone(); if intent.scope_kind == GcScopeKind::Branch && intent.project_id.is_none() { intent.project_id = one(tx, BRANCH_STACK, [intent.scope_id.as_slice()], |row| { cell::>(row, 0) })?; } - Ok((advisory_keys(catalog, &intent), intent.kind.shared())) + Ok(ReleaseKeys { + keys: advisory_keys(catalog, &intent), + shared: intent.kind.shared(), + }) } /// Registers one shared (`mount`, `init`, `merge`) or exclusive @@ -453,7 +464,7 @@ pub(crate) fn release(provider: &HistoryProvider, op_id: &[u8; 16]) -> HistoryRe release_in(tx, op_id)?; Ok(Some(keys_for_release(tx, provider.catalog_id, &intent)?)) })?; - if let Some((keys, shared)) = found { + if let Some(ReleaseKeys { keys, shared }) = found { let _ = session_unlock(session, provider.catalog_id, &keys, shared); } Ok(()) diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index 219d7b16e8..fec388af57 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -23,7 +23,9 @@ mod rows; mod transaction; pub use kept::{KeptHistory, KeptNode, KeptNodeId}; pub(crate) use lineage_backfill::backfill_lineage; -pub use objects::{GcTuple, KeptObjects, KeptWaste, MarkResult}; +#[cfg(feature = "postgresql")] +pub use objects::GcTuple; +pub use objects::{KeptObjects, KeptWaste, MarkResult}; pub use provider::HistoryProvider; pub(crate) use rows::{ branch_row as diagnostic_branch, commit_graph_row as diagnostic_commit_graph, diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 0f555b7211..f0ff344f80 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -14,7 +14,8 @@ mod report; pub use backend::postgresql::access::{access as postgresql_access, PostgresAccess}; #[cfg(feature = "postgresql")] pub use backend::postgresql::status::{ - status as postgresql_status, PostgresStatus, PostgresStatusError, + compact_status as postgresql_compact_status, status as postgresql_status, + PostgresCompactOpStatus, PostgresStatus, PostgresStatusError, }; pub use backend::sqlite::acquisition::AcquisitionProvider; pub use backend::sqlite::compact::{ @@ -23,7 +24,8 @@ pub use backend::sqlite::compact::{ }; pub use backend::sqlite::heads::{heads_digest, read_heads}; pub use report::{ - CompactGeneration, CompactReport, CompactReportSource, CompactStatus, SQLITE_COMPACT_PHASES, + CompactGeneration, CompactReport, CompactReportSource, CompactStatus, POSTGRES_COMPACT_PHASES, + SQLITE_COMPACT_PHASES, }; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use backend::sqlite::lifecycle::headroom::STORE_HEADROOM_BYTES; diff --git a/engine/crates/layerfs-persistence/src/report.rs b/engine/crates/layerfs-persistence/src/report.rs index 0c6a1fa80e..becb1eef5c 100644 --- a/engine/crates/layerfs-persistence/src/report.rs +++ b/engine/crates/layerfs-persistence/src/report.rs @@ -13,6 +13,11 @@ use crate::backend::sqlite::compact::{ /// leaves its record at the phase it reached. pub const SQLITE_COMPACT_PHASES: [&str; 4] = ["mark", "build", "verify", "install"]; +/// The PostgreSQL §8 compaction phases, in order: Phase M mark, Phase A +/// build, Phase B swap and Phase C cleanup. These are the values a +/// PostgreSQL `gc_op` row records, distinct from the SQLite §7 phases. +pub const POSTGRES_COMPACT_PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + /// The generation a compaction started at and, when it installed a rebuild, /// the one it left (`none` keeps both equal). #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/engine/crates/layerfs-persistence/tests/compact_metrics.rs b/engine/crates/layerfs-persistence/tests/compact_metrics.rs index 621fcfb8e6..585a5f8088 100644 --- a/engine/crates/layerfs-persistence/tests/compact_metrics.rs +++ b/engine/crates/layerfs-persistence/tests/compact_metrics.rs @@ -8,13 +8,14 @@ mod support; use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; use layerfs_history::HistoryError; -use layerfs_persistence::compact_metrics::{CompactMetrics, CompactMetricsCapture}; +use layerfs_persistence::compact_metrics::CompactMetricsCapture; use layerfs_persistence::{ CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, }; use layerfs_storage::{Storage, StoragePolicy}; use std::path::{Path, PathBuf}; +use support::metrics::{assert_bounded, metric, value}; use support::Temp; const BINDING: &[u8] = b"sqlite-compact-metrics-test-binding"; @@ -80,84 +81,6 @@ fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { object.id() } -/// The value of one sample, or `None` when this run did not emit it. -fn metric(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> Option { - metrics - .samples() - .into_iter() - .find(|sample| { - sample.name == name - && labels - .iter() - .all(|(key, value)| sample.label(key) == Some(*value)) - }) - .map(|sample| sample.value) -} - -fn value(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> u64 { - metric(metrics, name, labels).unwrap_or_else(|| { - panic!( - "no sample {name} {labels:?}; emitted: {:?}", - metrics.samples() - ) - }) -} - -/// Bounded cardinality: every label key is a closed dimension and every value -/// is one the enums name, never a Store name, path, operation id or artifact. -fn assert_bounded(metrics: &CompactMetrics) { - let keys = [ - "outcome", - "code", - "kind", - "reason", - "state", - "objective", - "result", - "phase", - "statistic", - "which", - "part", - ]; - let values = [ - "rebuild", - "none", - "failed", - "uncertain", - "busy", - "unresolved", - "mark", - "build", - "verify", - "install", - "run", - "leftover", - "count", - "sum", - "max", - "before", - "after", - "bound", - "refined", - "dead_in_retained_packs", - "delta_bases", - "inode_blocks", - "residue_free", - "generation", - "pass", - "fail", - ]; - for sample in metrics.samples() { - for (key, label) in &sample.labels { - assert!(keys.contains(key), "non-closed label key {key}"); - assert!( - values.contains(label), - "non-closed label value {label} for {key}" - ); - } - } -} - #[test] fn a_successful_rebuild_emits_phases_outcome_and_report_gauges() { let (_temp, handles, path) = create_store(); diff --git a/engine/crates/layerfs-persistence/tests/diagnostic_history.rs b/engine/crates/layerfs-persistence/tests/diagnostic_history.rs index 46cc3021e7..dbd394fa91 100644 --- a/engine/crates/layerfs-persistence/tests/diagnostic_history.rs +++ b/engine/crates/layerfs-persistence/tests/diagnostic_history.rs @@ -191,9 +191,9 @@ fn a_retained_commit_branch_reference_must_exist_in_its_stack() { let f = graph(); clean(&f.audit(true)); f.raw() - .execute_batch(&format!( - "PRAGMA foreign_keys=OFF; UPDATE \"commit\" SET branch_id=CAST(X'11'||zeroblob(16) AS BLOB)" - )) + .execute_batch( + "PRAGMA foreign_keys=OFF; UPDATE \"commit\" SET branch_id=CAST(X'11'||zeroblob(16) AS BLOB)", + ) .unwrap(); let out = f.audit(true); failure(&out, "store.history.branches"); diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json new file mode 100644 index 0000000000..6848f4ca47 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json @@ -0,0 +1,89 @@ +{ + "schema": "t12.3a-sqlite-compact-capacity/1", + "actualBytes": 9986795475, + "actualGib": "9.987", + "packCount": 76870, + "packLengthMin": 561, + "packLengthMax": 131108, + "objectBytes": 131071, + "groups": 701, + "groupSize": 99, + "liveObjects": 69399, + "initialDeadObjects": 6770, + "deadFractionPermille": 88, + "fixtureBuildSeconds": 150.0, + "noneDeadBytes": 887601160, + "noneTotalBytes": 9986795475, + "rebuildDeadBytes": 1011104896, + "rebuildTotalBytes": 10110299211, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)", + "cpus": 88, + "memBytes": 135033307136, + "none": { + "path": "none", + "runs": 3, + "micros": [148519470, 149703782, 149318362], + "medianMicros": 149318362, + "minMicros": 148519470, + "maxMicros": 149703782, + "deadBytesBound": [887601160, 887601160, 887601160], + "deadBytesRefined": [887601160, 887601160, 887601160], + "physicalShrink": [0, 0, 0], + "peakTempBytes": [0, 0, 0], + "oldFileBytes": [10034614272, 10034696192, 10034696192], + "newFileBytes": [10034696192, 10034696192, 10034696192], + "generationBefore": [7, 7, 7], + "generationAfter": [7, 7, 7], + "packCountBefore": [76870, 76870, 76870], + "packCountAfter": [76870, 76870, 76870], + "addedDeadObjects": [0, 0, 0], + "phasesNanos": { + "mark": [148512076807, 149699478879, 149314062028], + "build": [0, 0, 0], + "verify": [0, 0, 0], + "install": [0, 0, 0] + } + }, + "rebuild": { + "path": "rebuild", + "runs": 3, + "micros": [246370906, 255548454, 243305751], + "medianMicros": 246370906, + "minMicros": 243305751, + "maxMicros": 255548454, + "deadBytesBound": [1011104896, 1011104896, 1011104896], + "deadBytesRefined": [1011104896, 1011104896, 1011104896], + "physicalShrink": [1018978304, 1016524800, 1016504320], + "peakTempBytes": [9143750656, 9143750656, 9143750656], + "oldFileBytes": [10158751744, 10156294144, 10156273664], + "newFileBytes": [9139773440, 9139773440, 9139773440], + "generationBefore": [7, 8, 9], + "generationAfter": [8, 9, 10], + "packCountBefore": [77812, 77812, 77812], + "packCountAfter": [70100, 70100, 70100], + "addedDeadObjects": [942, 7712, 7712], + "phasesNanos": { + "mark": [149758815060, 148589673893, 149088791304], + "build": [94316711529, 104423746881, 91945016315], + "verify": [311503029, 431132794, 286831239], + "install": [1913022088, 2035474103, 1915532558] + } + }, + "durableRebuild": { + "path": "rebuild", + "runs": 1, + "micros": [284078209], + "deadBytesBound": [1011104896], + "physicalShrink": [1016483840], + "peakTempBytes": [9143750656], + "addedDeadObjects": [7712], + "phasesNanos": { + "mark": [149687630706], + "build": [132235308703], + "verify": [267889954], + "install": [1816879571] + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json new file mode 100644 index 0000000000..b2b13e713f --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json @@ -0,0 +1,74 @@ +{ + "schema": "t12.3a-sqlite-compact-capacity/1", + "actualBytes": 49998990030, + "actualGib": "49.999", + "packCount": 384850, + "packLengthMin": 561, + "packLengthMax": 131108, + "objectBytes": 131071, + "groups": 3510, + "groupSize": 99, + "liveObjects": 347490, + "initialDeadObjects": 33850, + "deadFractionPermille": 88, + "fixtureBuildSeconds": 761.2, + "noneDeadBytes": 4438005800, + "noneTotalBytes": 49998990030, + "rebuildDeadBytes": 5062342096, + "rebuildTotalBytes": 50623326326, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)", + "cpus": 88, + "memBytes": 135033307136, + "none": { + "path": "none", + "runs": 1, + "micros": [1633585041], + "medianMicros": 1633585041, + "minMicros": 1633585041, + "maxMicros": 1633585041, + "deadBytesBound": [4438005800], + "deadBytesRefined": [4438005800], + "physicalShrink": [0], + "peakTempBytes": [0], + "oldFileBytes": [50235379712], + "newFileBytes": [50235826176], + "generationBefore": [7], + "generationAfter": [7], + "packCountBefore": [384850], + "packCountAfter": [384850], + "addedDeadObjects": [0], + "phasesNanos": { + "mark": [1633578666486], + "build": [0], + "verify": [0], + "install": [0] + } + }, + "rebuild": { + "path": "rebuild", + "runs": 1, + "micros": [5131834971], + "medianMicros": 5131834971, + "minMicros": 5131834971, + "maxMicros": 5131834971, + "deadBytesBound": [5062342096], + "deadBytesRefined": [5062342096], + "physicalShrink": [5099651072], + "peakTempBytes": [45783568384], + "oldFileBytes": [50862854144], + "newFileBytes": [45763203072], + "generationBefore": [7], + "generationAfter": [8], + "packCountBefore": [389612], + "packCountAfter": [351000], + "addedDeadObjects": [4762], + "phasesNanos": { + "mark": [784250627182], + "build": [4254371054960], + "verify": [85556998080], + "install": [7311035596] + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json index c67cf099d2..1167973121 100644 --- a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json @@ -377,7 +377,7 @@ }, "logicalDiscarded": 160, "path": "rebuild", - "peakTempBytes": 389120, + "peakTempBytes": 1537112, "physicalShrink": 221184, "residue": { "artifacts": [], diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json new file mode 100644 index 0000000000..12a2c3be9c --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json @@ -0,0 +1,16 @@ +{ + "schema": "t12.3a-sqlite-compact-measure/1", + "runs": 3, + "micros": [26738, 28201, 28681], + "medianMicros": 28201, + "minMicros": 26738, + "maxMicros": 28681, + "deadBytesBound": 1100, + "physicalShrink": 20480, + "peakTempBytes": 758240, + "liveRoots": 20, + "deadPacks": 20, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)" +} diff --git a/engine/crates/layerfs-persistence/tests/gc_backfill.rs b/engine/crates/layerfs-persistence/tests/gc_backfill.rs index aa65bd1518..eb30210ae7 100644 --- a/engine/crates/layerfs-persistence/tests/gc_backfill.rs +++ b/engine/crates/layerfs-persistence/tests/gc_backfill.rs @@ -314,7 +314,7 @@ fn commits_layers_and_epochs_are_backfilled_in_deterministic_order() { // One epoch row per project, both zero. let rows = epochs(&sql); assert_eq!(rows.len(), 2); - let mut expected = vec![stack_one, stack_two]; + let mut expected = [stack_one, stack_two]; expected.sort_by(|a, b| a.as_slice().cmp(b.as_slice())); for (row, expected) in rows.iter().zip(expected.iter()) { assert_eq!(row.0, expected.as_slice()); diff --git a/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs b/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs new file mode 100644 index 0000000000..4b222d5067 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs @@ -0,0 +1,242 @@ +//! Connection-string rewriting shared by the diagnostic and doctor suites. + +/// Add one libpq connection option to a connection string. +/// +/// A URL (`…://…`) gets a query parameter whose value is percent-encoded. The +/// libpq keyword form that the docs and CI use gets `name='value'`, because +/// `?`/`&` appended to it is parsed as part of the preceding value and the +/// server refuses the connection. +pub fn with_option(url: &str, name: &str, value: &str) -> String { + if url.contains("://") { + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{name}={}", percent_encode(value)) + } else { + format!("{url} {name}='{value}'") + } +} + +/// Put an unresolvable host in front of the real fixture host. +/// +/// The diagnostic driver tries every configured host in order and treats a +/// failed lookup as "try the next host", so `missing.invalid` exercises the +/// failover path. Rewriting only the URL form would leave the keyword form +/// unchanged and the assertion would silently stop exercising failover. +pub fn with_failing_first_host(conninfo: &str) -> String { + const HOST: &str = "127.0.0.1"; + const MISSING: &str = "missing.invalid"; + if conninfo.contains("://") { + return conninfo.replacen(HOST, &format!("{MISSING},{HOST}"), 1); + } + for (from, to) in [ + (format!("host={HOST}"), format!("host={MISSING},{HOST}")), + (format!("host='{HOST}'"), format!("host='{MISSING},{HOST}'")), + ] { + if conninfo.contains(&from) { + return conninfo.replacen(&from, &to, 1); + } + } + conninfo.to_owned() +} + +/// Rewrite a connection string's endpoint to `host:port`. +/// +/// The physical doctor's byte-counting proxy forwards to the fixture, so the +/// selection it opens must reach the proxy's port in whichever connection +/// string form the environment supplied. A URL keeps its user information, +/// path and query and has its authority's host and port replaced; the libpq +/// keyword form has its `host=` and `port=` values replaced (and added when +/// absent), leaving every other keyword — quoted values with spaces included — +/// untouched. +/// +/// Rewriting only `:{port}/` would match the URL form alone: the keyword form +/// would keep connecting straight to the fixture, and the proxy's byte count +/// would silently stop measuring the wire. +pub fn with_host(conninfo: &str, host: &str, port: u16) -> String { + if conninfo.contains("://") { + url_with_host(conninfo, host, port) + } else { + keyword_with_host(conninfo, host, port) + } +} + +/// [`with_host`] for the physical doctor's loopback proxy, whose listener and +/// forwarder are both `127.0.0.1`. +pub fn with_local_proxy(conninfo: &str, proxy_port: u16) -> String { + with_host(conninfo, "127.0.0.1", proxy_port) +} + +fn url_with_host(url: &str, host: &str, port: u16) -> String { + let Some(scheme_end) = url.find("://").map(|at| at + 3) else { + return url.to_owned(); + }; + let (scheme, rest) = url.split_at(scheme_end); + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + let userinfo_end = authority.rfind('@').map(|at| at + 1).unwrap_or(0); + let userinfo = &authority[..userinfo_end]; + format!("{scheme}{userinfo}{host}:{port}{tail}") +} + +fn keyword_with_host(conninfo: &str, host: &str, port: u16) -> String { + let bytes = conninfo.as_bytes(); + let mut out = String::with_capacity(conninfo.len() + host.len() + 16); + let mut seen_host = false; + let mut seen_port = false; + let mut at = 0; + while at < bytes.len() { + let space_start = at; + while at < bytes.len() && bytes[at].is_ascii_whitespace() { + at += 1; + } + out.push_str(&conninfo[space_start..at]); + if at == bytes.len() { + break; + } + let key_start = at; + while at < bytes.len() && (bytes[at] == b'_' || bytes[at].is_ascii_alphanumeric()) { + at += 1; + } + let key_end = at; + if key_end == key_start || bytes.get(key_end) != Some(&b'=') { + // Not `keyword=value`: keep the tail exactly as it came in. + out.push_str(&conninfo[key_start..]); + break; + } + at += 1; + if bytes.get(at) == Some(&b'\'') { + at += 1; + while at < bytes.len() { + match bytes[at] { + b'\\' => at = (at + 2).min(bytes.len()), + b'\'' if bytes.get(at + 1) == Some(&b'\'') => at += 2, + b'\'' => { + at += 1; + break; + } + _ => at += 1, + } + } + } else { + while at < bytes.len() && !bytes[at].is_ascii_whitespace() { + at += 1; + } + } + match &conninfo[key_start..key_end] { + "host" if !seen_host => { + out.push_str("host="); + out.push_str(host); + seen_host = true; + } + "port" if !seen_port => { + out.push_str("port="); + out.push_str(&port.to_string()); + seen_port = true; + } + _ => out.push_str(&conninfo[key_start..at]), + } + } + if !seen_host { + out.push_str(&format!(" host={host}")); + } + if !seen_port { + out.push_str(&format!(" port={port}")); + } + out +} + +fn percent_encode(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for &byte in value.as_bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +#[cfg(test)] +mod tests { + use super::{with_failing_first_host, with_host, with_local_proxy, with_option}; + + #[test] + fn url_form_rewrites_options_and_endpoints() { + assert_eq!( + with_option("postgresql://u@h/db", "options", "-c role=r"), + "postgresql://u@h/db?options=-c%20role%3Dr" + ); + assert_eq!( + with_option( + "postgresql://u@h/db?sslmode=disable", + "options", + "-c role=r" + ), + "postgresql://u@h/db?sslmode=disable&options=-c%20role%3Dr" + ); + assert_eq!( + with_host( + "postgresql://u:p@127.0.0.1:5432/db?sslmode=disable", + "127.0.0.1", + 9000 + ), + "postgresql://u:p@127.0.0.1:9000/db?sslmode=disable" + ); + assert_eq!( + with_host("postgres://127.0.0.1/db", "127.0.0.1", 9000), + "postgres://127.0.0.1:9000/db" + ); + assert_eq!( + with_host("postgresql://u@127.0.0.1:5432", "127.0.0.1", 9000), + "postgresql://u@127.0.0.1:9000" + ); + assert_eq!( + with_local_proxy("postgresql://u@127.0.0.1:5432/db", 9000), + "postgresql://u@127.0.0.1:9000/db" + ); + } + + #[test] + fn keyword_form_rewrites_options_and_endpoints() { + assert_eq!( + with_option("host=h port=1 sslmode=disable", "options", "-c role=r"), + "host=h port=1 sslmode=disable options='-c role=r'" + ); + assert_eq!( + with_host("host=127.0.0.1 port=5432 sslmode=disable", "127.0.0.1", 9000), + "host=127.0.0.1 port=9000 sslmode=disable" + ); + assert_eq!( + with_host("host='127.0.0.1' port='5432'", "127.0.0.1", 9000), + "host=127.0.0.1 port=9000" + ); + assert_eq!( + with_host( + "host=127.0.0.1 port=5432 options='-c role=r'", + "127.0.0.1", + 9000 + ), + "host=127.0.0.1 port=9000 options='-c role=r'" + ); + assert_eq!( + with_local_proxy("host=127.0.0.1 port=5432", 9000), + "host=127.0.0.1 port=9000" + ); + } + + #[test] + fn url_failover_puts_the_missing_host_first() { + assert_eq!( + with_failing_first_host("postgresql://u:p@127.0.0.1:5432/db?sslmode=disable"), + "postgresql://u:p@missing.invalid,127.0.0.1:5432/db?sslmode=disable" + ); + } + + #[test] + fn keyword_failover_puts_the_missing_host_first() { + assert_eq!( + with_failing_first_host("host=127.0.0.1 port=5432 sslmode=disable"), + "host=missing.invalid,127.0.0.1 port=5432 sslmode=disable" + ); + } +} diff --git a/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs b/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs index cecca20de2..31ef6e1384 100644 --- a/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs +++ b/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs @@ -10,9 +10,21 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, time::{Duration, Instant}, }; +pub mod conninfo; pub mod namespace; pub mod proxy; pub const BINDING: &[u8] = b"doctor-postgres-acceptance"; +/// An auxiliary server the physical doctor's extra cases need, or `None` +/// with the usual NOT_RUN note. The primary `LAYERFS_TEST_POSTGRES_URL` +/// still drives the rest of the target. +pub fn auxiliary_url(variable: &str) -> Option { + let url = std::env::var(variable).ok(); + if url.is_none() { + eprintln!("NOT_RUN: {variable} is unset"); + } + url +} + pub struct Fixture { pub selection: PostgresSelection, pub writer: Option, @@ -70,9 +82,10 @@ impl Fixture { self.roles.push(role.clone()); role } + /// The same selection with `role` applied to every connection. pub fn restricted(&self, role: &str) -> PostgresSelection { let mut result = self.selection.clone(); - result.url.push_str(&format!("&options=-c%20role%3D{role}")); + result.url = conninfo::with_option(&result.url, "options", &format!("-c role={role}")); result } pub fn fingerprint(&mut self) -> Vec<(String, i64, String)> { diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs index acdcebce6a..9b12630626 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs @@ -5,6 +5,18 @@ //! scratch schema or op row. A leftover at `G` is removed; one at `G + 1` //! finishes Phase C (lost-reply resume). A conflicting lock is `Busy` (55P03). //! +//! T10.4 adds the bounded `compact.*` capture tests: a successful rebuild, the +//! `none` path, and the Busy/Unresolved refusals. They assert the same closed +//! labels and values as the SQLite capture test. +//! +//! AC-44 adds the kill arms: a real `pg_terminate_backend` of the recorded +//! `(pid, backend_start)` before the swap COMMIT (wholly at `G`) and during +//! Phase C after the swap COMMIT (wholly at `G + 1`), plus the deterministic +//! `PostgresAfterSwap` lost-reply arm whose owner is then really killed. A +//! named failpoint alone is a simulated lost reply: the process survives and +//! releases its intent. A killed backend leaves the op, the intent and (before +//! COMMIT) the scratch for the next call to resolve. +//! //! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for //! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres //! sslmode=disable`). Without it each test reports NOT_RUN. @@ -12,39 +24,54 @@ mod support; use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; use layerfs_history::{ AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, LayerStackId, LayerStackRecord, StackInitialization, }; +use layerfs_persistence::compact_metrics::CompactMetricsCapture; use layerfs_persistence::{ CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, }; use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; -use support::{branch_id, name, stack, BINDING}; +use support::metrics::{assert_bounded, metric, value}; +use support::{branch_id, name, stack}; /// Drops the test's schema when the test ends, pass or fail. struct Schema { url: String, selection: PostgresSelection, + /// This test's own catalog identity. The Store-scoped admission advisory + /// key derives from it, and PostgreSQL advisory locks are server-wide + /// (not schema-scoped), so tests that shared one binding would exclude + /// each other while running in parallel. + binding: Vec, } impl Schema { fn new(url: &str, tag: &str) -> Self { let name = format!("lfs_t9_{tag}_{}", std::process::id()); let selection = PostgresSelection::new(url, name); + let binding = binding(tag); let schema = Self { url: url.to_owned(), selection, + binding, }; schema.drop_schema(); schema } fn open(&self) -> PostgresHandles { - PostgresHandles::open_writable(&self.selection, BINDING, [71; 32]).expect("Store open") + PostgresHandles::open_writable(&self.selection, &self.binding, [71; 32]) + .expect("Store open") } fn create(&self) -> PostgresHandles { - PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) - .expect("Store creation") + PostgresHandles::create( + &self.selection, + StoragePolicy::frozen_default(), + &catalog(&self.binding), + ) + .expect("Store creation") } fn client(&self) -> postgres::Client { let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); @@ -95,9 +122,16 @@ impl Drop for Schema { } } -fn catalog() -> HistoryCatalogConfig { +/// Every test opens its own catalog identity, as a real Store does: the +/// PostgreSQL advisory keys are server-global, so two tests that shared a +/// binding could see each other's intents while running in parallel. +fn binding(tag: &str) -> Vec { + format!("postgresql-compact-{tag}").into_bytes() +} + +fn catalog(binding: &[u8]) -> HistoryCatalogConfig { HistoryCatalogConfig { - binding_key: BINDING.to_vec(), + binding_key: binding.to_vec(), cursor_key: [71; 32], incarnation: 7, } @@ -297,6 +331,345 @@ fn db_generation(schema: &Schema) -> u64 { .unwrap() } +/// The phase counts one PostgreSQL rebuild times: Phase M/A/B/C, each once. +fn assert_pg_rebuild_phases(metrics: &layerfs_persistence::compact_metrics::CompactMetrics) { + for phase in ["pg_mark", "pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "PostgreSQL {phase} was timed once" + ); + assert!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "sum")] + ) > 0, + "PostgreSQL {phase} took time" + ); + assert!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "max")] + ) > 0, + "PostgreSQL {phase} has a duration" + ); + } + for phase in ["mark", "build", "verify", "install"] { + assert_eq!( + metric( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "a PostgreSQL run never emits the SQLite phase {phase}" + ); + } +} + +#[test] +fn a_rebuild_records_bounded_metrics_with_pg_phases() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_rebuild"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the metrics rebuild collects"); + + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(21)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "rebuild")]), + 1 + ); + assert_pg_rebuild_phases(&metrics); + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert!(value(&metrics, "compact.duration_ns", &[("statistic", "sum")]) > 0); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + let kept = receipt + .kept_waste + .dead_in_retained_packs_bytes + .saturating_add(receipt.kept_waste.delta_bases_bytes) + .saturating_add(receipt.kept_waste.inode_blocks_bytes); + assert_eq!(value(&metrics, "compact.kept_waste_bytes", &[]), kept); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + receipt.residue.bytes + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + assert_eq!(value(&metrics, "compact.settles", &[("kind", "run")]), 1); + assert_eq!( + metric(&metrics, "compact.settles", &[("kind", "leftover")]), + None, + "a fresh rebuild settles no leftover" + ); + assert_eq!( + metric(&metrics, "compact.retries", &[("reason", "leftover")]), + None + ); + + clean(&schema.open()); +} + +#[test] +fn a_none_path_records_only_the_pg_mark_phase() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_none"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(22)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::None); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "none")]), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", "pg_mark"), ("statistic", "count")] + ), + 1 + ); + for phase in ["pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "the PostgreSQL none path never enters {phase}" + ); + } + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + 0, + "none leaves no artifact" + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after, + "none keeps the generation" + ); + assert_eq!(receipt.generation_after, receipt.generation_before); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + 1 + ); + assert_eq!(value(&metrics, "compact.settles", &[("kind", "run")]), 1); + + clean(&schema.open()); +} + +#[test] +fn a_busy_swap_records_a_failed_outcome_and_a_busy_refusal() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_busy"); + schema.drop_scratch(23); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the metrics busy run would collect"); + // Phase A never touches store_totals; the swap's ACCESS EXCLUSIVE NOWAIT + // does, so this lock is met exactly at Phase B (SQLSTATE 55P03). + let mut blocker = schema.client(); + blocker + .batch_execute(&format!( + "BEGIN; LOCK TABLE {}.binding_hint_block IN ACCESS EXCLUSIVE MODE;", + schema.selection.schema + )) + .unwrap(); + + let capture = CompactMetricsCapture::start(); + let error = handles.compact(&request(23)).unwrap_err(); + let metrics = capture.finish(); + blocker.batch_execute("ROLLBACK").unwrap(); + + assert!( + matches!(error, layerfs_history::HistoryError::Busy), + "{error:?}" + ); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "failed")]), + 1 + ); + assert_eq!(value(&metrics, "compact.refusals", &[("code", "busy")]), 1); + for phase in ["pg_mark", "pg_build", "pg_swap"] { + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "the refused run reached {phase}" + ); + } + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", "pg_cleanup"), ("statistic", "count")] + ), + None, + "the refused run never reaches Phase C" + ); + assert_eq!( + metric(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + None, + "a failed run publishes no report gauge" + ); + assert_eq!( + metric(&metrics, "compact.generation", &[("which", "after")]), + None + ); + assert_eq!( + metric(&metrics, "compact.settles", &[("kind", "run")]), + None + ); + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); +} + +#[test] +fn an_uncertain_intent_records_an_unresolved_refusal() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_unresolved"); + let handles = schema.create(); + // A dead owner's uncertain compaction intent with no `gc_op` row: the + // next call meets it at admission, before Phase M writes anything. + let dead = [0x44u8; 16]; + let intent = GcIntent::operation( + dead, + GcIntentKind::Compaction, + GcScopeKind::Store, + vec![0u8; 16], + ) + .with_phase(Some("mark")); + handles.history.gc_register(&intent).unwrap(); + handles.history.gc_mark_uncertain(&dead).unwrap(); + + let capture = CompactMetricsCapture::start(); + let error = handles.compact(&request(24)).unwrap_err(); + let metrics = capture.finish(); + + assert!( + matches!(error, layerfs_history::HistoryError::Unresolved(_)), + "{error:?}" + ); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "failed")]), + 1 + ); + assert_eq!( + value(&metrics, "compact.refusals", &[("code", "unresolved")]), + 1 + ); + for phase in ["pg_mark", "pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "the refusal stops before {phase}" + ); + } + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert_eq!( + metric(&metrics, "compact.generation", &[("which", "after")]), + None + ); + + handles.history.gc_release(&dead).unwrap(); +} + fn clean(handles: &PostgresHandles) { let mut out = Vec::new(); handles @@ -580,3 +953,716 @@ fn an_exclusive_table_lock_makes_the_swap_busy() { "a 55P03 lock at the swap is Busy, got {error:?}" ); } + +// --- AC-44: a killed backend and the lost COMMIT reply --------------------- +// +// The failpoints above stop a live session and return `UnknownOutcome`: the +// process survives and releases its own intent, so they are a *simulated* lost +// reply. The tests below kill the PostgreSQL backend that owns the round, at a +// point where the server's own `pg_terminate_backend(pid, timeout)` is scoped to +// the `(pid, backend_start)` recorded in `gc_op` (the §12 proof). The next call +// then meets a real leftover whose recorded owner is gone and must resolve it: +// +// * `a_killed_backend_before_the_swap_commit_leaves_the_store_at_g` parks the +// backend inside Phase B's own `UPDATE gc_op`, before COMMIT, and terminates +// it. The Store is wholly at `G`; the next compact proves the owner ended and +// adopts the `G` leftover. +// * `a_killed_backend_during_phase_c_resumes_at_g_plus_1` parks the backend in +// Phase C's `store_totals` settle, after COMMIT committed the swap, and +// terminates it. The Store is wholly at `G + 1`; the next compact reads the +// incarnation and finishes Phase C. +// * `a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1` uses the named +// `PostgresAfterSwap` failpoint as the deterministic stand-in for the instant +// a COMMIT reply is lost (the caller sees `UnknownOutcome` while the swap is +// committed), then kills that owner backend so the owner is provably ended +// before a fresh connection resolves the reply. +// +// The only injected condition is a test-only `BEFORE UPDATE` trigger on the +// statement whose commit must be interrupted; it calls `pg_sleep` so a second +// connection can find the backend in `pg_stat_activity` and terminate exactly +// that process. Nothing in the Store is modified by the injection. + +/// The four `store_totals` columns, so a killed round can be shown not to have +/// settled them. +fn totals(schema: &Schema) -> (i64, i64, i64, i64) { + let mut client = schema.client(); + let row = client + .query_one( + &format!( + "SELECT logical,canonical,stored,metadata FROM {}.store_totals WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("store totals"); + (row.get(0), row.get(1), row.get(2), row.get(3)) +} + +/// One `gc_op` row: `(pid, backend_start, phase)`. +fn op_owner(schema: &Schema) -> Option<(i64, String, String)> { + let mut client = schema.client(); + let row = client + .query_opt( + &format!( + "SELECT pid, backend_start::text, phase::text FROM {}.gc_op WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("the gc_op row"); + row.map(|row| (row.get(0), row.get(1), row.get(2))) +} + +/// Waits for the interrupted round to record its `(pid, backend_start)`. +fn wait_for_op_owner(schema: &Schema) -> (i64, String, String) { + for _ in 0..2400 { + if let Some(owner) = op_owner(schema) { + return owner; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the interrupted round never wrote its gc_op row"); +} + +/// Installs a test-only trigger that parks the backend in `pg_sleep` on the +/// next `UPDATE` of `table`, before the update becomes visible or is committed. +fn park_backend(schema: &Schema, table: &str) { + let name = format!("lfs_test_park_{table}"); + let mut client = schema.client(); + client + .batch_execute(&format!( + "CREATE OR REPLACE FUNCTION {schema}.{name}() RETURNS trigger LANGUAGE plpgsql AS \ + $fn$ BEGIN PERFORM pg_sleep(120); RETURN NEW; END $fn$;\ + CREATE TRIGGER {name} BEFORE UPDATE ON {schema}.{table} \ + FOR EACH ROW EXECUTE FUNCTION {schema}.{name}();", + schema = schema.selection.schema, + table = table, + name = name, + )) + .expect("the test parking trigger"); +} + +/// Removes the test-only parking trigger and its function. +fn unpark_backend(schema: &Schema, table: &str) { + let name = format!("lfs_test_park_{table}"); + let mut client = schema.client(); + let _ = client.batch_execute(&format!( + "DROP TRIGGER IF EXISTS {name} ON {schema}.{table};\ + DROP FUNCTION IF EXISTS {schema}.{name}();", + schema = schema.selection.schema, + table = table, + name = name, + )); +} + +/// Waits for the child backend parked in the unique schema-qualified +/// statement the test injected, and returns `(pid, backend_start)`. Matching on +/// the statement avoids reading `gc_op` while Phase B holds every base table +/// `ACCESS EXCLUSIVE`, so there is no race between the parent's poll and the +/// parked swap transaction. +fn wait_for_parked_statement(schema: &Schema, table: &str) -> (i64, String) { + let pattern = format!("%{}.{table}%", schema.selection.schema); + let mut client = schema.client(); + for _ in 0..4800 { + let rows = client + .query( + "SELECT pid, backend_start::text FROM pg_stat_activity \ + WHERE datname=current_database() AND state='active' \ + AND wait_event='PgSleep' AND query LIKE $1 AND pid<>pg_backend_pid()", + &[&pattern], + ) + .expect("pg_stat_activity"); + if let Some(row) = rows.first() { + let pid: i32 = row.get(0); + return (i64::from(pid), row.get(1)); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("no backend parked in a {table} statement"); +} + +/// Kills exactly the recorded owner: the two-argument +/// `pg_terminate_backend(pid, timeout)`, scoped by `(pid, backend_start)`, never +/// the caller and never another database. This is the §12 proof the next call +/// relies on. +fn kill_recorded_owner(schema: &Schema, pid: i64, backend_start: &str) { + let mut client = schema.client(); + let killed: bool = client + .query_one( + "SELECT COALESCE(bool_or(pg_terminate_backend(a.pid, 30000)), false) \ + FROM pg_stat_activity a \ + WHERE a.pid::bigint=$1 AND a.backend_start::text=$2 \ + AND a.datname=current_database() AND a.pid<>pg_backend_pid()", + &[&pid, &backend_start], + ) + .expect("pg_terminate_backend") + .get(0); + assert!(killed, "the recorded owner backend {pid} was terminated"); +} + +/// Proves the recorded owner ended: no `pg_stat_activity` row keeps the same +/// `(pid, backend_start)` pair. +fn assert_owner_ended(schema: &Schema, pid: i64, backend_start: &str) { + let mut client = schema.client(); + for _ in 0..2400 { + let alive: i64 = client + .query_one( + "SELECT count(*) FROM pg_stat_activity \ + WHERE pid::bigint=$1 AND backend_start::text=$2", + &[&pid, &backend_start], + ) + .expect("pg_stat_activity") + .get(0); + if alive == 0 { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend {pid} did not end"); +} + +/// The compaction intents a Store's admission table still holds. +fn compaction_intents(handles: &PostgresHandles, schema: &Schema) -> Vec { + let catalog = layerfs_history::CatalogId::derive(&schema.binding).expect("catalog identity"); + handles + .history + .gc_intents(GcScopeKind::Store, catalog.as_slice()) + .expect("compaction intents") +} + +/// Runs one compact on a connection of its own, so the parent can terminate +/// exactly that backend. +fn spawn_round( + selection: PostgresSelection, + binding: Vec, + op: u8, + failpoint: Option, +) -> std::thread::JoinHandle> { + std::thread::spawn(move || { + let handles = + PostgresHandles::open_writable(&selection, &binding, [71; 32]).expect("child open"); + handles.compact(&CompactRequest { + op_id: [op; 16], + failpoint, + ..CompactRequest::default() + }) + }) +} + +#[test] +fn a_killed_backend_before_the_swap_commit_leaves_the_store_at_g() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "kill_before_commit"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the killed round would collect"); + let before = generation(&setup); + let totals_before = totals(&schema); + + // Park Phase B's own `UPDATE gc_op` before COMMIT. The kill therefore lands + // after Phase A built the scratch and before the swap transaction can + // commit, which is the "killed before COMMIT" arm. + park_backend(&schema, "gc_op"); + let child = spawn_round(schema.selection.clone(), schema.binding.clone(), 41, None); + let (pid, start) = wait_for_parked_statement(&schema, "gc_op"); + kill_recorded_owner(&schema, pid, &start); + let error = child + .join() + .expect("the killed round's thread") + .unwrap_err(); + unpark_backend(&schema, "gc_op"); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "a killed backend loses the reply: {error:?}" + ); + + // Wholly old at `G`: the swap transaction rolled back, so the Store is not + // mixed and the old totals still hold. + assert_eq!(db_generation(&schema), before); + assert_eq!(totals(&schema), totals_before); + assert!(located(&setup, live), "the retained object survives"); + assert!(located(&setup, dead), "the swap did not commit"); + // The documented leftover of an interrupted round: Phase M's op row keeps + // the owner, Phase A's scratch schema and the dead set remain. + let (op_pid, op_start, op_phase) = op_owner(&schema).expect("the op row survives the kill"); + assert_eq!((op_pid, op_phase), (pid, "mark".to_owned())); + assert_eq!(op_start, start); + assert!( + schema.scratch_exists(41), + "Phase A's scratch schema remains" + ); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + + // The owner is provably ended, and the admission intent the killed process + // never released is still visible: a compaction intent is not an uncertain + // one, so the resume adopts it instead of reporting `Unresolved`. + assert_owner_ended(&schema, pid, &start); + let intents = compaction_intents(&setup, &schema); + assert_eq!( + intents.len(), + 1, + "the killed process never released its intent" + ); + assert_eq!(intents[0].kind, GcIntentKind::Compaction); + assert_eq!(intents[0].phase.as_deref(), Some("mark")); + assert!( + !intents[0].uncertain, + "a compaction intent carries no uncertain owner" + ); + + // The next compact proves the owner ended, adopts the `G` leftover and + // rebuilds once: `G -> G + 1`. + let resumed = schema.open(); + let receipt = resumed.compact(&request(42)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 42); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the next round collects the dead pack" + ); + assert!( + located(&resumed, live), + "the retained object survives the resume" + ); + clean(&schema.open()); +} + +#[test] +fn a_killed_backend_during_phase_c_resumes_at_g_plus_1() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "kill_phase_c"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the killed cleanup would collect"); + let before = generation(&setup); + let totals_before = totals(&schema); + + // Park Phase C's `store_totals` settle. The swap transaction already + // committed `G + 1` before Phase C starts, so the kill lands on the + // "installed but cleanup unfinished" arm. + park_backend(&schema, "store_totals"); + let child = spawn_round(schema.selection.clone(), schema.binding.clone(), 43, None); + let (pid, start) = wait_for_parked_statement(&schema, "store_totals"); + kill_recorded_owner(&schema, pid, &start); + let error = child + .join() + .expect("the killed round's thread") + .unwrap_err(); + unpark_backend(&schema, "store_totals"); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "a killed backend loses the reply: {error:?}" + ); + + // Wholly new at `G + 1`, but Phase C rolled back: the committed swap has + // already replaced the payload tables (the dead pack is gone), while the + // persisted dead history rows, the op row and the old totals remain. + assert_eq!(db_generation(&schema), before + 1); + assert_eq!(totals(&schema), totals_before); + assert!( + !located(&setup, dead), + "the committed swap already replaced the payload tables" + ); + assert!(located(&setup, live)); + let (op_pid, op_start, op_phase) = op_owner(&schema).expect("the op row survives the kill"); + assert_eq!((op_pid, op_phase), (pid, "swap".to_owned())); + assert_eq!(op_start, start); + assert!( + !schema.scratch_exists(43), + "the swap transaction dropped the scratch before COMMIT" + ); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + assert_owner_ended(&schema, pid, &start); + // A real kill runs no error path: the intent is still registered. + let intents = compaction_intents(&setup, &schema); + assert_eq!( + intents.len(), + 1, + "the killed process never released its intent" + ); + assert_eq!(intents[0].kind, GcIntentKind::Compaction); + assert!(!intents[0].uncertain); + + // The next compact reads the incarnation (`G + 1`) and finishes Phase C, + // reporting the outcome the lost reply would have reported. + let resumed = schema.open(); + let receipt = resumed.compact(&request(44)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 44); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the resumed Phase C collects the dead pack" + ); + assert!(located(&resumed, live)); + clean(&schema.open()); +} + +#[test] +fn a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "lost_commit"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the lost COMMIT would collect"); + let before = generation(&setup); + + // The named failpoint is the deterministic stand-in for the instant the + // swap's COMMIT reply is lost: the swap committed at `G + 1`, the caller + // saw `UnknownOutcome`, and its backend is still alive. The child holds its + // handles open so the parent can end that owner with a real kill. + let (started_tx, started_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let selection = schema.selection.clone(); + let binding = schema.binding.clone(); + let child = std::thread::spawn(move || { + let handles = + PostgresHandles::open_writable(&selection, &binding, [71; 32]).expect("child open"); + let result = handles.compact(&request_at(45, CompactFailpoint::PostgresAfterSwap)); + started_tx + .send(()) + .expect("the child reports its lost reply"); + release_rx.recv().expect("the parent releases the child"); + drop(handles); + result + }); + started_rx + .recv_timeout(std::time::Duration::from_secs(60)) + .expect("the failpoint returned"); + + // The committed incarnation says the swap happened; the op still names + // Phase C and the scratch is already gone. The child is still holding its + // handles open, so this backend is alive and idle: the real kill below + // proves the owner ended. + assert_eq!(db_generation(&schema), before + 1); + let (pid, start, phase) = wait_for_op_owner(&schema); + assert_eq!(phase, "swap"); + assert!(!schema.scratch_exists(45)); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + // A simulated lost reply keeps the process alive and releases its intent; + // the real kill below then proves the owner ended. + assert!( + compaction_intents(&setup, &schema).is_empty(), + "the failpoint path released its own intent" + ); + kill_recorded_owner(&schema, pid, &start); + assert_owner_ended(&schema, pid, &start); + release_tx.send(()).expect("release the lost-reply child"); + let error = child.join().expect("the lost-reply thread").unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "the failpoint simulates the lost reply: {error:?}" + ); + + // A fresh connection resolves the committed swap: it reads `G + 1` and + // finishes Phase C without replaying the swap. + let resumed = schema.open(); + let receipt = resumed.compact(&request(46)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 46); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the resumed Phase C collects the dead pack" + ); + assert!(located(&resumed, live)); + clean(&schema.open()); +} + +// --- AC-44: the privilege half --------------------------------------------- + +/// The base URL with `role` as the login user, for either the keyword or the +/// URL form. +fn role_url(url: &str, role: &str) -> String { + if let Some((scheme, tail)) = url.split_once("://") { + let authority_end = tail.find('/').unwrap_or(tail.len()); + let authority = &tail[..authority_end]; + let host = authority.rsplit('@').next().unwrap_or(authority); + format!("{scheme}://{role}@{host}{}", &tail[authority_end..]) + } else { + let mut parts = Vec::new(); + let mut replaced = false; + for part in url.split_whitespace() { + if part.starts_with("user=") { + parts.push(format!("user={role}")); + replaced = true; + } else { + parts.push(part.to_owned()); + } + } + if !replaced { + parts.push(format!("user={role}")); + } + parts.join(" ") + } +} + +/// Serializes the role and database ACL writes of the AC-44 matrix: the +/// tests run in parallel, and two `GRANT`/`REVOKE` statements on the same +/// `pg_database` row can fail with `tuple concurrently updated`. The lock is +/// held only while a fixture writes, never through a compaction. +static ROLE_SETUP: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn role_setup() -> std::sync::MutexGuard<'static, ()> { + ROLE_SETUP + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// A login role of the AC-44 privilege matrix. It owns the Store's tables and +/// sequences, so a swap may drop, move and re-create them, while the schema +/// and database stay the fixture administrator's; each test grants exactly +/// the Store-wide privileges its arm needs. The suite's PostgreSQL server is +/// this test binary's own, so it revokes the PUBLIC database grants once per +/// role: a superuser bypasses them. +struct Role { + name: String, + url: String, + admin_url: String, + database: String, +} +impl Role { + fn new(schema: &Schema, tag: &str) -> Self { + let _setup = role_setup(); + let name = format!("lfs_g54_{tag}_{}", std::process::id()); + let mut admin = + postgres::Client::connect(&schema.url, postgres::NoTls).expect("role admin"); + let _ = admin.batch_execute(&format!("DROP ROLE IF EXISTS {name}")); + admin + .batch_execute(&format!("CREATE ROLE {name} LOGIN")) + .expect("role creation"); + let database: String = admin + .query_one("SELECT current_database()", &[]) + .expect("database") + .get(0); + admin + .batch_execute(&format!( + "REVOKE CREATE, TEMP ON DATABASE {database} FROM PUBLIC" + )) + .expect("database public grants"); + admin + .batch_execute(&format!( + "GRANT USAGE ON SCHEMA {} TO {name}", + schema.selection.schema + )) + .expect("schema usage"); + let owned: Vec<(String, String)> = admin + .query( + "SELECT c.relname::text, c.relkind::text FROM pg_class c \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = $1 AND (c.relkind IN ('r', 'p') \ + OR (c.relkind = 'S' AND NOT EXISTS (SELECT 1 FROM pg_depend d \ + WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid \ + AND d.refclassid = 'pg_class'::regclass \ + AND d.deptype IN ('a', 'i'))))", + &[&schema.selection.schema], + ) + .expect("schema relations") + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + for (relation, kind) in owned { + let relation = relation.replace('"', "\"\""); + let keyword = if kind == "S" { "SEQUENCE" } else { "TABLE" }; + admin + .batch_execute(&format!( + "ALTER {keyword} \"{}\".\"{relation}\" OWNER TO {name}", + schema.selection.schema + )) + .expect("relation ownership"); + } + Self { + url: role_url(&schema.url, &name), + name, + admin_url: schema.url.clone(), + database, + } + } + fn admin(&self) -> postgres::Client { + postgres::Client::connect(&self.admin_url, postgres::NoTls).expect("role admin") + } + fn grant_schema_create(&self, schema: &Schema) { + let _setup = role_setup(); + self.admin() + .batch_execute(&format!( + "GRANT CREATE ON SCHEMA {} TO {}", + schema.selection.schema, self.name + )) + .expect("schema create"); + } + fn grant_database(&self, create: bool, temp: bool) { + let _setup = role_setup(); + let mut sql = String::new(); + if create { + sql.push_str(&format!( + "GRANT CREATE ON DATABASE {} TO {};", + self.database, self.name + )); + } + if temp { + sql.push_str(&format!( + "GRANT TEMP ON DATABASE {} TO {};", + self.database, self.name + )); + } + if !sql.is_empty() { + self.admin().batch_execute(&sql).expect("database grants"); + } + } + fn grant_read_all_stats(&self) { + let _setup = role_setup(); + self.admin() + .batch_execute(&format!("GRANT pg_read_all_stats TO {}", self.name)) + .expect("stats grant"); + } + fn open(&self, schema: &Schema) -> PostgresHandles { + let selection = PostgresSelection::new(&self.url, schema.selection.schema.clone()); + PostgresHandles::open_writable(&selection, &schema.binding, [71; 32]).expect("role open") + } +} +impl Drop for Role { + fn drop(&mut self) { + let _setup = role_setup(); + if let Ok(mut admin) = postgres::Client::connect(&self.admin_url, postgres::NoTls) { + let _ = admin.batch_execute(&format!("DROP OWNED BY {}", self.name)); + let _ = admin.batch_execute(&format!("DROP ROLE IF EXISTS {}", self.name)); + } + } +} + +/// A Store with a live root and a wholly dead pack, so an arm that got past +/// the privilege preflight would take the rebuild path and create the scratch +/// schema. +fn seed_rebuild(schema: &Schema) -> (PostgresHandles, ObjectId, ObjectId) { + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack a privileged compaction collects"); + (handles, live, dead) +} + +#[test] +fn a_role_without_create_on_the_database_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_database_create"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_database_create"); + role.grant_schema_create(&schema); + role.grant_database(false, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(61)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("CREATE on the database")), + "{error:?}" + ); + assert!(!schema.scratch_exists(61), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_create_on_the_store_schema_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_schema_create"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_schema_create"); + role.grant_database(true, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(62)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("CREATE on the Store schema")), + "{error:?}" + ); + assert!(!schema.scratch_exists(62), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_temp_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_temp"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_temp"); + role.grant_schema_create(&schema); + role.grant_database(true, false); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(63)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("TEMP on the database")), + "{error:?}" + ); + assert!(!schema.scratch_exists(63), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_pg_read_all_stats_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_stats"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_stats"); + role.grant_schema_create(&schema); + role.grant_database(true, true); + let handles = role.open(&schema); + let error = handles.compact(&request(64)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("pg_read_all_stats")), + "{error:?}" + ); + assert!(!schema.scratch_exists(64), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_fully_privileged_role_compacts() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_full"); + let (setup, live, dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "full"); + role.grant_schema_create(&schema); + role.grant_database(true, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let receipt = handles.compact(&request(65)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 65); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&handles, dead), + "the rebuild collected the dead pack" + ); + assert!( + located(&handles, live), + "the retained root survives the role's rebuild" + ); + drop(handles); + clean(&schema.open()); +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs new file mode 100644 index 0000000000..6d7a6fc643 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs @@ -0,0 +1,1790 @@ +//! #41 T12.3b: the PostgreSQL half of the §12 acceptance matrix. +//! +//! One fixture, built through the public APIs only, mirrors the T12.1 SQLite +//! fixture on PostgreSQL so the cross-backend rows have the same evidence on +//! both engines: the §6.4 live-heads digest is compared against the *SQLite +//! golden literal* (AC-40, AC-56), and the §11 report, §7 worklist and §8 +//! swap/cleanup are asserted on the Store rows (AC-02, AC-05, AC-17, AC-18, +//! AC-25, AC-41, AC-52). Two failpoint tests cover the §9 leftovers the T9 +//! note left open: a save after the swap survives the resumed Phase C +//! (AC-28) and the resumed Store is usable with totals, ACL, trigger and +//! hints checked (AC-43). A marker test covers the PostgreSQL `project_epoch` +//! bump and fence (AC-24, AC-39). +//! +//! Genuinely unreachable arms are named in +//! `core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md` with the +//! reason (real standby, killed-backend lost COMMIT, a save racing the swap). +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{ + encode_whole_file_payload, + file::mapping::{self, ExtentNode, ExtentSlice, FileState}, + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + root::profile_id, + scope_for_seed, FilesystemRoot, PathName, + }, + inode_leaf::{ + encode_inode_value, InodeKind, InodeLeaf, InodeLeafRow, InodeMetadata, InodeValue, + LEAF_ROW_BYTES, + }, + FinalizedObject, ObjectId, ObjectRole, +}; +use layerfs_history::error::{HistoryError, Missing}; +use layerfs_history::path::NormalizedPath; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, HistoryCatalog, + HistoryCatalogConfig, LayerId, LayerRecord, LayerStackId, LayerStackRecord, PathRemovalKind, + PathRemovalRequest, StackInitialization, +}; +use layerfs_persistence::{ + diagnostic::{Observation, Options, Outcome}, + CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, +}; +use layerfs_storage::{ + encoding::{encode_full, encode_prefix, raw_payload, CompressionWorkspace}, + location::{InodeValueRow, ObjectLocation, PackDomain}, + pack::{assemble, build_group, layout, PackLane}, + port::{PackPersistence, Publication, PublishedPack, Reserve}, + save::SaveProfile, + Storage, StorageCapacities, StoragePolicy, +}; +use std::sync::Arc; + +const BINDING: &[u8] = b"doctor-adversarial"; +const CURSOR: [u8; 32] = [71; 32]; +const SCOPE_SEED: [u8; 32] = [11; 32]; +/// The T12.1 SQLite transcript: one literal vector for both backends. +const SQLITE_GOLDEN: &str = include_str!("fixtures/sqlite_compact_golden.json"); + +/// Every string value the SQLite golden holds under `key`, in file order. +fn sqlite_literals(key: &str) -> Vec { + let needle = format!("\"{key}\": \""); + let mut out = Vec::new(); + let mut from = 0; + while let Some(offset) = SQLITE_GOLDEN[from..].find(&needle) { + let start = from + offset + needle.len(); + let rest = &SQLITE_GOLDEN[start..]; + let end = rest.find('"').expect("the literal ends"); + out.push(rest[..end].to_owned()); + from = start + end + 1; + } + out +} + +fn sqlite_literal(key: &str) -> String { + let mut values = sqlite_literals(key); + assert!( + values.len() == 1, + "the SQLite golden names {key} once, found {}", + values.len() + ); + values.remove(0) +} + +fn sqlite_commit_literal() -> String { + let needle = "\"commit\": ["; + let start = SQLITE_GOLDEN + .find(needle) + .expect("the SQLite golden commit") + + needle.len(); + let rest = &SQLITE_GOLDEN[start..]; + let first = rest.find('"').expect("the commit literal opens") + 1; + let rest = &rest[first..]; + rest[..rest.find('"').expect("the commit literal closes")].to_owned() +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(DIGITS[(byte >> 4) as usize] as char); + out.push(DIGITS[(byte & 15) as usize] as char); + } + out +} + +fn catalog() -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: CURSOR, + incarnation: 7, + } +} + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +/// One schema, dropped when the test ends, pass or fail. +struct Schema { + url: String, + selection: PostgresSelection, +} + +impl Schema { + fn new(url: &str, tag: &str) -> Self { + let name = format!("lfs123b_{tag}_{}", std::process::id()); + let selection = PostgresSelection::new(url, name); + let schema = Self { + url: url.to_owned(), + selection, + }; + schema.drop_schema(); + schema + } + fn open(&self) -> PostgresHandles { + PostgresHandles::open_writable(&self.selection, BINDING, CURSOR).expect("Store open") + } + fn create(&self) -> PostgresHandles { + PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) + .expect("Store creation") + } + fn client(&self) -> postgres::Client { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); + client + .batch_execute(&format!("SET search_path TO {}", self.selection.schema)) + .expect("search path"); + client + } + fn count(&self, table: &str) -> i64 { + let sql = format!("SELECT count(*) FROM {}.{table}", self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn count_where(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn scalar(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("scalar").get(0) + } + fn exists(&self, sql: &str) -> bool { + self.scalar(sql) != 0 + } + fn scratch_exists(&self, op: u8) -> bool { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + self.exists(&format!( + "SELECT (CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname='{scratch}') THEN 1 ELSE 0 END)::bigint" + )) + } + fn drop_schema(&self) { + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!( + "DROP SCHEMA IF EXISTS {} CASCADE", + self.selection.schema + )); + } + } +} + +impl Drop for Schema { + fn drop(&mut self) { + self.drop_schema(); + } +} + +/// A Store plus the storage helpers the fixture needs (`Fixture::put` and +/// `inode` publish directly, like the SQLite doctor fixture). +struct Fixture { + schema: Schema, + handles: PostgresHandles, + policy: StoragePolicy, +} + +impl Fixture { + fn new(url: &str, tag: &str) -> Self { + let schema = Schema::new(url, tag); + let handles = schema.create(); + Self { + schema, + handles, + policy: StoragePolicy::frozen_default(), + } + } + fn put(&self, role: ObjectRole, canonical: Vec) -> FinalizedObject { + let object = FinalizedObject::new(role, canonical).expect("object"); + let record = encode_full( + object.canonical(), + role, + &StorageCapacities::from_policy(self.policy).expect("capacities"), + &mut CompressionWorkspace::new().expect("workspace"), + &mut SaveProfile::default(), + ) + .expect("record"); + self.record(&object, record.lane, record.record, Vec::new()); + object + } + fn record( + &self, + object: &FinalizedObject, + lane: PackLane, + record: Vec, + values: Vec, + ) { + let body = + assemble(lane, &[build_group(lane, &[record], None).expect("group")]).expect("body"); + self.body(object, lane, body, values); + } + fn body( + &self, + object: &FinalizedObject, + lane: PackLane, + body: Vec, + values: Vec, + ) { + let reserved = self + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .expect("reserve"); + self.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::for_lane(lane), + Arc::new(body), + )], + objects: vec![ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: 0, + record_number: 0, + }], + inode_values: values, + ..Publication::default() + }) + .expect("publication"); + } + fn inode(&self, canonical: Vec, values: Vec<[u8; 73]>) -> ObjectId { + let object = FinalizedObject::new(ObjectRole::InodeLeaf, canonical).expect("inode"); + let reserved = self + .handles + .storage + .reserve(Reserve { + packs: 0, + ordinals: values.len(), + }) + .expect("ordinals"); + let ordinals: Vec<_> = + (reserved.first_ordinal..reserved.first_ordinal + values.len() as u32).collect(); + let pooled = layerfs_content::inode_leaf::pooled_body(object.canonical(), &ordinals) + .expect("pooled"); + let record = layerfs_storage::encoding::pool::leaf::encode_full(&pooled).expect("leaf"); + self.record( + &object, + PackLane::Ordinary, + record, + ordinals + .into_iter() + .zip(values) + .map(|(ordinal, value)| InodeValueRow { ordinal, value }) + .collect(), + ); + object.id() + } +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +// ---- fixture objects ------------------------------------------------------- + +fn pseudo_bytes(len: usize, seed: u8) -> Vec { + let mut state = u64::from(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.to_le_bytes()); + } + out.truncate(len); + out +} + +fn whole_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + f.put( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole file"), + ) + .id() +} + +/// One object published through the real Save path, so its +/// `content_signature` slot exists and the mark can list it in `gc_dead`. +fn saved_whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).expect("storage"); + let save = storage.begin_save().expect("begin save"); + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole file"), + ) + .expect("object"); + let id = object.id(); + save.accept(object).expect("accept"); + save.finish().expect("finish"); + id +} + +/// One chunked file's content root (a FileState): the only kind whose +/// `file_digest` row the Store records (a WholeFile root is below the +/// small-file threshold). +fn chunked_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + let mut extents = Vec::new(); + for part in bytes.chunks(32_768) { + let chunk = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(part).expect("chunk"), + ) + .id(); + extents.push(ExtentSlice::new(chunk, 0, part.len() as u32).expect("slice")); + } + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: bytes.len() as u64, + extents, + }, + true, + ) + .expect("node"), + ) + .id(); + f.put( + ObjectRole::FileState, + mapping::encode_file_state(FileState { + logical_len: bytes.len() as u64, + extent_count: (bytes.len() as u64).div_ceil(32_768), + tree_level: 0, + profile_id: mapping::profile_id(), + mapping_root: leaf, + }) + .expect("file state"), + ) + .id() +} + +/// One valid filesystem tree: a root directory whose only entry is a regular +/// file whose content root is `content`. The inode leaf is pooled. +fn tree(f: &Fixture, content: ObjectId, seed: u8) -> ObjectId { + let directory = f + .put( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: vec![(PathName::new(&format!("file-{seed}")).expect("name"), 2)], + }) + .expect("directory"), + ) + .id(); + let mut values = Vec::new(); + let mut rows = Vec::new(); + for (serial, kind, count, root, mode) in [ + (1u64, InodeKind::Directory, 0u64, directory, 0o755), + (2u64, InodeKind::RegularFile, 1u64, content, 0o644), + ] { + let value = encode_inode_value(InodeValue { + kind, + namespace_ref_count: count, + content_root: root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode, + mtime_seconds: 0, + mtime_nanoseconds: 0, + }), + }); + values.push(value); + rows.push(InodeLeafRow { serial, value }); + } + let inode = f.inode( + InodeLeaf { + subtree_bytes: LEAF_ROW_BYTES as u64 * rows.len() as u64, + rows, + } + .encode() + .expect("inode leaf"), + values, + ); + f.put( + ObjectRole::FilesystemRoot, + FilesystemRoot::new(profile_id(), scope_for_seed(SCOPE_SEED), 1, inode) + .expect("root") + .encode() + .expect("root bytes"), + ) + .id() +} + +/// Publishes one pack holding each object as its own group, so a retained pack +/// can hold one reached and one unreached record. +fn publish_groups(f: &Fixture, objects: &[FinalizedObject]) -> (i64, Vec) { + let capacities = StorageCapacities::from_policy(f.policy).expect("capacities"); + let mut workspace = CompressionWorkspace::new().expect("workspace"); + let mut profile = SaveProfile::default(); + let groups: Vec<_> = objects + .iter() + .map(|object| { + let record = encode_full( + object.canonical(), + object.role(), + &capacities, + &mut workspace, + &mut profile, + ) + .expect("record"); + build_group(PackLane::WholeFile, &[record.record], None).expect("group") + }) + .collect(); + let body = assemble(PackLane::WholeFile, &groups).expect("body"); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .expect("reserve"); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body.clone()), + )], + objects: objects + .iter() + .enumerate() + .map(|(group, object)| ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: group, + record_number: 0, + }) + .collect(), + ..Publication::default() + }) + .expect("publication"); + (reserved.first_pack_id, body) +} + +/// Publishes `target` as a PREFIX record over `base`, asserting the encoder +/// kept the delta (a full record would hide the base from the mark). +fn prefix_record(f: &Fixture, target: &FinalizedObject, base: &FinalizedObject, base_id: ObjectId) { + let capacities = StorageCapacities::from_policy(f.policy).expect("capacities"); + let record = encode_prefix( + target.canonical(), + target.role(), + base_id, + raw_payload(base.canonical(), base.role()).expect("payload"), + &capacities, + &mut CompressionWorkspace::new().expect("workspace"), + &mut SaveProfile::default(), + ) + .expect("prefix"); + assert_eq!( + record.base, + Some(base_id), + "the fixture needs a PREFIX record, not a full one" + ); + f.record(target, record.lane, record.record, Vec::new()); +} + +// ---- fixture history ------------------------------------------------------ + +fn initialize(handles: &PostgresHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: scope_for_seed(SCOPE_SEED).object(), + profile: profile_id(), + genesis_root: genesis, + }) + .expect("stack") +} + +fn fork(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .expect("fork"); +} + +fn expected_root( + handles: &PostgresHandles, + branch: BranchId, +) -> (Option, LayerId, ObjectId) { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + (snapshot.head_commit, snapshot.base_layer, root) +} + +fn commit_request( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, +) -> CommitRequest { + let (head, base, root) = expected_root(handles, branch); + CommitRequest { + branch, + expected_head: head, + expected_base: base, + expected_root: root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + } +} + +fn commit( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, + logical: u64, +) -> CommitId { + let (head, base, root) = expected_root(handles, branch); + let meta = CommitMeta { + statistics: CommitStatistics { + logical, + ..CommitStatistics::ZERO + }, + ..CommitMeta::new(CommitVia::Files) + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: head, + expected_base: base, + expected_root: root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta, + force: false, + }) + .expect("commit") + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .expect("publish") + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +// ---- raw Store facts ------------------------------------------------------ + +#[derive(Clone, Copy, Debug, Default)] +struct Totals { + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, +} + +fn totals(schema: &Schema) -> Totals { + let row = schema + .client() + .query_one( + &format!( + "SELECT logical,canonical,stored,metadata FROM {}.store_totals WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("totals"); + Totals { + logical: u64::try_from(row.get::<_, i64>(0)).unwrap(), + canonical: u64::try_from(row.get::<_, i64>(1)).unwrap(), + stored: u64::try_from(row.get::<_, i64>(2)).unwrap(), + metadata: u64::try_from(row.get::<_, i64>(3)).unwrap(), + } +} + +#[derive(Debug)] +struct PackFacts { + total: u64, + live_bytes: u64, + dead_bytes: u64, + live: Vec, + dead: Vec, +} + +fn pack_facts(schema: &Schema, live_packs: &[i64]) -> PackFacts { + let rows = schema + .client() + .query( + &format!( + "SELECT pack_id, length FROM {}.pack ORDER BY pack_id", + schema.selection.schema + ), + &[], + ) + .expect("packs"); + let mut facts = PackFacts { + total: 0, + live_bytes: 0, + dead_bytes: 0, + live: Vec::new(), + dead: Vec::new(), + }; + for row in rows { + let pack: i64 = row.get(0); + let length = u64::try_from(row.get::<_, i64>(1)).unwrap(); + facts.total += length; + if live_packs.contains(&pack) { + facts.live.push(pack); + facts.live_bytes += length; + } else { + facts.dead.push(pack); + } + } + facts.dead_bytes = facts.total - facts.live_bytes; + facts +} + +fn pack_of(schema: &Schema, id: ObjectId) -> Option { + let object = id.as_bytes(); + let row = schema + .client() + .query_opt( + &format!( + "SELECT pack_id FROM {}.object_location WHERE object_id=$1", + schema.selection.schema + ), + &[&object.as_slice()], + ) + .expect("object location"); + row.map(|row| row.get(0)) +} + +fn object_count(schema: &Schema, id: ObjectId) -> i64 { + let object = id.as_bytes(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.object_location WHERE object_id=$1", + schema.selection.schema + ), + &[&object.as_slice()], + ) + .expect("object count") + .get(0) +} + +fn pack_length(schema: &Schema, pack: i64) -> u64 { + let length = schema + .client() + .query_one( + &format!( + "SELECT length FROM {}.pack WHERE pack_id=$1", + schema.selection.schema + ), + &[&pack], + ) + .expect("pack length") + .get::<_, i64>(0); + u64::try_from(length).unwrap() +} + +fn pack_row(schema: &Schema, pack: i64) -> i64 { + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.pack WHERE pack_id=$1", + schema.selection.schema + ), + &[&pack], + ) + .expect("pack row") + .get(0) +} + +fn branch_row(schema: &Schema, id: BranchId) -> i64 { + let bytes = id.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.branch WHERE branch_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("branch row") + .get(0) +} + +fn stack_row(schema: &Schema, id: LayerStackId) -> i64 { + let bytes = id.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.layer_stack WHERE layer_stack_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("stack row") + .get(0) +} + +fn retired_branches(schema: &Schema) -> Vec { + let rows = schema + .client() + .query( + &format!( + "SELECT branch_id FROM {}.branch_retired ORDER BY branch_id", + schema.selection.schema + ), + &[], + ) + .expect("retired branches"); + rows.iter() + .map(|row| { + let bytes: Vec = row.get(0); + BranchId::from_slice(&bytes).unwrap() + }) + .collect() +} + +fn retired_stacks(schema: &Schema) -> i64 { + schema.count("layer_stack_retired") +} + +fn digest_rows(schema: &Schema, digest: &[u8; 32]) -> i64 { + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.file_digest WHERE digest=$1", + schema.selection.schema + ), + &[&digest.as_slice()], + ) + .expect("digest rows") + .get(0) +} + +fn epoch(schema: &Schema, stack: LayerStackId) -> i64 { + let bytes = stack.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT epoch FROM {}.project_epoch WHERE layer_stack_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("project epoch") + .get(0) +} + +fn epoch_rows(schema: &Schema) -> i64 { + schema.count("project_epoch") +} + +fn next_serial(schema: &Schema) -> i64 { + schema.scalar("SELECT next_history_serial FROM {s}.history_meta WHERE id=1") +} + +fn inode_block_bytes(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(octet_length(body)),0) FROM {s}.inode_value_block"), + ) + .unwrap() +} + +fn sum_pack_lengths(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack")).unwrap() +} + +fn sum_canonical(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"), + ) + .unwrap() +} + +fn sum_metadata_packs(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"), + ) + .unwrap() +} + +fn sum_history_logical(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(logical),0)::bigint FROM {s}.\"commit\"") + + schema.scalar("SELECT COALESCE(SUM(logical),0)::bigint FROM {s}.layer"), + ) + .unwrap() +} + +fn generation(handles: &PostgresHandles) -> u64 { + handles.history.incarnation() +} + +fn db_generation(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT catalog_incarnation FROM {s}.history_meta WHERE id=1")) + .unwrap() +} + +fn located(handles: &PostgresHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn audit(handles: &PostgresHandles) -> Vec { + let mut out = Vec::new(); + handles + .diagnose( + Options { + quick: false, + ..Options::default() + }, + &mut |observation| out.push(observation), + ) + .expect("diagnose"); + out +} + +fn assert_doctor_clean(handles: &PostgresHandles) { + let out = audit(handles); + assert!( + !out.iter() + .any(|observation| matches!(observation.outcome, Outcome::Fail | Outcome::Unknown)), + "the compacted Store must pass the full doctor: {out:?}" + ); +} + +/// §9/§8: after a settle no op row, dead set, intent or scratch schema remains +/// and every `store_totals` row equals the sum of the rows it summarises. +fn assert_settled(schema: &Schema, op: u8) { + assert_eq!(schema.count("gc_op"), 0, "gc_op is cleared on settle"); + assert_eq!(schema.count("gc_dead"), 0, "gc_dead is cleared on settle"); + assert_eq!( + schema.count_where("SELECT count(*) FROM {s}.gc_intent WHERE kind='compaction'"), + 0, + "the compaction intent is released" + ); + assert!( + !schema.scratch_exists(op), + "the recorded scratch schema is dropped" + ); + let logical = schema.scalar("SELECT logical FROM {s}.store_totals WHERE id=1"); + let sum = schema.scalar( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.layer)", + ); + assert_eq!(logical, sum, "store_totals.logical tracks the history rows"); + let canonical = schema.scalar("SELECT canonical FROM {s}.store_totals WHERE id=1"); + let canonical_sum = + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"); + assert_eq!(canonical, canonical_sum, "store_totals.canonical"); + let stored = schema.scalar("SELECT stored FROM {s}.store_totals WHERE id=1"); + let stored_sum = schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack"); + assert_eq!(stored, stored_sum, "store_totals.stored"); + let metadata = schema.scalar("SELECT metadata FROM {s}.store_totals WHERE id=1"); + let metadata_sum = + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"); + assert_eq!(metadata, metadata_sum, "store_totals.metadata"); +} + +/// AC-43: the Store is usable at `G + 1` with its totals, the recreated +/// `pack_immutable` trigger and function, the same owner and cleared hints. +fn assert_phase_c_settled(schema: &Schema, op: u8) { + assert_settled(schema, op); + let trigger = schema.scalar( + "SELECT count(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='{s}' AND t.tgname='pack_immutable' AND NOT t.tgisinternal", + ); + assert_eq!(trigger, 1, "the swap recreates pack_immutable"); + let function = schema.scalar( + "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='{s}' AND p.proname='pack_immutable'", + ); + assert_eq!(function, 1, "the trigger is bound to the schema's function"); + let mut client = schema.client(); + let refused = client.batch_execute(&format!( + "UPDATE {schema}.pack SET length=length WHERE pack_id=(SELECT min(pack_id) FROM {schema}.pack)", + schema = schema.selection.schema + )); + assert!(refused.is_err(), "the trigger still refuses an update"); + let owner: String = client + .query_one( + &format!( + "SELECT tableowner FROM pg_tables WHERE schemaname='{}' AND tablename='pack'", + schema.selection.schema + ), + &[], + ) + .expect("owner") + .get(0); + let current: String = client + .query_one("SELECT current_user", &[]) + .expect("current user") + .get(0); + assert_eq!(owner, current, "the swapped table keeps its owner"); + assert_eq!( + schema.count("binding_hint_block"), + 0, + "Phase C clears the binding hints" + ); + assert_eq!( + schema.scalar( + "SELECT count(*) FROM information_schema.tables WHERE table_schema='{s}' AND table_name='inode'" + ), + 0, + "no inode column on PostgreSQL" + ); +} + +// ---- cases ---------------------------------------------------------------- + +/// The §6.4 vector: two live stacks, an absent head and a retirement, exactly +/// the T12.1 SQLite fixture. One literal must hold on both backends. +#[test] +fn the_live_heads_digest_vector_is_the_sqlite_literal() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "digest"); + // Out-of-order creation: the digest orders by identity, not creation. + let root_9 = tree(&f, whole_file(&f, b"digest stack nine content"), 9); + let stack_9 = initialize(&f.handles, 9, root_9); + let root_3 = tree(&f, whole_file(&f, b"digest stack three content"), 10); + let stack_3 = initialize(&f.handles, 3, root_3); + let root_7 = tree(&f, whole_file(&f, b"digest retired stack content"), 11); + let stack_7 = initialize(&f.handles, 7, root_7); + f.handles + .history + .mark_stack_deleted(stack_7.id, 7_000) + .unwrap(); + + let branch_9 = support::branch_id(9); + fork(&f.handles, stack_9.id, branch_9, 9); + let branch_3 = support::branch_id(3); + fork(&f.handles, stack_3.id, branch_3, 3); + let branch_5 = support::branch_id(5); + fork(&f.handles, stack_9.id, branch_5, 5); + let commit_root = tree(&f, whole_file(&f, b"digest commit content"), 12); + let commit_id = commit(&f.handles, branch_3, commit_root, 90); + assert_eq!( + hex(commit_id.as_slice()), + sqlite_commit_literal(), + "the same content-addressed Commit on both backends" + ); + + // branch_5 has never committed: an absent head is hashed as 33 zero bytes. + let with_empty_head = f.handles.history.mark().unwrap().live_heads_digest(); + assert_eq!( + with_empty_head, + f.handles.history.mark().unwrap().live_heads_digest(), + "the fixed fixture pins one digest" + ); + assert_eq!( + hex(&with_empty_head), + sqlite_literal("absentHeadDigest"), + "an absent head is the SQLite golden literal" + ); + // A retired Branch is absent, not zeroed. + f.handles + .history + .mark_branch_removed(branch_5, 5_000) + .unwrap(); + let retired_branch = f.handles.history.mark().unwrap().live_heads_digest(); + assert_ne!(with_empty_head, retired_branch); + assert_eq!( + hex(&retired_branch), + sqlite_literal("retiredBranchDigest"), + "a retired Branch row is absent, not zeroed" + ); + assert_doctor_clean(&f.handles); + assert_eq!(retired_stacks(&f.schema), 1); +} + +struct FixtureIds { + stacks_live: Vec, + stacks_collected: Vec, + branches_live: Vec, + branches_retained_retired: Vec, + branches_collected: Vec, + commits_retained: Vec, + commits_collected: Vec, + layers_retained: Vec, + layers_collected: Vec, + objects_live: Vec, + objects_dead: Vec, + delta_bases_sorted: Vec, + delta_chain: Vec<(ObjectId, i64)>, + delta_target: ObjectId, + live_group: ObjectId, + dead_group: ObjectId, + shared_pack: i64, + shared_pack_length: u64, + shared_dead_group: u64, + digest_collected: [u8; 32], + digest_retained: [u8; 32], +} + +/// The T12.1 rebuild fixture, backend-for-backend: retained history with a +/// deleted project, a retained retired Branch, a collected retired Branch, a +/// depth-2 delta chain held only as bases, one retained pack with an unreached +/// group, pooled inode values and the two digest roots. +fn rebuild_fixture(f: &Fixture) -> FixtureIds { + // The dead packs: unreferenced whole files, each in its own pack. + let mut objects_dead = Vec::new(); + for index in 0..12u8 { + let payload = pseudo_bytes(4096, 0xA0 + index); + objects_dead.push(whole_file(f, &payload)); + } + let retained_chunked_payload = pseudo_bytes(131_072, 0x77); + let retained_chunked = chunked_file(f, &retained_chunked_payload); + let collected_chunked_payload = pseudo_bytes(131_072, 0x88); + let collected_chunked = chunked_file(f, &collected_chunked_payload); + let digest_collected = *blake3::hash(&collected_chunked_payload).as_bytes(); + let digest_retained = *blake3::hash(&retained_chunked_payload).as_bytes(); + + // A depth-2 delta chain: base2 <- base1 <- target. Only the target is a + // retained root, so both bases are kept only as delta bases. + let base2_payload = pseudo_bytes(512, 0x11); + let base1_payload = [base2_payload.as_slice(), b" v2"].concat(); + let target_payload = [base1_payload.as_slice(), b" v3"].concat(); + let base2 = whole_file(f, &base2_payload); + let base2_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base2_payload).unwrap(), + ) + .unwrap(); + let base1_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base1_payload).unwrap(), + ) + .unwrap(); + prefix_record(f, &base1_object, &base2_object, base2); + let base1 = base1_object.id(); + let target_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&target_payload).unwrap(), + ) + .unwrap(); + prefix_record(f, &target_object, &base1_object, base1); + let delta_target = target_object.id(); + + // One retained pack holding one reached and one unreached record. + let live_payload = pseudo_bytes(256, 0x55); + let dead_payload = pseudo_bytes(256, 0x66); + let live_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&live_payload).unwrap(), + ) + .unwrap(); + let dead_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&dead_payload).unwrap(), + ) + .unwrap(); + let (shared_pack, shared_body) = publish_groups(f, &[live_group.clone(), dead_group.clone()]); + let shared_header = layout::parse_header(&shared_body).unwrap(); + let dead_view = layout::group_view(&shared_body, shared_header, 1).unwrap(); + let shared_dead_group = (dead_view.end - dead_view.start) as u64; + let shared_pack_length = pack_length(&f.schema, shared_pack); + + // Content objects of the trees that are expected to be collected. + let collected_content_a4 = collected_chunked; + let collected_content_b0 = whole_file(f, b"deleted project genesis content"); + let collected_content_b1 = whole_file(f, b"deleted project commit content"); + let content_c3 = whole_file(f, b"live branch commit content"); + + let root_a0 = tree(f, live_group.id(), 0); + let root_a1 = tree(f, delta_target, 1); + let root_a2 = tree(f, retained_chunked, 2); + let root_a3 = tree(f, content_c3, 3); + let root_a4 = tree(f, collected_content_a4, 4); + let root_b0 = tree(f, collected_content_b0, 5); + let root_b1 = tree(f, collected_content_b1, 6); + + let stack_a = initialize(&f.handles, 1, root_a0); + let branch_published = support::branch_id(1); + fork(&f.handles, stack_a.id, branch_published, 1); + let commit_1 = commit(&f.handles, branch_published, root_a1, 120); + let layer_1 = publish(&f.handles, stack_a.id, branch_published); + let commit_2 = commit(&f.handles, branch_published, root_a2, 240); + let layer_2 = publish(&f.handles, stack_a.id, branch_published); + + let branch_live = support::branch_id(2); + fork(&f.handles, stack_a.id, branch_live, 2); + let commit_3 = commit(&f.handles, branch_live, root_a3, 360); + + let branch_retired = support::branch_id(3); + fork(&f.handles, stack_a.id, branch_retired, 3); + let commit_4 = commit(&f.handles, branch_retired, root_a4, 70); + + let stack_b = initialize(&f.handles, 2, root_b0); + let branch_b = support::branch_id(4); + fork(&f.handles, stack_b.id, branch_b, 4); + let commit_5 = commit(&f.handles, branch_b, root_b1, 40); + let layer_b = publish(&f.handles, stack_b.id, branch_b); + + // A retired Branch a retained Layer still names keeps its row and marker. + f.handles + .history + .mark_branch_removed(branch_published, 1_000) + .unwrap(); + // A retired Branch nothing retained reaches is collected whole. + f.handles + .history + .mark_branch_removed(branch_retired, 1_500) + .unwrap(); + // A deleted project goes whole, marker included. + f.handles + .history + .mark_stack_deleted(stack_b.id, 2_000) + .unwrap(); + + // AC-18: one digest root no retained root reaches, one it does. + f.handles + .history + .record_file_digests(&[ + FileDigest { + root: *collected_chunked.as_bytes(), + length: collected_chunked_payload.len() as u64, + digest: digest_collected, + }, + FileDigest { + root: *retained_chunked.as_bytes(), + length: retained_chunked_payload.len() as u64, + digest: digest_retained, + }, + ]) + .unwrap(); + + let delta_chain: Vec<(ObjectId, i64)> = [base2, base1, delta_target] + .iter() + .map(|id| (*id, pack_of(&f.schema, *id).expect("a delta-chain pack"))) + .collect(); + let mut delta_bases_sorted = vec![base1, base2]; + delta_bases_sorted.sort(); + objects_dead.push(dead_group.id()); + objects_dead.extend([ + root_a4, + root_b0, + root_b1, + collected_content_a4, + collected_content_b0, + collected_content_b1, + ]); + let objects_live = vec![ + root_a0, + root_a1, + root_a2, + root_a3, + live_group.id(), + retained_chunked, + base2, + base1, + delta_target, + content_c3, + ]; + FixtureIds { + stacks_live: vec![stack_a.id], + stacks_collected: vec![stack_b.id], + branches_live: vec![branch_live], + branches_retained_retired: vec![branch_published], + branches_collected: vec![branch_retired, branch_b], + commits_retained: vec![commit_1, commit_2, commit_3], + commits_collected: vec![commit_4, commit_5], + layers_retained: vec![stack_a.head_layer, layer_1.id, layer_2.id], + layers_collected: vec![stack_b.head_layer, layer_b.id], + objects_live, + objects_dead, + delta_bases_sorted, + delta_chain, + delta_target, + live_group: live_group.id(), + dead_group: dead_group.id(), + shared_pack, + shared_pack_length, + shared_dead_group, + digest_collected, + digest_retained, + } +} + +// ---- tests ---------------------------------------------------------------- + +/// AC-02, AC-05, AC-17, AC-18, AC-25, AC-40, AC-41, AC-43, AC-52: the whole +/// T12.1 rebuild fixture on PostgreSQL. The `live_heads_digest` is the SQLite +/// golden literal; the §11 report is non-trivial; the depth-2 chain is kept +/// and its packs keep their ids and lengths; markers and digest rows follow +/// the retained set; the settled Store passes the full doctor. +#[test] +fn the_rebuild_report_and_markers_hold_on_postgresql() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "rebuild"); + let ids = rebuild_fixture(&f); + let schema = &f.schema; + let before = totals(schema); + let generation_before = generation(&f.handles); + let serial_before = next_serial(schema); + assert_eq!(epoch_rows(schema), 2, "one project_epoch row per stack"); + let epoch_before = epoch(schema, ids.stacks_live[0]); + + let mark = f.handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + f.handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + let literals = sqlite_literals("liveHeadsDigest"); + assert_eq!(literals.len(), 3, "the SQLite golden holds three digests"); + assert_eq!( + hex(&mark.live_heads_digest()), + literals[2], + "the rebuild digest is the SQLite literal: one vector on both backends" + ); + let live_packs = mark.objects().packs().to_vec(); + let live_lengths: Vec<(i64, u64)> = live_packs + .iter() + .map(|pack| (*pack, pack_length(schema, *pack))) + .collect(); + let facts = pack_facts(schema, &live_packs); + let inode_blocks = inode_block_bytes(schema); + assert_eq!(mark.objects().waste().inode_blocks_bytes(), inode_blocks); + assert_eq!( + mark.objects().waste().dead_in_retained_packs_bytes(), + ids.shared_dead_group, + "the unreached group of the retained pack is keptWaste" + ); + assert_eq!( + mark.objects().waste().delta_base_references(), + ids.delta_bases_sorted.as_slice(), + "a depth-2 chain reports both bases, in identity order" + ); + let expected_bases: Vec<[u8; 32]> = ids + .delta_bases_sorted + .iter() + .map(|id| *id.as_bytes()) + .collect(); + assert!( + mark.objects().waste().delta_bases_bytes() > 0, + "the bases are kept only as bases" + ); + for id in &ids.objects_live { + assert!(mark.objects().contains(*id), "live object {id} is retained"); + } + for id in &ids.objects_dead { + assert!(!mark.objects().contains(*id), "object {id} is collected"); + } + assert!( + facts.dead_bytes * 10 >= facts.total, + "the fixture must take the rebuild path: {} dead of {}", + facts.dead_bytes, + facts.total + ); + + let receipt = f.handles.compact(&request(0x72)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before + 1); + assert_eq!(receipt.dead_bytes_bound, facts.dead_bytes); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + assert!(receipt.estimate); + assert_eq!( + receipt.kept_waste.dead_in_retained_packs_bytes, + ids.shared_dead_group + ); + assert_eq!(receipt.kept_waste.inode_blocks_bytes, inode_blocks); + assert_eq!( + receipt.kept_waste.delta_base_references, expected_bases, + "PG keeps the delta bases and reports them" + ); + assert!(receipt.kept_waste.delta_bases_bytes > 0); + assert_eq!( + receipt.residue.bytes, 0, + "a successful settle leaves no residue" + ); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert_eq!( + receipt.backend_reusable, 0, + "PostgreSQL rewrites in place: no reusable free pages" + ); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "backendReusable" && zero.reason == "postgresql-in-place")); + + let reopened = schema.open(); + assert_eq!(generation(&reopened), generation_before + 1); + let after = totals(schema); + assert_eq!( + receipt.logical_discarded, + before.logical - after.logical, + "logicalDiscarded is the history logical delta" + ); + assert_eq!(before.logical - after.logical, 160); + assert_eq!(after.stored, sum_pack_lengths(schema)); + assert_eq!(after.canonical, sum_canonical(schema)); + assert_eq!(after.metadata, sum_metadata_packs(schema)); + assert_eq!(after.logical, sum_history_logical(schema)); + + // Live objects are readable, including the delta target through its chain. + let storage = Storage::new(reopened.storage.clone()).unwrap(); + let reader = storage.reader().unwrap(); + let canonicals = reader + .read_objects(&[ids.delta_target, ids.live_group]) + .unwrap(); + assert_eq!(canonicals.len(), 2); + assert!(!canonicals[0].is_empty()); + assert!(!canonicals[1].is_empty()); + for id in &ids.objects_live { + assert!(located(&reopened, *id), "live object {id} survives"); + } + for id in &ids.objects_dead { + assert!(!located(&reopened, *id), "dead object {id} is gone"); + } + for pack in &facts.dead { + assert_eq!(pack_row(schema, *pack), 0, "dead pack {pack} is removed"); + } + for (pack, length) in &live_lengths { + assert_eq!(pack_row(schema, *pack), 1, "live pack {pack} stays"); + assert_eq!( + pack_length(schema, *pack), + *length, + "AC-41: the swap copies the row and moves no pack bytes" + ); + } + // The retained pack keeps the unreached group: repacking is out of 2.0. + assert_eq!(pack_of(schema, ids.live_group), Some(ids.shared_pack)); + assert_eq!(object_count(schema, ids.live_group), 1); + assert_eq!(object_count(schema, ids.dead_group), 0); + assert_eq!(pack_length(schema, ids.shared_pack), ids.shared_pack_length); + // AC-17: a copied pack keeps its identity. + for (id, pack) in &ids.delta_chain { + assert_eq!( + pack_of(schema, *id), + Some(*pack), + "pack_id of {id} is preserved" + ); + } + + for id in &ids.commits_retained { + assert!( + reopened.history.commit(*id).unwrap().is_some(), + "commit {id}" + ); + } + for id in &ids.commits_collected { + assert!( + reopened.history.commit(*id).unwrap().is_none(), + "commit {id}" + ); + } + for id in &ids.layers_retained { + assert!(reopened.history.layer(*id).unwrap().is_some(), "layer {id}"); + } + for id in &ids.layers_collected { + assert!(reopened.history.layer(*id).unwrap().is_none(), "layer {id}"); + } + for id in &ids.stacks_live { + assert!(reopened.history.layer_stack(*id).unwrap().is_some()); + assert_eq!(stack_row(schema, *id), 1); + } + for id in &ids.stacks_collected { + assert!(reopened.history.layer_stack(*id).unwrap().is_none()); + assert_eq!(stack_row(schema, *id), 0); + } + for id in &ids.branches_live { + assert!(reopened.history.branch(*id).unwrap().is_some()); + } + assert_eq!(retired_branches(schema), ids.branches_retained_retired); + for id in &ids.branches_collected { + assert_eq!(branch_row(schema, *id), 0, "collected branch row {id}"); + } + assert_eq!(retired_stacks(schema), 0); + + // AC-18: a digest row survives only when a retained root reaches the root. + assert_eq!(digest_rows(schema, &ids.digest_retained), 1); + assert_eq!(digest_rows(schema, &ids.digest_collected), 0); + + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + mark.live_heads_digest(), + "the live-heads digest matches before and after" + ); + // AC-24/AC-39: the rebuild copies the epoch, never bumps it, and never + // lowers the serial counter. + assert_eq!(epoch(schema, ids.stacks_live[0]), epoch_before); + assert_eq!( + epoch_rows(schema), + 1, + "the collected project takes its epoch" + ); + assert!(next_serial(schema) >= serial_before); + // AC-43/AC-02: the settled PostgreSQL Store is usable at G + 1. + assert_phase_c_settled(schema, 0x72); +} + +/// AC-28, AC-43, AC-49 (Phase C arm): the swap is durably at `G + 1` when the +/// reply is lost. A save made after the swap is not a member of the persisted +/// dead set, so the resumed cleanup keeps it, and the settled Store passes the +/// full doctor. +#[test] +fn a_save_after_the_swap_survives_the_resumed_cleanup() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "late"); + let live = tree(&f, whole_file(&f, b"late live content"), 1); + let stack = initialize(&f.handles, 1, live); + let dead = saved_whole_file(&f.handles, b"a pack Phase C collects"); + let before = generation(&f.handles); + let error = f + .handles + .compact(&request_at(0x31, CompactFailpoint::PostgresAfterSwap)) + .unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome)); + let schema = &f.schema; + assert_eq!(db_generation(schema), before + 1); + assert_eq!(schema.count("gc_op"), 1, "the op still names Phase C"); + let phase: String = schema + .client() + .query_one( + &format!( + "SELECT phase FROM {}.gc_op WHERE id=1", + schema.selection.schema + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(phase, "swap", "AC-51: the op records its §8 phase"); + assert!( + schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0, + "the persisted dead set waits for Phase C" + ); + assert!( + !located(&f.handles, dead), + "the swap dropped the dead pack already" + ); + assert!( + schema.scalar("SELECT pid FROM {s}.gc_op WHERE id=1") > 0, + "the op records its owner backend" + ); + + // AC-28: a publication after the swap is not in `gc_dead`. + let late = saved_whole_file(&f.handles, b"written between swap and cleanup"); + assert!(located(&f.handles, late)); + let late_pack = pack_of(schema, late).expect("a late pack"); + let receipt = f.handles.compact(&request(0x32)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert!( + located(&f.handles, late), + "the late write survives the high-water bound" + ); + assert_eq!(pack_row(schema, late_pack), 1); + assert!(!located(&f.handles, dead)); + assert_phase_c_settled(schema, 0x31); + + let reopened = schema.open(); + assert_doctor_clean(&reopened); + assert!(located(&reopened, late)); + assert!(!located(&reopened, dead)); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); +} + +/// AC-21, AC-24, AC-39: the PostgreSQL `project_epoch` fence. Commits and +/// Layers leave it alone; a marker write bumps it once; a repeated marker is +/// `UpToDate` and does not bump it; compact copies it; a missing row refuses +/// every writer with `Conflict` and writes nothing. +#[test] +fn a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "epoch"); + let root = tree(&f, whole_file(&f, b"epoch genesis content"), 1); + let stack = initialize(&f.handles, 1, root); + let branch = support::branch_id(1); + fork(&f.handles, stack.id, branch, 1); + let commit_root = tree(&f, whole_file(&f, b"epoch commit content"), 2); + let _first = commit(&f.handles, branch, commit_root, 50); + let _layer = publish(&f.handles, stack.id, branch); + assert_eq!( + epoch(&f.schema, stack.id), + 0, + "a Commit or a published Layer does not bump the epoch" + ); + assert_eq!(epoch_rows(&f.schema), 1); + + // Capture the request while the Branch is still visible, then let the + // removal win: the fence refuses and leaves no Commit behind. + let candidate = whole_file(&f, b"epoch racing commit content"); + let stale = commit_request(&f.handles, branch, candidate); + let commits_before = f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""); + f.handles + .history + .mark_branch_removed(branch, 1_000) + .unwrap(); + assert_eq!( + epoch(&f.schema, stack.id), + 1, + "the marker bumped the epoch once" + ); + let refused = f.handles.history.commit_branch(&stale).unwrap_err(); + assert!( + matches!( + refused, + HistoryError::Missing(Missing::Branch) | HistoryError::Conflict("project epoch") + ), + "{refused:?}" + ); + assert_eq!( + f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""), + commits_before, + "the refused write left no Commit" + ); + + // A live project's marker write bumps it again; a repeat is not a write. + let removal = || PathRemovalRequest { + stack: stack.id, + path: NormalizedPath::new("file-1").unwrap().as_bytes().to_vec(), + removed_ms: 2_000, + }; + let outcome = f.handles.history.remove_path(&removal()).unwrap(); + assert_eq!(outcome.kind, PathRemovalKind::Removed); + assert_eq!(epoch(&f.schema, stack.id), 2); + let repeat = f.handles.history.remove_path(&removal()).unwrap(); + assert_eq!(repeat.kind, PathRemovalKind::UpToDate); + assert_eq!( + epoch(&f.schema, stack.id), + 2, + "UpToDate is not a marker write" + ); + + // AC-24/AC-39: compaction copies the epoch and never lowers the serial. + let serial = next_serial(&f.schema); + let receipt = f.handles.compact(&request(0x51)).unwrap(); + assert_eq!( + receipt.path, + CompactPath::None, + "a live Store takes the none path" + ); + assert_eq!(epoch(&f.schema, stack.id), 2); + assert_eq!(epoch_rows(&f.schema), 1); + assert!(next_serial(&f.schema) >= serial); + assert_settled(&f.schema, 0x51); + + // No epoch row: every writer refuses `Conflict`, never "unchanged". + let branch_2 = support::branch_id(2); + fork(&f.handles, stack.id, branch_2, 2); + f.schema + .client() + .batch_execute(&format!( + "DELETE FROM {}.project_epoch WHERE layer_stack_id=decode('{}','hex')", + f.schema.selection.schema, + hex(stack.id.as_slice()) + )) + .unwrap(); + let candidate = whole_file(&f, b"epoch missing row content"); + let stale = commit_request(&f.handles, branch_2, candidate); + let before = f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""); + let refused = f.handles.history.commit_branch(&stale).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!( + f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""), + before, + "the missing fence row wrote nothing" + ); +} + +/// AC-01, AC-03, AC-28, AC-43: the PostgreSQL failpoint matrix. Each named +/// §8 point aborts the round, then the next `store.compact` resumes it: `mark` +/// and `build` leave the old Store at `G` and re-mark, `swap` leaves the new +/// Store at `G + 1` and finishes Phase C, `cleanup` leaves nothing to resume. +/// Every arm settles with the same heads digest and a clean doctor. +#[test] +fn every_postgresql_failpoint_leaves_a_resumable_leftover() { + let Some(url) = url() else { return }; + for (tag, point, op, phase, scratch) in [ + ( + "cmark", + CompactFailpoint::PostgresAfterMark, + 0x61u8, + Some("mark"), + false, + ), + ( + "cbuild", + CompactFailpoint::PostgresAfterBuild, + 0x62, + Some("build"), + true, + ), + ( + "cswap", + CompactFailpoint::PostgresAfterSwap, + 0x63, + Some("swap"), + false, + ), + ( + "ccleanup", + CompactFailpoint::PostgresAfterCleanup, + 0x64, + None, + false, + ), + ] { + let f = Fixture::new(&url, tag); + let live = tree(&f, whole_file(&f, b"crash matrix live content"), 1); + let stack = initialize(&f.handles, 1, live); + let dead = saved_whole_file(&f.handles, b"crash matrix dead content"); + let before = generation(&f.handles); + let digest = f.handles.history.mark().unwrap().live_heads_digest(); + let error = f.handles.compact(&request_at(op, point)).unwrap_err(); + assert!( + matches!(error, HistoryError::UnknownOutcome), + "{tag}: {error:?}" + ); + let swapped = matches!( + point, + CompactFailpoint::PostgresAfterSwap | CompactFailpoint::PostgresAfterCleanup + ); + let expected = if swapped { before + 1 } else { before }; + assert_eq!(db_generation(&f.schema), expected, "{tag}: generation"); + assert_eq!( + located(&f.handles, dead), + !swapped, + "{tag}: the dead pack moves only at the swap" + ); + assert!(located(&f.handles, live), "{tag}: the live object stays"); + assert_eq!( + f.schema.scratch_exists(op), + scratch, + "{tag}: the scratch schema state" + ); + if let Some(phase) = phase { + let recorded: String = f + .schema + .client() + .query_one( + &format!( + "SELECT phase FROM {}.gc_op WHERE id=1", + f.schema.selection.schema + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(recorded, phase, "{tag}: the op records its phase"); + } + // The next call either adopts the recorded round or starts clean. + let receipt = f.handles.compact(&request(op + 1)).unwrap(); + assert_eq!(receipt.generation_after, before + 1, "{tag}: installed"); + if point == CompactFailpoint::PostgresAfterCleanup { + assert_eq!( + receipt.path, + CompactPath::None, + "{tag}: the lost cleanup reply left a clean Store" + ); + } else { + assert_eq!( + receipt.path, + CompactPath::Rebuild, + "{tag}: the adopted round rebuilds" + ); + } + assert!(!located(&f.handles, dead), "{tag}: collected"); + assert!(located(&f.handles, live), "{tag}: retained"); + assert_settled(&f.schema, op); + let reopened = f.schema.open(); + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + digest, + "{tag}: the heads digest is unchanged" + ); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); + } +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs b/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs index c0fae1e0fe..49cdb04151 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs @@ -13,6 +13,9 @@ use std::{ time::{Duration, Instant}, }; +#[path = "pg_doctor/conninfo.rs"] +mod conninfo; + #[test] fn tcp_peer_that_stalls_authentication_is_bounded_and_closed() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); @@ -152,10 +155,17 @@ fn deadline_driver_reads_one_view_and_ordinary_sync_writes_still_work() { "{connection:?}" ); drop(writer); - let fallback = PostgresSelection::new( - url.replace("127.0.0.1:", "missing.invalid,127.0.0.1:"), - &schema, + let fallback_url = conninfo::with_failing_first_host(&url); + assert_ne!( + fallback_url, url, + "failover fixture must actually rewrite the conninfo host" ); + let fallback = PostgresSelection::new(&fallback_url, &schema); + let mut attempted = Vec::new(); + let mut recording = |host: &str, port: u16, until: Instant| { + attempted.push((host.to_owned(), port)); + fixture_resolver(host, port, until) + }; let mut fallback_observations = Vec::new(); diagnostic::postgresql_connection( &fallback, @@ -164,10 +174,14 @@ fn deadline_driver_reads_one_view_and_ordinary_sync_writes_still_work() { deadline: Some(Instant::now() + Duration::from_secs(3)), ..Options::default() }, - &mut fixture_resolver, + &mut recording, &mut |o| fallback_observations.push(o), ) .unwrap(); + assert!( + attempted.iter().any(|(host, _)| host == "missing.invalid"), + "failover case must first attempt the unresolvable host: {attempted:?}" + ); assert!( fallback_observations .iter() @@ -188,12 +202,9 @@ fn lock_wait_cancellation_releases_backend_work_and_snapshot() { }; let schema = format!("doctor_cancel_{}", std::process::id()); let application = format!("doctor_cancel_{}", std::process::id()); - // URL syntax is not reserialized or logged; an explicit startup option - // marks the diagnostic session for independent backend observation. - let diagnostic_url = format!( - "{url}{}application_name={application}", - if url.contains('?') { "&" } else { "?" } - ); + // The connection string is not reserialized or logged; an explicit startup + // option marks the diagnostic session for independent backend observation. + let diagnostic_url = conninfo::with_option(&url, "application_name", &application); let selection = PostgresSelection::new(&diagnostic_url, &schema); let authority = HistoryCatalogConfig { binding_key: b"doctor-pg-cancel".to_vec(), @@ -250,12 +261,9 @@ fn cancellation_signal_without_deadline_releases_backend_work_and_snapshot() { }; let schema = format!("doctor_signal_{}", std::process::id()); let application = format!("doctor_signal_{}", std::process::id()); - // URL syntax is not reserialized or logged; an explicit startup option - // marks the diagnostic session for independent backend observation. - let diagnostic_url = format!( - "{url}{}application_name={application}", - if url.contains('?') { "&" } else { "?" } - ); + // The connection string is not reserialized or logged; an explicit startup + // option marks the diagnostic session for independent backend observation. + let diagnostic_url = conninfo::with_option(&url, "application_name", &application); let selection = PostgresSelection::new(&diagnostic_url, &schema); let authority = HistoryCatalogConfig { binding_key: b"doctor-pg-cancel".to_vec(), @@ -343,10 +351,7 @@ fn denied_signature_read_is_unknown_instead_of_corruption() { PostgresHandles::create(&selection, StoragePolicy::frozen_default(), &authority).unwrap(); let mut admin = postgres::Client::connect(&url, postgres::NoTls).unwrap(); admin.batch_execute(&format!("CREATE ROLE {role}; GRANT USAGE ON SCHEMA {schema} TO {role}; GRANT SELECT ON ALL TABLES IN SCHEMA {schema} TO {role}")).unwrap(); - let restricted_url = format!( - "{url}{}options=-c%20role%3D{role}", - if url.contains('?') { "&" } else { "?" } - ); + let restricted_url = conninfo::with_option(&url, "options", &format!("-c role={role}")); let restricted = PostgresSelection::new(&restricted_url, &schema); let reader = PostgresHandles::open_diagnostic( &restricted, diff --git a/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs b/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs index c7bdcbdf6b..70bca616a0 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs @@ -91,9 +91,7 @@ fn namespace_phase_transport_loss_is_unknown_and_releases_its_snapshot() { let before = f.fingerprint(); let mut selection = f.selection.clone(); let application = f.selection.schema.clone(); - selection - .url - .push_str(&format!("&application_name={application}")); + selection.url = conninfo::with_option(&selection.url, "application_name", &application); let reader = PostgresHandles::open_diagnostic( &selection, BINDING, @@ -230,9 +228,8 @@ fn configured_unix_then_tcp_fallback_preserves_saved_options() { "{out:?}" ); let mut random = f.selection.clone(); - random - .url - .push_str("&load_balance_hosts=random&host=127.0.0.1,127.0.0.1"); + random.url = conninfo::with_option(&random.url, "load_balance_hosts", "random"); + random.url = conninfo::with_option(&random.url, "host", "127.0.0.1,127.0.0.1"); let out = connection(&random, BINDING); assert!( has(&out, "store.connection.configuration", Outcome::Unknown), diff --git a/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs b/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs index 4b923ef872..4c2d3ac398 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs @@ -32,7 +32,9 @@ fn supported_amcheck_reports_each_relation_and_missing_extension_is_unknown() { assert_eq!(actual, expected); drop(reader); assert_eq!(before, f.fingerprint()); - let missing = std::env::var("LAYERFS_TEST_POSTGRES_MISSING_URL").unwrap(); + let Some(missing) = auxiliary_url("LAYERFS_TEST_POSTGRES_MISSING_URL") else { + return; + }; let mut f = Fixture::at("no_extension", missing).unwrap(); let reader = f.reader(); let (result, out) = audit(&reader, false); @@ -107,10 +109,12 @@ fn denied_amcheck_and_non_btree_indexes_leave_explicit_incomplete_coverage() { } #[test] fn physical_standby_uses_supported_online_checks_without_primary_writes() { + let Some(url) = auxiliary_url("LAYERFS_TEST_POSTGRES_REPLICA_URL") else { + return; + }; let Some(f) = Fixture::new("replica") else { return; }; - let url = std::env::var("LAYERFS_TEST_POSTGRES_REPLICA_URL").unwrap(); let mut monitor = postgres::Client::connect(&url, postgres::NoTls).unwrap(); assert!(monitor .query_one("SELECT pg_is_in_recovery()", &[]) @@ -173,9 +177,7 @@ fn oversized_physical_payloads_are_rejected_before_client_materialization() { let port = config.get_ports()[0]; let proxy = proxy::Proxy::new(port); let selected = layerfs_persistence::PostgresSelection::new( - f.selection - .url - .replace(&format!(":{port}/"), &format!(":{}/", proxy.port)), + conninfo::with_local_proxy(&f.selection.url, proxy.port), &f.selection.schema, ); let reader = PostgresHandles::open_diagnostic( @@ -189,6 +191,10 @@ fn oversized_physical_payloads_are_rejected_before_client_materialization() { let (result, out) = audit(&reader, false); result.unwrap(); assert!(has(&out, "store.packs", Outcome::Fail), "{out:?}"); + assert!( + proxy.bytes() > 0, + "the diagnostic bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "one-megabyte corrupted body crossed wire: {} bytes", @@ -377,9 +383,9 @@ fn pinned_catalog_replacement_is_rejected_as_authority_change() { fn oversized_authority_and_saved_ddl_stay_on_server_during_admission() { use std::str::FromStr; for (tag,mutation) in [("binding","ALTER TABLE {s}.history_meta DROP CONSTRAINT history_meta_binding_key_check; UPDATE {s}.history_meta SET binding_key=decode(repeat('00',1048576),'hex')"),("source","UPDATE {s}.history_meta SET schema_source=repeat('x',1048576)"),("definition","UPDATE {s}.history_meta SET schema_definition=repeat('x',1048576)")]{ - let Some(mut f)=Fixture::new(tag)else{return};f.admin.batch_execute(&mutation.replace("{s}",&f.selection.schema)).unwrap();let before=f.fingerprint();let config=postgres::Config::from_str(&f.selection.url).unwrap();let port=config.get_ports()[0];let proxy=proxy::Proxy::new(port);let selection=layerfs_persistence::PostgresSelection::new(f.selection.url.replace(&format!(":{port}/"),&format!(":{}/",proxy.port)),&f.selection.schema); + let Some(mut f)=Fixture::new(tag)else{return};f.admin.batch_execute(&mutation.replace("{s}",&f.selection.schema)).unwrap();let before=f.fingerprint();let config=postgres::Config::from_str(&f.selection.url).unwrap();let port=config.get_ports()[0];let proxy=proxy::Proxy::new(port);let selection=layerfs_persistence::PostgresSelection::new(conninfo::with_local_proxy(&f.selection.url,proxy.port),&f.selection.schema); if tag=="binding"{let out=connection(&selection,BINDING);assert!(has(&out,"store.connection.identity",Outcome::Fail),"{out:?}");}else{assert!(PostgresHandles::open_diagnostic(&selection,BINDING,[71;32],Some(Instant::now()+Duration::from_secs(5)),&mut resolver).is_err());} - assert!(proxy.bytes()<512*1024,"oversized metadata crossed wire: {} bytes",proxy.bytes());drop(proxy);assert_eq!(before,f.fingerprint()); + assert!(proxy.bytes()>0,"oversized metadata bypassed the byte-counting proxy (direct connection?)");assert!(proxy.bytes()<512*1024,"oversized metadata crossed wire: {} bytes",proxy.bytes());drop(proxy);assert_eq!(before,f.fingerprint()); } } @@ -412,9 +418,7 @@ fn oversized_pack_descriptor_digest_is_rejected_before_wire_extraction() { .get_ports()[0]; let proxy = proxy::Proxy::new(port); let selected = layerfs_persistence::PostgresSelection::new( - f.selection - .url - .replace(&format!(":{port}/"), &format!(":{}/", proxy.port)), + conninfo::with_local_proxy(&f.selection.url, proxy.port), &f.selection.schema, ); let reader = PostgresHandles::open_diagnostic( @@ -432,6 +436,10 @@ fn oversized_pack_descriptor_digest_is_rejected_before_wire_extraction() { has(&out, "store.objects", Outcome::Fail), "locator must independently reject invalid pack descriptor: {out:?}" ); + assert!( + proxy.bytes() > 0, + "the pack descriptor bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "oversized digest crossed wire: {} bytes", @@ -467,15 +475,7 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { .get_ports()[0]; let proxied = |port: u16| { layerfs_persistence::PostgresSelection::new( - f.selection.url.replace( - &format!( - ":{}/", - postgres::Config::from_str(&f.selection.url) - .unwrap() - .get_ports()[0] - ), - &format!(":{port}/"), - ), + conninfo::with_local_proxy(&f.selection.url, port), &f.selection.schema, ) }; @@ -486,6 +486,10 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { has(&out, "store.connection.identity", Outcome::Fail), "{out:?}" ); + assert!( + proxy.bytes() > 0, + "the preflight scalar bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "corrupt scalar crossed preflight wire" @@ -501,6 +505,10 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { &mut resolver ) .is_err()); + assert!( + proxy.bytes() > 0, + "the admitted-open scalar bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "corrupt scalar crossed admitted-open wire" diff --git a/engine/crates/layerfs-persistence/tests/postgresql_status.rs b/engine/crates/layerfs-persistence/tests/postgresql_status.rs new file mode 100644 index 0000000000..6f4f54a1be --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_status.rs @@ -0,0 +1,272 @@ +//! #41 T10.5: the PostgreSQL `store.status` compaction projection. +//! +//! `store.status` reads the Store-side `gc_op` row with its compaction +//! `gc_intent` in one status query — no Store open, schema mutation or +//! artifact read — and projects the same shape as the SQLite sidecar: the op +//! `{id, name, phase, started, uncertain}` from the row, `state` from the +//! owner's liveness, and PostgreSQL's `mark`/`build`/`swap`/`cleanup` phases. +//! An idle Store (no row, or a run that settled and removed it) projects +//! `None`. The record's own fact `uncertain` stays true for every remaining +//! row; `state` is what separates a busy run from a leftover. +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=lfs_t105 +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::HistoryCatalog as _; +use layerfs_persistence::{ + postgresql_compact_status, CompactFailpoint, CompactRequest, CompactStatus, PostgresHandles, +}; +use layerfs_storage::Storage; +use support::pg::Schema; + +fn binding(tag: &str) -> Vec { + format!("postgresql-status-{tag}").into_bytes() +} + +fn whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &PostgresHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn request(op: u8, failpoint: Option) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint, + ..CompactRequest::default() + } +} + +/// The live §8 phases, in order, as `store.status` names them. +const PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + +/// Waits until the recorded owner backend is gone from `pg_stat_activity`. +fn wait_gone(admin: &mut postgres::Client, pid: i32, start: &str) { + for _ in 0..400 { + let live: bool = admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2)", + &[&pid, &start], + ) + .unwrap() + .get(0); + if !live { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend never ended"); +} + +#[test] +fn the_projection_reads_gc_op_gc_intent_and_owner_liveness() { + let Some(url) = support::pg::url() else { + return; + }; + let schema = Schema::new(url.clone(), "row"); + let _handles = schema.create(&binding("row"), [71; 32]); + let name = schema.name.clone(); + let s = &schema.name; + + // Idle: no compaction row, so no op and no phases. + let idle = postgresql_compact_status(&url, &name).unwrap(); + assert!(idle.is_none(), "an idle Store projects no op"); + + // A live owner: an open, non-idle backend is exactly what the §9 + // dead-owner rule treats as "still running the op". + let mut owner = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + owner.batch_execute("BEGIN; SELECT 1").unwrap(); + let (pid, start): (i32, String) = { + let row = owner + .query_one( + "SELECT pg_backend_pid(),backend_start::text FROM pg_stat_activity \ + WHERE pid=pg_backend_pid()", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + let op_id = [0xabu8; 16]; + let artifact = format!("lfs_gc_{}", "ab".repeat(16)); + let digest = [9u8; 32]; + let mut admin = schema.admin(); + admin + .execute( + &format!( + "INSERT INTO {s}.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,\ + started_ms,artifact,pid,backend_start) \ + VALUES (1,$1,'build',$2,$3,$4,$5,$6,$2,$7,$8)" + ), + &[ + &7i64, + &artifact, + &11i64, + &12i64, + &digest.to_vec(), + &1_700_000_000_000i64, + &i64::from(pid), + &start, + ], + ) + .unwrap(); + admin + .execute( + &format!( + "INSERT INTO {s}.gc_intent \ + (op_id,kind,scope_kind,owner_runtime,owner_boot,state,uncertain,phase,\ + generation,artifact,updated_ms) \ + VALUES ($1,'compaction','store',$2,$3,'live',FALSE,'build',7,$4,1700000000000)" + ), + &[&op_id.to_vec(), &"runtime-t105", &"boot-t105", &artifact], + ) + .unwrap(); + + let live = postgresql_compact_status(&url, &name) + .unwrap() + .expect("a live op is projected"); + assert_eq!(live.op_id, op_id, "the intent carries the op identity"); + assert_eq!(live.generation, 7); + assert_eq!(live.phase, "build"); + assert_eq!(live.started_ms, 1_700_000_000_000); + assert_eq!(live.owner_pid, Some(i64::from(pid))); + assert_eq!(live.owner_backend_start.as_deref(), Some(start.as_str())); + assert_eq!(live.owner_runtime.as_deref(), Some("runtime-t105")); + assert_eq!(live.owner_boot.as_deref(), Some("boot-t105")); + assert_eq!(live.intent_state.as_deref(), Some("live")); + assert_eq!(live.mark_serial, Some(11)); + assert_eq!(live.mark_pack_id, Some(12)); + assert_eq!(live.mark_digest.as_deref(), Some(&digest[..])); + assert!(live.alive, "a non-idle backend is a live owner"); + assert!(live.uncertain, "the remaining row is uncertain"); + let busy: CompactStatus = live.status(); + assert_eq!(busy.id, format!("op-{}", "ab".repeat(16))); + assert_eq!(busy.name, "store.compact"); + assert_eq!(busy.phase, "build"); + assert_eq!(busy.started, 1_700_000_000_000); + assert!(busy.uncertain); + assert_eq!(busy.state, "busy"); + assert_eq!(busy.phases, &PHASES[..]); + + // A gone owner leaves the same record as an uncertain leftover. + owner.close().unwrap(); + wait_gone(&mut admin, pid, &start); + let leftover = postgresql_compact_status(&url, &name) + .unwrap() + .expect("the row remains"); + assert!(!leftover.alive); + assert!(leftover.uncertain); + assert_eq!(leftover.op_id, op_id); + let gone = leftover.status(); + assert_eq!(gone.state, "unresolved"); + assert_eq!(gone.phase, "build"); + assert_eq!(gone.id, build_op(0xab)); + assert!(gone.uncertain); + + // A settled run removes the row: the idle projection is back. + admin + .batch_execute(&format!( + "DELETE FROM {s}.gc_op; DELETE FROM {s}.gc_intent WHERE kind='compaction'" + )) + .unwrap(); + assert!(postgresql_compact_status(&url, &name).unwrap().is_none()); +} + +fn build_op(byte: u8) -> String { + format!("op-{}", format!("{byte:02x}").repeat(16)) +} + +#[test] +fn every_postgresql_failpoint_projects_its_phase() { + let Some(url) = support::pg::url() else { + return; + }; + for (tag, point, phase, op) in [ + ("cmark", CompactFailpoint::PostgresAfterMark, "mark", 0x61u8), + ("cbuild", CompactFailpoint::PostgresAfterBuild, "build", 0x62), + ("cswap", CompactFailpoint::PostgresAfterSwap, "swap", 0x63), + ] { + let schema = Schema::new(url.clone(), tag); + let handles = schema.create(&binding(tag), [71; 32]); + let _ = whole_file(&handles, b"t10.5 failpoint dead content"); + let error = handles.compact(&request(op, Some(point))).unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{tag}: {error:?}" + ); + let status = postgresql_compact_status(&url, &schema.name) + .unwrap() + .unwrap_or_else(|| panic!("{tag}: the op row remains")); + assert_eq!(status.phase, phase, "{tag}: the row records the phase"); + // The released `gc_intent` is gone, so the scratch name carries the + // op identity the projection shows. + assert_eq!(status.op_id, [op; 16], "{tag}: the artifact names the op"); + assert!(!status.alive, "{tag}: the failed owner is not running the op"); + assert!(status.uncertain, "{tag}: a failed run stays uncertain"); + let projected = status.status(); + assert_eq!(projected.name, "store.compact"); + assert_eq!(projected.id, build_op(op)); + assert_eq!(projected.state, "unresolved", "{tag}: the leftover"); + assert_eq!(projected.phase, phase); + assert_eq!(projected.phases, &PHASES[..]); + // The next call settles the leftover and removes the row. + let receipt = handles + .compact(&request(op.wrapping_add(1), None)) + .unwrap(); + assert!( + receipt.generation_after >= receipt.generation_before, + "{tag}: settled" + ); + assert!( + postgresql_compact_status(&url, &schema.name).unwrap().is_none(), + "{tag}: a settled run projects no op" + ); + } +} + +#[test] +fn a_completed_run_leaves_no_projection() { + let Some(url) = support::pg::url() else { + return; + }; + let schema = Schema::new(url.clone(), "done"); + let handles = schema.create(&binding("done"), [71; 32]); + let _ = whole_file(&handles, b"t10.5 completed dead content"); + let before = handles.history.incarnation(); + let receipt = handles.compact(&request(0x71, None)).unwrap(); + assert_eq!(receipt.generation_after, before + 1, "the rebuild installed"); + // `store.status` after a completed run: idle, with the intent released. + assert!(postgresql_compact_status(&url, &schema.name).unwrap().is_none()); + let intents: i64 = schema + .admin() + .query_one( + &format!( + "SELECT count(*) FROM {}.gc_intent WHERE kind='compaction'", + schema.name + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(intents, 0, "the compaction intent was released"); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs index 8445f4f917..c2c4ff7f23 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -438,7 +438,7 @@ fn an_installed_but_unsettled_run_is_judged_at_g_plus_one() { #[test] fn the_report_projects_every_section_11_field() { - let (_temp, handles, path) = create_store(); + let (_temp, handles, _path) = create_store(); let live = filesystem_root(&handles, 1); initialize(&handles, 1, live); let _ = whole_file(&handles, b"dead"); diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs new file mode 100644 index 0000000000..0510d5790e --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs @@ -0,0 +1,902 @@ +//! AC-57 section 13 capacity arm: a real, large SQLite Store compaction measurement. +//! +//! The acceptance matrix's 50 GB arms were extrapolated from the 28.2 ms tiny +//! fixture. This opt-in test builds a Store of the requested size through the +//! public persistence/storage APIs, measures the `none` path at ~9% wholly dead +//! bytes and the `rebuild` path at ~10%, and records wall time, the section 11 +//! receipt values and the T10.3 phase durations. +//! +//! It is ignored by default: `cargo test` never allocates more than a few MiB. +//! Run it with a TMPDIR that has at least 3x the target bytes free: +//! +//! ```text +//! CAPACITY_MEASURE=1 TMPDIR=/var/tmp/g61 UPDATE_CAPACITY=1 \ +//! cargo test -p layerfs-persistence --release \ +//! --test sqlite_compact_capacity -- --ignored --nocapture +//! ``` +//! +//! Knobs: `CAPACITY_GROUPS` (live inode leaves, default 701), +//! `CAPACITY_GROUP` (files per inode leaf, default 99; the page cap is 100 rows including the root directory), +//! `CAPACITY_DEAD_OBJECTS` (initial dead objects, default 6770), +//! `CAPACITY_OBJECT_BYTES` (default 131071, the largest whole-file +//! payload), `CAPACITY_RUNS` (default 3 per arm), `CAPACITY_DURABLE=1` (adds +//! one WAL/FULL rebuild after the WAL/OFF runs), `CAPACITY_KEEP=1` (keeps the +//! temporary Store) and `CAPACITY_OUT` (fixture path override). +//! +//! Fixture shape: every live file is the largest whole-file object (131071 B) +//! reached through one `InodeLeaf` of `CAPACITY_GROUP` file entries and one +//! `FilesystemRoot`, whose `LayerStack` genesis publishes it. The dead payload +//! is whole-file objects no retained root reaches. Every pack holds one object, +//! so `SELECT count(*), sum(length) FROM pack` gives the exact pack byte total +//! and the section 11 `deadBytesBound` can be checked against it. Production +//! saves would add a directory namespace over these inodes; that changes the +//! object count a little, not the bytes compaction copies. +#![cfg(target_os = "linux")] + +mod support; + +use layerfs_content::filesystem::attributes::PortableMetadata; +use layerfs_content::filesystem::directory::{encode_directory_page, DirectoryPage}; +use layerfs_content::filesystem::inode::{encode_inode_page, InodePage}; +use layerfs_content::filesystem::root::profile_id; +use layerfs_content::filesystem::{scope_for_seed, FilesystemRoot}; +use layerfs_content::inode_leaf::{InodeKind, InodeMetadata, InodeValue}; +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId, StackInitialization, +}; +use layerfs_persistence::compact_metrics::{CompactMetrics, CompactMetricsCapture}; +use layerfs_persistence::{ + CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{Storage, StoragePolicy}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-capacity-measure"; +const DEFAULT_GROUPS: u64 = 701; +const DEFAULT_GROUP: u64 = 99; +const DEFAULT_DEAD_OBJECTS: u64 = 6_770; +const DEFAULT_OBJECT_BYTES: usize = 131_071; +const BATCH: usize = 100; + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +fn profile_label(profile: SqlitePersistenceProfile) -> &'static str { + match profile { + SqlitePersistenceProfile::Durable => "SqlitePersistenceProfile::Durable (WAL/FULL)", + SqlitePersistenceProfile::Disposable => "SqlitePersistenceProfile::Disposable (WAL/OFF)", + } +} + +fn create_store(path: &Path, profile: SqlitePersistenceProfile) -> SqliteHandles { + SqliteHandles::create( + PersistenceConfig::sqlite(path.to_path_buf()).with_sqlite_profile(profile), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .expect("Store creation") +} + +fn reopen(path: &Path, profile: SqlitePersistenceProfile) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()).with_sqlite_profile(profile), + BINDING, + [71; 32], + ) + .expect("Store reopen") +} + +/// Deterministic payload; the seed makes every object distinct. The first eight +/// bytes carry the object index so two seeds cannot collide by accident. +fn payload(seed: u64, len: usize) -> Vec { + let mut out = vec![0u8; len]; + out[..8].copy_from_slice(&seed.to_le_bytes()); + let mut state = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + for chunk in out[8..].chunks_exact_mut(8) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + chunk.copy_from_slice(&state.to_le_bytes()); + } + out +} + +fn whole_file(bytes: &[u8]) -> FinalizedObject { + FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole-file payload"), + ) + .expect("finalized object") +} + +/// Saves a batch of already-finalized objects in one Save. +fn save_batch(handles: &SqliteHandles, objects: Vec) -> Vec { + let storage = Storage::new(handles.storage.clone()).expect("storage"); + let save = storage.begin_save().expect("save"); + let mut ids = Vec::with_capacity(objects.len()); + for object in objects { + ids.push(object.id()); + save.accept(object).expect("accept"); + } + save.finish().expect("save finish"); + ids +} + +fn save_whole_files(handles: &SqliteHandles, payloads: &[Vec]) -> Vec { + let objects = payloads.iter().map(|bytes| whole_file(bytes)).collect(); + save_batch(handles, objects) +} + +/// Publishes one `InodeLeaf` of whole-file content roots and its +/// `FilesystemRoot` as the genesis of its own LayerStack. +fn publish_group(handles: &SqliteHandles, index: u64, roots: &[ObjectId]) { + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).expect("directory"), + ) + .expect("directory leaf"); + let directory_id = directory.id(); + let mut entries = Vec::with_capacity(roots.len() + 1); + entries.push(( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::try_from(index).unwrap_or(0), + mtime_nanoseconds: 0, + }), + }, + )); + for (offset, root) in roots.iter().enumerate() { + entries.push(( + u64::try_from(offset + 2).expect("inode serial"), + InodeValue { + kind: InodeKind::RegularFile, + namespace_ref_count: 1, + content_root: *root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o644, + mtime_seconds: i64::try_from(index).unwrap_or(0), + mtime_nanoseconds: 0, + }), + }, + )); + } + let mut references = Vec::with_capacity(roots.len() + 1); + references.push(directory_id); + references.extend_from_slice(roots); + let leaf = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { entries }).expect("inode page"), + ) + .expect("inode leaf") + .with_references(references); + let leaf_id = leaf.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, leaf_id) + .expect("filesystem root"); + let root_object = + FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().expect("root encode")) + .expect("filesystem root object") + .with_references(vec![leaf_id]); + let root_id = root_object.id(); + save_batch(handles, vec![directory, leaf, root_object]); + + let mut stack = [0u8; 16]; + stack[..8].copy_from_slice(&index.to_le_bytes()); + stack[8..].copy_from_slice(b"g61cap00"); + handles + .history + .initialize_layerstack(&StackInitialization { + stack: LayerStackId::from_authority(stack), + name: HistoryName::new(&format!("capacity-{index:07}")).expect("name"), + scope: scope_for_seed([1; 32]).object(), + profile: profile_id(), + genesis_root: root_id, + logical: 0, + }) + .expect("layerstack initialization"); +} + +#[derive(Clone, Copy, Debug, Default)] +struct PackStats { + count: u64, + total: u64, + min: u64, + max: u64, +} + +/// Reads the exact pack population straight from the Store file. The read-only +/// connection is closed before any compaction starts. +fn pack_stats(path: &Path) -> PackStats { + let open = |flags| rusqlite::Connection::open_with_flags(path, flags); + let connection = open(rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .or_else(|_| open(rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE)) + .expect("Store probe"); + let row: (i64, i64, i64, i64) = connection + .query_row( + "SELECT count(*), COALESCE(sum(length),0), COALESCE(min(length),0), \ + COALESCE(max(length),0) FROM pack", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("pack statistics"); + PackStats { + count: u64::try_from(row.0).unwrap_or(0), + total: u64::try_from(row.1).unwrap_or(0), + min: u64::try_from(row.2).unwrap_or(0), + max: u64::try_from(row.3).unwrap_or(0), + } +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +#[derive(Clone, Debug)] +struct Measured { + micros: u64, + path: &'static str, + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + physical_shrink: u64, + peak_temp_bytes: u64, + old_file_bytes: u64, + new_file_bytes: u64, + added_objects: u64, + pack_before: PackStats, + pack_after: PackStats, + phases_nanos: Vec<(&'static str, u64)>, +} + +fn measured(path: &Path, profile: SqlitePersistenceProfile, op: u8, added: u64) -> Measured { + let handles = reopen(path, profile); + let pack_before = pack_stats(path); + let old_file_bytes = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let capture = CompactMetricsCapture::start(); + let started = Instant::now(); + let receipt = handles.compact(&request(op)).expect("compaction"); + let micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + let metrics: CompactMetrics = capture.finish(); + let new_file_bytes = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let pack_after = pack_stats(path); + Measured { + micros, + path: match receipt.path { + CompactPath::None => "none", + CompactPath::Rebuild => "rebuild", + }, + generation_before: receipt.generation_before, + generation_after: receipt.generation_after, + dead_bytes_bound: receipt.dead_bytes_bound, + dead_bytes_refined: receipt.dead_bytes_refined, + physical_shrink: receipt.physical_shrink, + peak_temp_bytes: receipt.peak_temp_bytes, + old_file_bytes, + new_file_bytes, + added_objects: added, + pack_before, + pack_after, + phases_nanos: phase_nanos(&metrics), + } +} + +fn phase_nanos(metrics: &CompactMetrics) -> Vec<(&'static str, u64)> { + let mut phases: Vec<(&'static str, u64)> = metrics + .samples() + .into_iter() + .filter(|sample| sample.name == "compact.phase.duration_ns") + .filter(|sample| sample.label("statistic") == Some("sum")) + .filter_map(|sample| Some((sample.label("phase")?, sample.value))) + .collect(); + phases.sort_by_key(|(phase, _)| *phase); + phases +} + +fn median(values: &[u64]) -> u64 { + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] +} + +fn json_u64_array(values: &[u64]) -> String { + values + .iter() + .map(u64::to_string) + .collect::>() + .join(", ") +} + +fn phase_value(run: &Measured, phase: &str) -> u64 { + run.phases_nanos + .iter() + .find(|(name, _)| *name == phase) + .map(|(_, value)| *value) + .unwrap_or(0) +} + +/// One arm as a JSON object, with the per-run values and the median/range. +fn json_arm(name: &str, runs: &[Measured]) -> String { + let micros: Vec = runs.iter().map(|run| run.micros).collect(); + let old: Vec = runs.iter().map(|run| run.old_file_bytes).collect(); + let new: Vec = runs.iter().map(|run| run.new_file_bytes).collect(); + let dead: Vec = runs.iter().map(|run| run.dead_bytes_bound).collect(); + let refined: Vec = runs.iter().map(|run| run.dead_bytes_refined).collect(); + let shrink: Vec = runs.iter().map(|run| run.physical_shrink).collect(); + let peak: Vec = runs.iter().map(|run| run.peak_temp_bytes).collect(); + let generation_before: Vec = runs.iter().map(|run| run.generation_before).collect(); + let generation_after: Vec = runs.iter().map(|run| run.generation_after).collect(); + let packs_before: Vec = runs.iter().map(|run| run.pack_before.count).collect(); + let packs_after: Vec = runs.iter().map(|run| run.pack_after.count).collect(); + let added: Vec = runs.iter().map(|run| run.added_objects).collect(); + let phases = ["mark", "build", "verify", "install"] + .iter() + .map(|phase| { + let values: Vec = runs.iter().map(|run| phase_value(run, phase)).collect(); + format!("\n \"{phase}\": [{}]", json_u64_array(&values)) + }) + .collect::>() + .join(","); + format!( + "\n \"{name}\": {{\n \"path\": \"{}\",\n \"runs\": {},\n \"micros\": [{}],\n \ + \"medianMicros\": {},\n \"minMicros\": {},\n \"maxMicros\": {},\n \ + \"deadBytesBound\": [{}],\n \"deadBytesRefined\": [{}],\n \ + \"physicalShrink\": [{}],\n \"peakTempBytes\": [{}],\n \ + \"oldFileBytes\": [{}],\n \"newFileBytes\": [{}],\n \ + \"generationBefore\": [{}],\n \"generationAfter\": [{}],\n \ + \"packCountBefore\": [{}],\n \"packCountAfter\": [{}],\n \ + \"addedDeadObjects\": [{}],\n \"phasesNanos\": {{{}\n }}\n }}", + runs[0].path, + runs.len(), + json_u64_array(µs), + median(µs), + micros.iter().min().copied().unwrap_or(0), + micros.iter().max().copied().unwrap_or(0), + json_u64_array(&dead), + json_u64_array(&refined), + json_u64_array(&shrink), + json_u64_array(&peak), + json_u64_array(&old), + json_u64_array(&new), + json_u64_array(&generation_before), + json_u64_array(&generation_after), + json_u64_array(&packs_before), + json_u64_array(&packs_after), + json_u64_array(&added), + phases, + ) +} + +fn rustc_version() -> String { + std::process::Command::new("rustc") + .arg("--version") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|text| text.trim().to_owned()) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +fn cpus() -> u64 { + std::thread::available_parallelism() + .map(|value| u64::try_from(value.get()).unwrap_or(0)) + .unwrap_or(0) +} + +fn mem_bytes() -> u64 { + std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|text| { + text.lines().find_map(|line| { + let rest = line.strip_prefix("MemTotal:")?; + let kib: u64 = rest.trim().trim_end_matches(" kB").trim().parse().ok()?; + Some(kib.saturating_mul(1024)) + }) + }) + .unwrap_or(0) +} + +fn update_fixture(path: &Path, json: &str) { + std::fs::write(path, json.as_bytes()).expect("fixture write"); +} + +/// Saves `count` dead whole-file objects in batches and returns how many it added. +fn add_dead(handles: &SqliteHandles, count: u64, seed_base: u64, object_bytes: usize) { + let mut left = count; + let mut seed = seed_base; + while left > 0 { + let take = left.min(BATCH as u64); + let payloads: Vec> = (0..take) + .map(|offset| payload(seed + offset, object_bytes)) + .collect(); + let _ = save_whole_files(handles, &payloads); + seed += take; + left -= take; + } +} + +/// Adds the smallest number of whole-file dead objects that makes the pack-byte +/// dead share reach 10%. Returns `(dead_bytes, objects_added, next_seed)`. +fn cross_ten( + store: &Path, + profile: SqlitePersistenceProfile, + dead_bytes: u64, + mut seed: u64, + object_bytes: usize, +) -> (u64, u64, u64) { + // One object is one singleton pack: measure its width instead of assuming + // the pack framing. + let before = pack_stats(store); + { + let handles = reopen(store, profile); + add_dead(&handles, 1, seed, object_bytes); + drop(handles); + } + seed += 1; + let after = pack_stats(store); + let unit = after.total.saturating_sub(before.total); + let mut dead = dead_bytes.saturating_add(unit); + let mut added = 1u64; + loop { + let before = pack_stats(store); + if before.total > 0 && dead.saturating_mul(10) >= before.total { + break; + } + let need = before + .total + .saturating_sub(dead.saturating_mul(10)) + .div_ceil(unit.saturating_mul(9)) + .max(1); + let handles = reopen(store, profile); + add_dead(&handles, need, seed, object_bytes); + drop(handles); + seed += need; + added += need; + let after = pack_stats(store); + dead = dead.saturating_add(after.total.saturating_sub(before.total)); + if added > 10_000_000 { + panic!("could not cross the 10% threshold"); + } + } + (dead, added, seed) +} + +/// The opt-in section 13 capacity measurement. See the module docs. +#[test] +#[ignore = "opt-in 50 GB capacity measurement: set CAPACITY_MEASURE=1 and a large TMPDIR"] +fn capacity_measure() { + if std::env::var_os("CAPACITY_MEASURE").is_none() { + eprintln!("capacity_measure: skipped; set CAPACITY_MEASURE=1 to run the section 13 arm"); + return; + } + let groups = env_u64("CAPACITY_GROUPS", DEFAULT_GROUPS); + let group_size = env_u64("CAPACITY_GROUP", DEFAULT_GROUP); + let initial_dead = env_u64("CAPACITY_DEAD_OBJECTS", DEFAULT_DEAD_OBJECTS); + let runs_per_arm = env_u64("CAPACITY_RUNS", 3).max(1); + let object_bytes = + usize::try_from(env_u64("CAPACITY_OBJECT_BYTES", DEFAULT_OBJECT_BYTES as u64)) + .expect("object bytes"); + let live_objects = groups.saturating_mul(group_size); + let durable = std::env::var_os("CAPACITY_DURABLE").is_some(); + let keep = std::env::var_os("CAPACITY_KEEP").is_some(); + + let temp = Temp::new("sqlite-compact-capacity"); + let store = temp.join("store.sqlite"); + eprintln!( + "capacity_measure: {} groups x {} whole-file objects of {} B + {} dead objects under {}", + groups, + group_size, + object_bytes, + initial_dead, + temp.path().display() + ); + + // Build the live payload through the public APIs: whole-file objects reached + // through an inode leaf and a filesystem root, published as a stack genesis. + let handles = create_store(&store, SqlitePersistenceProfile::Disposable); + let build_started = Instant::now(); + let mut seed = 0u64; + for group in 0..groups { + let payloads: Vec> = (0..group_size) + .map(|offset| payload(seed + offset, object_bytes)) + .collect(); + seed += group_size; + let roots = save_whole_files(&handles, &payloads); + publish_group(&handles, group, &roots); + if group % 50 == 0 || group + 1 == groups { + let elapsed = build_started.elapsed().as_secs_f64(); + let written = (group + 1) as f64 * group_size as f64 * object_bytes as f64; + eprintln!( + "capacity_measure: build live groups {}/{} ({:.2} GB, {:.2} GB/s)", + group + 1, + groups, + written / 1e9, + written / 1e9 / elapsed.max(1e-9) + ); + let _ = std::io::stderr().flush(); + } + } + add_dead(&handles, initial_dead, 1_000_000_000, object_bytes); + let fixture_build_seconds = build_started.elapsed().as_secs_f64(); + let built = pack_stats(&store); + eprintln!( + "capacity_measure: fixture built in {:.1} s; {} packs, {:.3} GB, pack length {}..{}", + fixture_build_seconds, + built.count, + built.total as f64 / 1e9, + built.min, + built.max + ); + drop(handles); + + // The `none` path at ~9%: mark only, repeated for the E3 median and range. + let mut none_runs = Vec::new(); + for run in 0..u8::try_from(runs_per_arm).expect("run count") { + let measured = measured(&store, SqlitePersistenceProfile::Disposable, 90 + run, 0); + assert_eq!(measured.path, "none", "~9% dead takes the none path"); + eprintln!( + "capacity_measure: none run {} -> {} us, dead {} B, shrink {} B", + run + 1, + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink + ); + none_runs.push(measured); + } + + // The `rebuild` path at the 10% threshold, then again after re-adding enough + // dead packs for the refined decision on each compacted Store. + let mut rebuild_runs = Vec::new(); + let mut dead_bytes = none_runs[0].dead_bytes_bound; + let mut dead_seed = 2_000_000_000u64; + for run in 0..u8::try_from(runs_per_arm).expect("run count") { + let (_crossed, added, seed) = cross_ten( + &store, + SqlitePersistenceProfile::Disposable, + dead_bytes, + dead_seed, + object_bytes, + ); + dead_seed = seed; + let measured = measured(&store, SqlitePersistenceProfile::Disposable, 100 + run, added); + assert_eq!(measured.path, "rebuild", "10% dead takes the rebuild path"); + eprintln!( + "capacity_measure: rebuild run {} -> {} us, dead {} B, shrink {} B, peak {} B", + run + 1, + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink, + measured.peak_temp_bytes + ); + dead_bytes = 0; + rebuild_runs.push(measured); + } + + // Optional WAL/FULL sensitivity run: production's default profile. + let mut durable_run = None; + if durable { + let (_crossed, added, _seed) = cross_ten( + &store, + SqlitePersistenceProfile::Durable, + dead_bytes, + dead_seed, + object_bytes, + ); + let measured = measured(&store, SqlitePersistenceProfile::Durable, 120, added); + assert_eq!(measured.path, "rebuild"); + eprintln!( + "capacity_measure: durable rebuild -> {} us, dead {} B, shrink {} B, peak {} B", + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink, + measured.peak_temp_bytes + ); + durable_run = Some(measured); + } + + let none_dead = none_runs[0].dead_bytes_bound; + let none_total = none_runs[0].pack_before.total; + let rebuild_dead = rebuild_runs[0].dead_bytes_bound; + let rebuild_total = rebuild_runs[0].pack_before.total; + let mut arms = format!( + "{},{}", + json_arm("none", &none_runs), + json_arm("rebuild", &rebuild_runs) + ); + if let Some(run) = &durable_run { + arms.push_str(&format!( + ",\n \"durableRebuild\": {{\n \"path\": \"rebuild\",\n \"runs\": 1,\n \ + \"micros\": [{}],\n \"deadBytesBound\": [{}],\n \ + \"physicalShrink\": [{}],\n \"peakTempBytes\": [{}],\n \ + \"addedDeadObjects\": [{}],\n \"phasesNanos\": {{\n \"mark\": [{}],\n \ + \"build\": [{}],\n \"verify\": [{}],\n \"install\": [{}]\n }}\n }}", + run.micros, + run.dead_bytes_bound, + run.physical_shrink, + run.peak_temp_bytes, + run.added_objects, + phase_value(run, "mark"), + phase_value(run, "build"), + phase_value(run, "verify"), + phase_value(run, "install"), + )); + } + let json = format!( + "{{\n \"schema\": \"t12.3a-sqlite-compact-capacity/1\",\n \ + \"actualBytes\": {},\n \"actualGib\": \"{:.3}\",\n \ + \"packCount\": {},\n \"packLengthMin\": {},\n \"packLengthMax\": {},\n \ + \"objectBytes\": {},\n \"groups\": {},\n \"groupSize\": {},\n \ + \"liveObjects\": {},\n \"initialDeadObjects\": {},\n \ + \"deadFractionPermille\": {},\n \"fixtureBuildSeconds\": {:.1},\n \ + \"noneDeadBytes\": {},\n \"noneTotalBytes\": {},\n \ + \"rebuildDeadBytes\": {},\n \"rebuildTotalBytes\": {},\n \ + \"profile\": \"{}\",\n \"platform\": \"{}-{}\",\n \ + \"rustc\": \"{}\",\n \"cpus\": {},\n \"memBytes\": {},{}\n}}\n", + built.total, + built.total as f64 / 1e9, + built.count, + built.min, + built.max, + object_bytes, + groups, + group_size, + live_objects, + initial_dead, + none_dead.saturating_mul(1000) / none_total.max(1), + fixture_build_seconds, + none_dead, + none_total, + rebuild_dead, + rebuild_total, + profile_label(SqlitePersistenceProfile::Disposable), + std::env::consts::OS, + std::env::consts::ARCH, + rustc_version(), + cpus(), + mem_bytes(), + arms, + ); + println!("{json}"); + if std::env::var_os("UPDATE_CAPACITY").is_some() { + let out = std::env::var_os("CAPACITY_OUT").map(PathBuf::from).unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../../engine/crates/layerfs-persistence/tests/fixtures/\ + sqlite_compact_capacity_measure.json", + ) + }); + update_fixture(&out, &json); + eprintln!("capacity_measure: wrote {}", out.display()); + } + if keep { + eprintln!("capacity_measure: kept {}", temp.path().display()); + std::mem::forget(temp); + } else { + eprintln!("capacity_measure: removing {}", temp.path().display()); + } +} + + +// ---- the peak-temp receipt covers the whole run's temp artifacts ------------ + +/// The maximum simultaneous extent of the rebuild's temp artifacts one sample +/// saw: the pre-truncate staged copy, its rollback journal, the visited scratch +/// and SQLite's unlinked rewrite temp database. +#[derive(Clone, Copy, Debug, Default)] +struct TempFootprint { + total: u64, + staged: u64, + journal: u64, + visited: u64, + rewrite: u64, +} + +/// Samples the run's temp artifacts from a second thread. The staged file, +/// its `-journal` and the visited scratch are visible in the Store's directory; +/// SQLite writes the rewrite's temp database to the process temp directory and +/// unlinks it, so it is reachable only through `/proc/self/fd` (Linux). +struct FootprintSampler { + stop: Arc, + handle: Option>, + total: Arc, + staged: Arc, + journal: Arc, + visited: Arc, + rewrite: Arc, +} + +impl FootprintSampler { + fn start(store: &Path) -> Self { + let name = store + .file_name() + .map(|file| file.to_string_lossy().into_owned()) + .unwrap_or_default(); + let directory = store + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let stop = Arc::new(AtomicBool::new(false)); + let total = Arc::new(AtomicU64::new(0)); + let staged = Arc::new(AtomicU64::new(0)); + let journal = Arc::new(AtomicU64::new(0)); + let visited = Arc::new(AtomicU64::new(0)); + let rewrite = Arc::new(AtomicU64::new(0)); + let handle = { + let stop = Arc::clone(&stop); + let total = Arc::clone(&total); + let staged_at = Arc::clone(&staged); + let journal_at = Arc::clone(&journal); + let visited_at = Arc::clone(&visited); + let rewrite_at = Arc::clone(&rewrite); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let (mut staged, mut journal, mut visited) = (0u64, 0u64, 0u64); + if let Ok(entries) = std::fs::read_dir(&directory) { + for entry in entries.flatten() { + let file = entry.file_name().to_string_lossy().into_owned(); + let Some(suffix) = file.strip_prefix(&name) else { + continue; + }; + let Some(suffix) = suffix.strip_prefix('.') else { + continue; + }; + let size = entry.metadata().map(|meta| meta.len()).unwrap_or(0); + if suffix == "compact" { + staged = staged.max(size); + } else if suffix.starts_with("compact-journal") + || suffix.starts_with("compact-mj") + { + journal = journal.max(size); + } else if suffix.starts_with("compact-visited") { + visited = visited.max(size); + } + } + } + let mut rewrite = 0u64; + if let Ok(fds) = std::fs::read_dir("/proc/self/fd") { + for fd in fds.flatten() { + let Ok(link) = std::fs::read_link(fd.path()) else { + continue; + }; + if !link.to_string_lossy().contains("etilqs_") { + continue; + } + if let Ok(meta) = std::fs::metadata(fd.path()) { + rewrite = rewrite.max(meta.len()); + } + } + } + staged_at.fetch_max(staged, Ordering::Relaxed); + journal_at.fetch_max(journal, Ordering::Relaxed); + visited_at.fetch_max(visited, Ordering::Relaxed); + rewrite_at.fetch_max(rewrite, Ordering::Relaxed); + total.fetch_max( + staged + journal + visited + rewrite, + Ordering::Relaxed, + ); + std::thread::sleep(Duration::from_micros(200)); + } + }) + }; + Self { + stop, + handle: Some(handle), + total, + staged, + journal, + visited, + rewrite, + } + } + + fn finish(mut self) -> TempFootprint { + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + TempFootprint { + total: self.total.load(Ordering::Relaxed), + staged: self.staged.load(Ordering::Relaxed), + journal: self.journal.load(Ordering::Relaxed), + visited: self.visited.load(Ordering::Relaxed), + rewrite: self.rewrite.load(Ordering::Relaxed), + } + } +} + +/// The §11 peak-temp receipt on a moderate fixture. The run really does create +/// the pre-truncate staged copy, a rollback journal (the prune and the rewrite) +/// and SQLite's unlinked rewrite temp database, and the receipt must bound all +/// of them; the old `new + visited` formula fails the sampled journal. The +/// fixture is ~40 MB, not the 50 GB T-G66 arm: the same code path, sampled. +#[test] +fn the_peak_temp_receipt_covers_the_staged_copy_and_the_rollback_journal() { + const GROUPS: u64 = 16; + const GROUP: u64 = 48; + const OBJECT_BYTES: usize = 40 * 1024; + const DEAD_OBJECTS: u64 = 240; + + let temp = Temp::new("sqlite-compact-peak-temp"); + let store = temp.join("store.sqlite"); + let handles = create_store(&store, SqlitePersistenceProfile::Disposable); + let mut seed = 0u64; + for group in 0..GROUPS { + let payloads: Vec> = (0..GROUP) + .map(|offset| payload(seed + offset, OBJECT_BYTES)) + .collect(); + seed += GROUP; + let roots = save_whole_files(&handles, &payloads); + publish_group(&handles, group, &roots); + } + add_dead(&handles, DEAD_OBJECTS, 1_000_000_000, OBJECT_BYTES); + drop(handles); + + let sampler = FootprintSampler::start(&store); + let handles = reopen(&store, SqlitePersistenceProfile::Disposable); + let receipt = handles.compact(&request(0x7a)).expect("compaction"); + let footprint = sampler.finish(); + + assert_eq!( + receipt.path, + CompactPath::Rebuild, + "the fixture must take the rebuild path" + ); + assert!( + footprint.staged > 0, + "the staged copy must be sampled: {footprint:?}" + ); + assert!( + footprint.journal > 0, + "the rollback journal must be sampled: {footprint:?}" + ); + assert!( + footprint.rewrite > 0, + "the unlinked rewrite temp database must be sampled: {footprint:?}" + ); + assert!( + footprint.visited > 0, + "the visited scratch must be sampled: {footprint:?}" + ); + let new_bytes = std::fs::metadata(&store).expect("installed Store").len(); + eprintln!( + "peak-temp fixture: new file {new_bytes} B, sampled temp footprint {footprint:?}, \ + receipt {} B", + receipt.peak_temp_bytes + ); + assert!( + receipt.peak_temp_bytes >= new_bytes + footprint.journal + footprint.visited, + "peakTempBytes {} must cover the new file {} plus the sampled {}-byte journal \ + and {}-byte visited scratch", + receipt.peak_temp_bytes, + new_bytes, + footprint.journal, + footprint.visited + ); + assert!( + receipt.peak_temp_bytes >= footprint.total, + "peakTempBytes {} must bound the sampled {}-byte temp footprint: {footprint:?}", + receipt.peak_temp_bytes, + footprint.total + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs index 79751dd869..d4fbc3eae3 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs @@ -38,6 +38,10 @@ struct Case { installed: bool, target: bool, visited: bool, + /// A real `SIGKILL` before the seal ends the process with the writer + /// session's WAL on disk (`-wal`, `-shm`); the in-process matrix stops and + /// drops cleanly, so only the real-process `BeforeSeal` arm sets this. + live_sidecars: bool, } fn create_store() -> (Temp, SqliteHandles, PathBuf) { @@ -313,26 +317,12 @@ fn assert_no_sidecars(store: &Path, case: Case) { } } -fn exercise(case: Case) { - let (_temp, handles, path) = create_store(); - let live = filesystem_root(&handles, 1); - let stack = initialize(&handles, 1, live); - let dead = whole_file(&handles, b"a pack no retained root reaches"); - let live_bytes = object_bytes(&handles, live); - let before = generation(&handles); - - let op = [41u8; 16]; - let error = handles - .compact(&request_at(op, case.failpoint)) - .unwrap_err(); - assert!( - matches!(error, layerfs_history::HistoryError::UnknownOutcome), - "{error:?}" - ); - +/// Asserts the operation record and the artifact set the failpoint leaves +/// beside the live Store, before any handle reopens it. +fn assert_crash_state(case: Case, store: &Path, before: u64, op: [u8; 16]) { // `store.status` (T10.1) projects the recorded operation, its phase and // its uncertain outcome; the engine reads them from the op record. - let status = compact_op_status(&path).expect("the op record remains"); + let status = compact_op_status(store).expect("the op record remains"); assert_eq!(status.op_id, op); assert_eq!(status.generation, i64::try_from(before).unwrap()); assert_eq!(status.phase, case.phase, "phase at {case:?}"); @@ -343,20 +333,65 @@ fn exercise(case: Case) { // The exact artifact set the point leaves beside the live Store. assert_eq!( - compact_path(&path).exists(), + compact_path(store).exists(), case.target, "staged Store at {case:?}" ); assert_eq!( - visited_path(&path).exists(), + visited_path(store).exists(), case.visited, "visited scratch at {case:?}" ); - assert!(op_path(&path).exists(), "op record at {case:?}"); - assert_no_sidecars(&path, case); - assert_no_sidecars(&compact_path(&path), case); + assert!(op_path(store).exists(), "op record at {case:?}"); + if case.live_sidecars { + // A real kill before the seal ends the process with the writer + // session's WAL still on disk: the next open replays it, and the crash + // state itself keeps both sidecars, exactly as a killed daemon does. + assert!( + with_suffix(store, "-wal").exists(), + "the killed session's WAL at {case:?}" + ); + assert!( + with_suffix(store, "-shm").exists(), + "the killed session's shared memory at {case:?}" + ); + assert!( + !with_suffix(store, "-journal").exists(), + "no rollback journal at {case:?}" + ); + } else { + assert_no_sidecars(store, case); + } + assert_no_sidecars(&compact_path(store), case); +} - let reopened = reopen(&path); +/// The fixtures one crash case needs to prove its recovery. +struct Recovery<'a> { + case: Case, + store: &'a Path, + before: u64, + live: ObjectId, + live_bytes: &'a [u8], + dead: ObjectId, + stack: LayerStackRecord, + op: [u8; 16], +} + +/// Reopens the crashed Store, asserts the §9 state (wholly old or wholly new, +/// live data readable, exact artifacts) and then proves the next +/// `store.compact` settles or rebuilds and leaves a clean Store. +fn assert_recovery(recovery: Recovery<'_>) { + let Recovery { + case, + store, + before, + live, + live_bytes, + dead, + stack, + op, + } = recovery; + let reopened = reopen(store); // The admission intent is released before the seal (T-G5), so only a // crash before that release still leaves it; a rebuilt Store was copied @@ -380,7 +415,7 @@ fn exercise(case: Case) { ); assert!(located(&reopened, live), "live root at {case:?}"); assert_eq!( - object_bytes(&reopened, live), + object_bytes(&reopened, live).as_slice(), live_bytes, "live bytes readable at {case:?}" ); @@ -397,7 +432,7 @@ fn exercise(case: Case) { // A crash during the copy can tear the staged file. The next run must // remove the recorded set instead of trusting it. if case.failpoint == CompactFailpoint::DuringCopy { - let target = compact_path(&path); + let target = compact_path(store); std::fs::OpenOptions::new() .write(true) .open(&target) @@ -410,11 +445,11 @@ fn exercise(case: Case) { // Put one back to prove the next run removes the whole set. if case.failpoint == CompactFailpoint::DuringSettle { std::fs::write( - with_suffix(&compact_path(&path), "-journal"), + with_suffix(&compact_path(store), "-journal"), b"partial settle", ) .unwrap(); - std::fs::write(visited_path(&path), b"partial settle").unwrap(); + std::fs::write(visited_path(store), b"partial settle").unwrap(); } // The next run judges the leftover and reaches a clean state: a rebuild @@ -435,17 +470,17 @@ fn exercise(case: Case) { receipt.residue ); - assert!(compact_op_status(&path).is_none(), "op settled at {case:?}"); - assert!(!compact_path(&path).exists(), "target removed at {case:?}"); - assert!(!visited_path(&path).exists(), "visited removed at {case:?}"); - assert!(!op_path(&path).exists(), "op removed at {case:?}"); - assert_no_sidecars(&path, case); + assert!(compact_op_status(store).is_none(), "op settled at {case:?}"); + assert!(!compact_path(store).exists(), "target removed at {case:?}"); + assert!(!visited_path(store).exists(), "visited removed at {case:?}"); + assert!(!op_path(store).exists(), "op removed at {case:?}"); + assert_no_sidecars(store, case); - let final_handles = reopen(&path); + let final_handles = reopen(store); assert_eq!(generation(&final_handles), before + 1, "final generation"); assert!(located(&final_handles, live), "final live root"); assert_eq!( - object_bytes(&final_handles, live), + object_bytes(&final_handles, live).as_slice(), live_bytes, "final bytes" ); @@ -456,6 +491,36 @@ fn exercise(case: Case) { clean(&final_handles); } +fn exercise(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + let op = [41u8; 16]; + let error = handles + .compact(&request_at(op, case.failpoint)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + + assert_crash_state(case, &path, before, op); + assert_recovery(Recovery { + case, + store: &path, + before, + live, + live_bytes: &live_bytes, + dead, + stack, + op, + }); +} + macro_rules! crash_at { ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal) => { #[test] @@ -466,6 +531,7 @@ macro_rules! crash_at { installed: $installed, target: $target, visited: $visited, + live_sidecars: false, }); } }; @@ -575,3 +641,317 @@ crash_at!( false, false ); + +/// The same §9 crash points, but the compaction runs in a **separate process** +/// that the parent `SIGKILL`s while it is stopped inside the failpoint. This +/// is the crash arm the in-process matrix above cannot claim: no Rust `Drop`, +/// no connection close and no guard release runs after the kill. +/// +/// The child re-executes this test binary by name and reopens the fixture +/// Store the parent built. The parent learns the child reached the point from +/// a Unix-socket gate (no sleeps), then kills it, then makes the same +/// assertions as `assert_crash_state` / `assert_recovery`. +/// +/// Nine points are covered: the mark phase (`BeforeSeal`, `AfterSeal`); the +/// copy (`BeforeVacuum`, immediately before `VACUUM INTO` because SQLite runs +/// that copy as one uninterruptible call, so a kill cannot be injected inside +/// it); verification (`DuringVerify`); the rename boundary (`DuringRename`); +/// and the earlier build, install and settle points. +/// +/// Linux only: `SIGKILL` delivery and `ExitStatusExt::signal` are asserted +/// through the Unix process API. On other Unix targets the test below prints +/// a skip line instead of claiming coverage. +#[cfg(target_os = "linux")] +mod real_process { + use super::*; + use std::io::Read; + use std::os::unix::process::ExitStatusExt; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + + /// The operation identity the child records and the parent asserts. + const OP: [u8; 16] = [43u8; 16]; + const STORE_ENV: &str = "LAYERFS_COMPACT_CRASH_STORE"; + const FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + const GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; + + /// A short socket path: `Temp` lives under the (ext4) sandbox run + /// directory, whose absolute path exceeds `SUN_LEN`. + fn gate_socket() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + PathBuf::from(format!( + "/tmp/layerfs-compact-gate-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) + } + + fn parse_failpoint(text: &str) -> CompactFailpoint { + match text { + "BeforeSeal" => CompactFailpoint::BeforeSeal, + "AfterSeal" => CompactFailpoint::AfterSeal, + "BeforeVacuum" => CompactFailpoint::BeforeVacuum, + "DuringVerify" => CompactFailpoint::DuringVerify, + "DuringRename" => CompactFailpoint::DuringRename, + "AfterBuild" => CompactFailpoint::AfterBuild, + "BeforeInstall" => CompactFailpoint::BeforeInstall, + "AfterInstall" => CompactFailpoint::AfterInstall, + "DuringSettle" => CompactFailpoint::DuringSettle, + other => panic!("unknown failpoint {other}"), + } + } + + /// The child body. A normal test run returns immediately; the parent + /// re-executes this test by name with the fixture variables set. + #[test] + fn sigkill_at_failpoint() { + let Some(store) = std::env::var_os(STORE_ENV) else { + return; + }; + let point = parse_failpoint( + &std::env::var(FAILPOINT_ENV).expect("a failpoint for the crash child"), + ); + let handles = reopen(Path::new(&store)); + let outcome = handles.compact(&request_at(OP, point)); + panic!("the parent did not SIGKILL the run: {outcome:?}"); + } + + /// Kills the child on every path out of the test, including a panic. + struct KillOnDrop(Option); + + impl KillOnDrop { + fn new(child: Child) -> Self { + Self(Some(child)) + } + + fn child_mut(&mut self) -> &mut Child { + self.0.as_mut().expect("a running child") + } + + fn kill(&mut self) -> std::process::ExitStatus { + let mut child = self.0.take().expect("a running child"); + child.kill().expect("SIGKILL the crash child"); + child.wait().expect("the crash child exits") + } + } + + impl Drop for KillOnDrop { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + + /// Waits for the child's gate connection. `yield_now` keeps the bounded + /// wait from sleeping and still notices a child that ended early. + fn wait_for_gate( + listener: &std::os::unix::net::UnixListener, + child: &mut Child, + ) -> std::os::unix::net::UnixStream { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + match listener.accept() { + Ok((stream, _)) => return stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the crash child's status") { + panic!("the crash child ended before the gate: {status}"); + } + assert!( + Instant::now() < deadline, + "the crash child never reached the gate" + ); + std::thread::yield_now(); + } + Err(error) => panic!("crash gate accept: {error}"), + } + } + } + + fn exercise_sigkill(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + // The child owns the writable Store alone; the parent keeps only the + // fixture paths and the expected bytes. + drop(handles); + + let gate = gate_socket(); + let listener = + std::os::unix::net::UnixListener::bind(&gate).expect("the crash gate socket"); + listener + .set_nonblocking(true) + .expect("a nonblocking crash gate"); + let child = Command::new(std::env::current_exe().expect("the test binary")) + .args([ + "--exact", + "real_process::sigkill_at_failpoint", + "--nocapture", + ]) + .env(STORE_ENV, &path) + .env(FAILPOINT_ENV, format!("{:?}", case.failpoint)) + .env(GATE_ENV, &gate) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("a separate crash process"); + let mut child = KillOnDrop::new(child); + + let mut gate_stream = wait_for_gate(&listener, child.child_mut()); + gate_stream + .set_read_timeout(Some(Duration::from_secs(60))) + .expect("a bounded gate read"); + let mut ready = [0u8; 1]; + gate_stream + .read_exact(&mut ready) + .expect("the child reached the crash gate"); + assert_eq!(ready[0], b'R', "the child stopped at the crash gate"); + + // The real crash: SIGKILL while the child is still inside the run. + let status = child.kill(); + assert_eq!( + status.signal(), + Some(9), + "the crash child was SIGKILLed: {status:?}" + ); + drop(gate_stream); + let _ = std::fs::remove_file(&gate); + + assert_crash_state(case, &path, before, OP); + assert_recovery(Recovery { + case, + store: &path, + before, + live, + live_bytes: &live_bytes, + dead, + stack, + op: OP, + }); + } + + macro_rules! sigkill_at { + ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal, $live_sidecars:literal) => { + #[test] + fn $name() { + exercise_sigkill(Case { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + live_sidecars: $live_sidecars, + }); + } + }; + } + + // The mark phase: the open session and admission intent at `BeforeSeal` + // (the real kill leaves the writer's WAL sidecars), and the sealed, + // intent-released Store at `AfterSeal`. + sigkill_at!( + sigkill_before_seal, + CompactFailpoint::BeforeSeal, + "mark", + false, + false, + false, + true + ); + sigkill_at!( + sigkill_after_seal, + CompactFailpoint::AfterSeal, + "mark", + false, + false, + false, + false + ); + // The copy: SQLite runs `VACUUM INTO` as one call that cannot be + // interrupted deterministically, so this arm stops immediately before it. + // The in-process `DuringCopy` arm keeps the torn-target state; this real + // kill proves the crash after the visited scratch exists and before the + // staged file does. + sigkill_at!( + sigkill_during_copy, + CompactFailpoint::BeforeVacuum, + "build", + false, + false, + true, + false + ); + // The verify pass and the rename boundary, at the same failpoints the + // in-process matrix uses. + sigkill_at!( + sigkill_during_verify, + CompactFailpoint::DuringVerify, + "verify", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_during_rename, + CompactFailpoint::DuringRename, + "install", + true, + false, + true, + false + ); + + sigkill_at!( + sigkill_after_build, + CompactFailpoint::AfterBuild, + "build", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_before_install, + CompactFailpoint::BeforeInstall, + "verify", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_after_install, + CompactFailpoint::AfterInstall, + "install", + true, + false, + true, + false + ); + sigkill_at!( + sigkill_during_settle, + CompactFailpoint::DuringSettle, + "install", + true, + false, + false, + false + ); +} + +#[cfg(not(target_os = "linux"))] +#[test] +fn real_process_sigkill_skipped_outside_linux() { + eprintln!( + "skipped: the real-process SIGKILL compaction test requires Linux, not {}", + std::env::consts::OS + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs index 3107ef5ccc..8ffc9736f3 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs @@ -1182,7 +1182,6 @@ fn rebuild_fixture() -> (Temp, SqliteHandles, FixtureIds) { .collect(); let mut delta_bases_sorted = vec![base1, base2]; delta_bases_sorted.sort(); - let mut objects_dead = objects_dead; objects_dead.push(dead_group.id()); objects_dead.extend([ root_a4, diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs new file mode 100644 index 0000000000..1b7a4b344e --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs @@ -0,0 +1,1952 @@ +//! #41 T12.3a: the SQLite half of the section 12 acceptance matrix, executed +//! and reported. +//! +//! One runner covers AC-01..AC-60: every row is classified as executed here, +//! covered by a named existing test or fixture (asserted to exist), partial +//! (some SQLite arms evidenced, the rest named), deferred (the row is +//! PostgreSQL-only and belongs to T12.3b) or an explicit SQLite gap. The +//! previously uncovered priority rows with SQLite-only contract arms are +//! executed in this file: AC-33 (checkpoint busy and sidecar-at-install), +//! AC-46 (install fence), AC-51 (the recorded op phase), AC-55 (exact SQLite +//! refusal texts) and AC-59 (seal). AC-57 gets a three-run measurement arm; +//! AC-04 gets a seeded reachability property arm (the full random +//! fork/commit/merge/publish history generator is deferred). +//! +//! T12.3c closes the remaining SQLite gaps this runner can execute: AC-06 +//! (a `file_digest` row whose root the rebuild collected), and AC-07/AC-49 +//! (a mount and a compaction exclude each other; a Commit racing the rebuild +//! is fenced or lands). The derived-tree rows (AC-11..AC-14, AC-22), the bind +//! failure (AC-34) and the worker lost reply (AC-36) are named Store +//! service/control tests; their rows point at those files. +//! +//! The table is checked, not decorative: a `covered`/`partial` row names a +//! file and test function that must exist, and an `executed` row names an arm +//! this binary really runs. The report is written to +//! `core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md`; run with +//! `UPDATE_REPORT=1` to regenerate it, and `UPDATE_MEASURE=1` to refresh the +//! three-run measurement fixture. A normal run asserts both files are current +//! and shape-valid. +//! +//! This page adds no contract: the frozen spec section 12 and the acceptance +//! matrix win. Where a row is cross-backend, its PostgreSQL half is deferred +//! by name. +#![cfg(any(target_os = "linux", target_os = "macos"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, CommitVia, ForkRequest, + ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryError, LayerStackId, + LayerStackRecord, StackInitialization, +}; +use layerfs_history::records::{FileDigest, GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_persistence::{ + compact_op_status, CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, + SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-matrix-binding"; +const REPORT_RELATIVE: &str = "core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md"; +const MEASURE_RELATIVE: &str = + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json"; + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn report_path() -> PathBuf { + repo_root().join(REPORT_RELATIVE) +} + +fn measure_path() -> PathBuf { + repo_root().join(MEASURE_RELATIVE) +} + +// ---- the Store helpers (the T12.1/T12.2 fixture pattern) ------------------- + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-matrix"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: [u8; 16], failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: op, + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact") +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-op") +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-visited") +} + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &SqliteHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn object_bytes(handles: &SqliteHandles, id: ObjectId) -> Vec { + let storage = Storage::new(handles.storage.clone()).unwrap(); + storage + .reader() + .unwrap() + .read_objects(&[id]) + .unwrap() + .into_iter() + .next() + .expect("one object") +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +/// One live root, one Stack and a wholly dead pack: the minimal `rebuild` arm. +fn rebuild_store() -> (Temp, SqliteHandles, PathBuf, ObjectId) { + let (temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack no retained root reaches"); + (temp, handles, path, live) +} + +// ---- arms executed by the runner ------------------------------------------- + +fn arm_checkpoint_busy() { + let (_temp, handles, path, live) = rebuild_store(); + // A live reader's snapshot in the WAL makes wal_checkpoint(TRUNCATE) + // report busy > 0; step 1 refuses Busy before any artifact or op. + let reader = rusqlite::Connection::open(&path).expect("a second connection"); + let mode: String = reader + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .expect("journal mode"); + assert_eq!( + mode.to_ascii_lowercase(), + "wal", + "the disposable profile is WAL" + ); + reader.execute_batch("BEGIN").unwrap(); + let rows: i64 = reader + .query_row("SELECT count(*) FROM pack", [], |row| row.get(0)) + .unwrap(); + assert!(rows >= 1, "the reader holds a nonempty snapshot"); + + let error = handles.compact(&request(41)).unwrap_err(); + assert!(matches!(error, HistoryError::Busy), "{error:?}"); + assert!(!compact_path(&path).exists(), "no staged Store on Busy"); + assert!(!visited_path(&path).exists(), "no visited scratch on Busy"); + assert!(!op_path(&path).exists(), "no op record on Busy"); + let wal = with_suffix(&path, "-wal"); + assert!( + std::fs::metadata(&wal).map(|meta| meta.len()).unwrap_or(0) > 0, + "the refused seal never unlinks the WAL" + ); + + reader.execute_batch("ROLLBACK").unwrap(); + drop(reader); + // The Store is left wholly old at G and the next run rebuilds it. + let reopened = reopen(&path); + let receipt = reopened.compact(&request(42)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + assert!(located(&final_handles, live), "the live root survives"); + clean(&final_handles); +} + +/// The install hook writes a -mj* super-journal beside the sealed Store: +/// step 7's sidecar fence must refuse Conflict and write nothing. +fn place_super_journal(store: &Path) { + let mut name = store.file_name().expect("a file name").to_os_string(); + name.push("-mj-t123a"); + std::fs::write(store.with_file_name(name), b"t123a super-journal").unwrap(); +} + +fn arm_sidecar_at_install() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let hook_request = CompactRequest { + op_id: [43; 16], + before_install_hook: Some(place_super_journal), + ..CompactRequest::default() + }; + let error = handles.compact(&hook_request).unwrap_err(); + match error { + HistoryError::Conflict(text) => assert_eq!(text, "Store sidecar at install"), + other => panic!("expected the sidecar Conflict, got {other:?}"), + } + assert!(!compact_path(&path).exists(), "Conflict leaves no staged Store"); + assert!( + !visited_path(&path).exists(), + "Conflict leaves no visited scratch" + ); + assert!(!op_path(&path).exists(), "Conflict leaves no op record"); + let mut name = path.file_name().unwrap().to_os_string(); + name.push("-mj-t123a"); + let _ = std::fs::remove_file(path.with_file_name(name)); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before, "generation is unchanged"); + assert!(located(&reopened, live), "the live root survives"); +} + +fn arm_status_phases() { + // One stop point per SQLite phase, including an installed one (install). + let cases: [(CompactFailpoint, &str, bool); 4] = [ + (CompactFailpoint::BeforeSeal, "mark", false), + (CompactFailpoint::BeforeCopy, "build", false), + (CompactFailpoint::DuringVerify, "verify", false), + (CompactFailpoint::DuringRename, "install", true), + ]; + let mut seen = BTreeSet::new(); + for (index, (failpoint, phase, installed)) in cases.iter().enumerate() { + let op = [u8::try_from(50 + index).unwrap(); 16]; + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let error = handles.compact(&request_at(op, *failpoint)).unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome), "{error:?}"); + let status = compact_op_status(&path).expect("the op record remains"); + assert_eq!(status.op_id, op); + assert_eq!(status.generation, i64::try_from(before).unwrap()); + assert_eq!(status.phase, *phase); + assert!(status.uncertain, "an unfinished run is uncertain"); + seen.insert(status.phase.clone()); + + // A pre-install stop rebuilds at G + 1; an installed one settles. + let reopened = reopen(&path); + let receipt = reopened + .compact(&request(op[0].wrapping_add(20))) + .unwrap(); + if *installed { + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before + 1); + assert_eq!(receipt.generation_after, before + 1); + } else { + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + } + assert!(compact_op_status(&path).is_none(), "the op settled"); + } + assert_eq!( + seen.iter().map(String::as_str).collect::>(), + vec!["build", "install", "mark", "verify"], + "every SQLite phase was observed" + ); +} + +fn arm_error_codes() { + // A stray staged name no record names is Exists, and is not deleted. + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let stray = compact_path(&path); + std::fs::write(&stray, b"not a Store").unwrap(); + let error = handles.compact(&request(44)).unwrap_err(); + match error { + HistoryError::Exists(text) => assert_eq!(text, "unrecorded compaction artifact"), + other => panic!("expected Exists, got {other:?}"), + } + assert!(stray.exists(), "a name no record names is not deleted"); + + // The bounded-filesystem failpoint is the exact Capacity("disk space"). + let (_temp2, handles2, path2) = create_store(); + let live2 = filesystem_root(&handles2, 1); + initialize(&handles2, 1, live2); + let _dead2 = whole_file(&handles2, b"a pack the rebuild would collect"); + let before = generation(&handles2); + let error = handles2 + .compact(&request_at([45; 16], CompactFailpoint::CapacityAtBuild)) + .unwrap_err(); + match error { + HistoryError::Capacity(text) => assert_eq!(text, "disk space"), + other => panic!("expected Capacity, got {other:?}"), + } + assert!(!compact_path(&path2).exists()); + assert!(!visited_path(&path2).exists()); + assert!(!op_path(&path2).exists()); + let reopened = reopen(&path2); + assert_eq!(generation(&reopened), before, "the old Store is unchanged"); + assert!(located(&reopened, live2)); +} + +fn arm_seal() { + let (_temp, handles, path, live) = rebuild_store(); + let sealed = handles.seal().expect("the seal"); + let bytes = std::fs::metadata(&path).unwrap().len(); + assert_eq!(sealed.path, path); + assert_eq!(sealed.bytes, bytes); + assert!(sealed.bytes > 0); + assert!( + sealed.sqlite_version.starts_with("3."), + "{}", + sealed.sqlite_version + ); + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(&path, suffix).exists(), + "the seal leaves no {suffix} sidecar" + ); + } + let reopened = reopen(&path); + assert!(located(&reopened, live)); + clean(&reopened); +} + +/// §12 AC-06: a `file_digest` row named a root the rebuild collects; the row +/// goes with its root and a later Save of the same content records the newer +/// root without a stale row blocking it. +fn arm_digest_row_after_collection() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead_root = filesystem_root(&handles, 9); + let digest = [0xA6; 32]; + let length = 16 * 1024 * 1024u64; + handles + .history + .record_file_digests(&[FileDigest { + root: *dead_root.as_bytes(), + length, + digest, + }]) + .unwrap(); + assert_eq!( + handles.history.file_roots(&[(digest, length)]).unwrap(), + vec![Some(*dead_root.as_bytes())] + ); + + let receipt = handles.compact(&request(80)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let reopened = reopen(&path); + assert!(!located(&reopened, dead_root), "the named root was collected"); + assert_eq!( + reopened.history.file_roots(&[(digest, length)]).unwrap(), + vec![None], + "the row went with its root" + ); + + // A Save of bytes with that content records the new root; no stale row + // remains to block it. + let new_root = filesystem_root(&reopened, 2); + initialize(&reopened, 2, new_root); + reopened + .history + .record_file_digests(&[FileDigest { + root: *new_root.as_bytes(), + length, + digest, + }]) + .unwrap(); + assert_eq!( + reopened.history.file_roots(&[(digest, length)]).unwrap(), + vec![Some(*new_root.as_bytes())] + ); +} + +/// One mount intent naming its Workspace, Sandbox and machine. +fn mount_intent( + stack: LayerStackId, + branch: BranchId, + op: u8, + workspace: [u8; 32], + sandbox: &str, + machine: &str, +) -> GcIntent { + GcIntent::shared( + [op; 16], + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice().to_vec(), + ) + .in_project(stack.as_slice()) + .named(Some("project-1"), Some("work")) + .placed(Some(workspace), Some(sandbox), Some(machine)) + .owned(Some("runtime-t123c"), Some("boot-t123c")) +} + +fn hex16(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +/// A Stack, a Branch and one wholly dead pack: the fixture the admission arms +/// need. +fn mounted_store() -> (Temp, SqliteHandles, PathBuf, LayerStackId, BranchId) { + let (temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let stack = support::stack(1); + let branch = support::branch_id(1); + let head = handles.history.layer_stack(stack).unwrap().unwrap().head_layer; + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name("work"), + source: ForkSource::Layer(head), + }) + .unwrap(); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + (temp, handles, path, stack, branch) +} + +/// §12 AC-07 / AC-49: a mount excludes compaction and a live compaction +/// excludes a mount and a second compaction. +fn arm_mount_and_compaction_exclude_each_other() { + let (_temp, handles, path, stack, branch) = mounted_store(); + let mount = mount_intent(stack, branch, 81, [0x81; 32], "sandbox-t123c", "machine-t123c"); + handles.history.gc_register(&mount).unwrap(); + + let error = handles.compact(&request(82)).unwrap_err(); + match error { + HistoryError::MountsActive(state) => { + assert_eq!(state.holders.len(), 1, "{state:?}"); + let holder = &state.holders[0]; + assert_eq!(holder.kind, GcIntentKind::Mount); + assert_eq!(holder.workspace.as_deref(), Some(hex16(&[0x81; 32]).as_str())); + assert_eq!(holder.sandbox.as_deref(), Some("sandbox-t123c")); + assert_eq!(holder.machine.as_deref(), Some("machine-t123c")); + assert_eq!(holder.op, hex16(&[81; 16])); + } + other => panic!("expected MountsActive, got {other:?}"), + } + assert!(!compact_path(&path).exists(), "a refused run stages no Store"); + assert!(!visited_path(&path).exists(), "and no visited scratch"); + assert!(!op_path(&path).exists(), "and no op record"); + let reopened = reopen(&path); + reopened.history.gc_release(&mount.op_id).unwrap(); + + // A live compaction intent refuses a mount and a second compaction. + let catalog = reopened.history.catalog_id(); + let compaction = GcIntent::operation( + [83; 16], + GcIntentKind::Compaction, + GcScopeKind::Store, + catalog.as_slice().to_vec(), + ) + .owned(Some("runtime-t123c"), Some("boot-t123c")) + .with_phase(Some("build")); + reopened.history.gc_register(&compaction).unwrap(); + let mount_again = mount_intent(stack, branch, 84, [0x84; 32], "s", "m"); + match reopened.history.gc_register(&mount_again).unwrap_err() { + HistoryError::BusyOp(state) => { + assert_eq!(state.kind, GcIntentKind::Compaction); + assert_eq!(state.entity, "store"); + assert_eq!(state.phase.as_deref(), Some("build")); + assert_eq!(state.op, hex16(&[83; 16])); + assert!(!state.uncertain); + } + other => panic!("expected BusyOp, got {other:?}"), + } + let second = reopen(&path).compact(&request(85)).unwrap_err(); + assert!(matches!(second, HistoryError::BusyOp(_)), "{second:?}"); + assert!(!op_path(&path).exists(), "a refused second run leaves no op"); + reopened.history.gc_release(&compaction.op_id).unwrap(); + drop(reopened); + + // The Store is still wholly old and the next run rebuilds it. + let receipt = reopen(&path).compact(&request(86)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + let mount = mount_intent(stack, branch, 87, [0x87; 32], "s", "m"); + final_handles.history.gc_register(&mount).unwrap(); + final_handles.history.gc_release(&mount.op_id).unwrap(); + clean(&final_handles); +} + +/// §12 AC-49: a mount started while a compaction is live is `Busy` with the +/// compaction's operation; the next run adopts the dead owner's leftover. +fn arm_mount_during_compaction_is_busy() { + let (_temp, handles, path, stack, branch) = mounted_store(); + let error = handles + .compact(&request_at([88; 16], CompactFailpoint::BeforeSeal)) + .unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome), "{error:?}"); + + // The stopped run left its live admission intent: a mount meets the + // compaction's operation, not a clean Store. + let reopened = reopen(&path); + let mount = mount_intent(stack, branch, 89, [0x89; 32], "s", "m"); + match reopened.history.gc_register(&mount).unwrap_err() { + HistoryError::BusyOp(state) => { + assert_eq!(state.kind, GcIntentKind::Compaction); + assert_eq!(state.op, hex16(&[88; 16])); + assert_eq!(state.phase.as_deref(), Some("mark")); + } + other => panic!("expected BusyOp, got {other:?}"), + } + + // The next run judges the leftover and rebuilds; the mount then proceeds. + drop(reopened); + let receipt = reopen(&path).compact(&request(90)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + final_handles.history.gc_register(&mount).unwrap(); + final_handles.history.gc_release(&mount.op_id).unwrap(); + clean(&final_handles); +} + +/// The Commit thread the install hook starts: `(Commit, root)` identities. +type RebuildCommit = Result<(Vec, [u8; 32]), String>; + +static REBUILD_COMMIT: std::sync::Mutex>> = + std::sync::Mutex::new(None); + +/// Commits one fresh filesystem root on `work`, opening the Store afresh in +/// this thread, so the rebuilt Store keeps a real tree. +fn commit_once(path: &Path) -> Result<(Vec, [u8; 32]), String> { + let handles = reopen(path); + let branch = support::branch_id(1); + let snapshot = handles + .history + .branch_snapshot(branch) + .map_err(|error| format!("{error:?}"))? + .ok_or("the test Branch is absent")?; + let candidate = filesystem_root(&handles, 2); + let outcome = handles.history.commit_branch(&CommitRequest { + branch, + expected_head: snapshot.branch.head_commit, + expected_base: snapshot.branch.base_layer, + expected_root: snapshot.effective_root, + candidate_root: candidate, + profile: snapshot.profile, + scope: snapshot.scope, + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }); + match outcome { + Ok(CommitOutcome::Committed { record, .. }) => { + Ok((record.id.as_slice().to_vec(), *candidate.as_bytes())) + } + Ok(other) => Err(format!("{other:?}")), + Err(error) => Err(format!("{error:?}")), + } +} + +/// The §10 install hook: start a Commit while the rebuild holds the lane and +/// before the rename, so its outcome is the fence or a landed Commit. +fn commit_during_rebuild(path: &Path) { + let path = path.to_path_buf(); + let handle = std::thread::spawn(move || commit_once(&path)); + *REBUILD_COMMIT.lock().unwrap() = Some(handle); + std::thread::sleep(std::time::Duration::from_millis(150)); +} + +/// §12 AC-49: a Commit started during the rebuild is never lost silently: the +/// install fence refuses its stale handle and the retry lands on `G + 1`. +fn arm_commit_during_rebuild() { + let (_temp, handles, path, _stack, branch) = mounted_store(); + let hook_request = CompactRequest { + op_id: [91; 16], + before_install_hook: Some(commit_during_rebuild), + ..CompactRequest::default() + }; + let receipt = handles.compact(&hook_request).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let committed = REBUILD_COMMIT + .lock() + .unwrap() + .take() + .expect("the install hook spawned the Commit") + .join() + .unwrap(); + let (id, root) = match committed { + // The thread may have started after the rename, when its handles are + // already the rebuilt Store's: the Commit landed on `G + 1`. + Ok(committed) => committed, + Err(text) => { + assert!( + text.contains("StaleStore"), + "the racing Commit was refused for another reason: {text}" + ); + commit_once(&path).expect("the Commit retried on G + 1") + } + }; + let reopened = reopen(&path); + let id = CommitId::from_slice(&id).unwrap(); + let record = reopened + .history + .commit(id) + .unwrap() + .expect("the Commit survives the rebuild"); + assert_eq!(record.root.as_bytes(), &root); + assert_eq!( + reopened.history.branch(branch).unwrap().unwrap().head_commit, + Some(id) + ); + clean(&reopened); +} + +fn arm_random_store_shapes() { + // A seeded property arm over reachability shapes: every live object stays + // readable and every wholly dead object is collected, for the retained + // closure to be unchanged. + let mut state = 0x9E37_79B9_7F4A_7C15u64; + for case in 0..6u8 { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let live_count = 12 + usize::try_from(state % 9).unwrap(); + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let dead_count = live_count; + let (_temp, handles, path) = create_store(); + let mut live = Vec::new(); + for index in 0..live_count { + let seed = case * 32 + u8::try_from(index).unwrap() + 1; + let root = filesystem_root(&handles, seed); + let tag = u8::try_from(index + 1).unwrap(); + initialize(&handles, tag, root); + live.push((root, object_bytes(&handles, root))); + } + let mut dead = Vec::new(); + for index in 0..dead_count { + dead.push(whole_file( + &handles, + format!("dead-{case}-{index:03}").as_bytes(), + )); + } + let before = handles.history.mark().unwrap().live_heads_digest(); + let before_generation = generation(&handles); + let receipt = handles + .compact(&request(60 + case)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild, "case {case}"); + assert_eq!(receipt.generation_before, before_generation); + assert_eq!(receipt.generation_after, before_generation + 1); + let reopened = reopen(&path); + for (id, bytes) in &live { + assert!(located(&reopened, *id), "live {id} at case {case}"); + assert_eq!(&object_bytes(&reopened, *id), bytes, "bytes at case {case}"); + } + for id in &dead { + assert!(!located(&reopened, *id), "dead {id} at case {case}"); + } + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + before, + "the retained closure digest at case {case}" + ); + clean(&reopened); + } +} + +/// Three runs of the same rebuild: the E3 shape (median and range), on a small +/// fixture. The 50 GB and PostgreSQL speed/space arms stay deferred (section 13). +fn arm_measurement() { + let mut runs = Vec::new(); + for run in 0..3u8 { + let (_temp, handles, path) = create_store(); + let mut live = Vec::new(); + for index in 0..20 { + let seed = run * 32 + u8::try_from(index).unwrap() + 1; + let root = filesystem_root(&handles, seed); + initialize(&handles, u8::try_from(index + 1).unwrap(), root); + live.push(root); + } + for index in 0..20 { + let _ = whole_file( + &handles, + format!("measure-dead-{run}-{index:03}").as_bytes(), + ); + } + let before = generation(&handles); + let started = Instant::now(); + let receipt = handles + .compact(&request(70 + run)) + .unwrap(); + let micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + assert!(receipt.dead_bytes_bound > 0); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + let reopened = reopen(&path); + for id in &live { + assert!(located(&reopened, *id)); + } + clean(&reopened); + runs.push(( + micros, + receipt.dead_bytes_bound, + receipt.physical_shrink, + receipt.peak_temp_bytes, + )); + } + let mut times: Vec = runs.iter().map(|run| run.0).collect(); + times.sort_unstable(); + let median = times[1]; + assert!((times[0]..=times[2]).contains(&median)); + let (dead, shrink, peak) = (runs[0].1, runs[0].2, runs[0].3); + for run in &runs[1..] { + assert_eq!(run.1, dead, "dead bytes are deterministic"); + assert_eq!(run.2, shrink, "physical shrink is deterministic"); + assert_eq!(run.3, peak, "peak temp is deterministic"); + } + let json = format!( + "{{\n \"schema\": \"t12.3a-sqlite-compact-measure/1\",\n \ + \"runs\": {},\n \"micros\": [{}, {}, {}],\n \"medianMicros\": {},\n \ + \"minMicros\": {},\n \"maxMicros\": {},\n \"deadBytesBound\": {},\n \ + \"physicalShrink\": {},\n \"peakTempBytes\": {},\n \ + \"liveRoots\": 20,\n \"deadPacks\": 20,\n \ + \"profile\": \"SqlitePersistenceProfile::Disposable (WAL/OFF)\",\n \ + \"platform\": \"{}-{}\",\n \"rustc\": \"{}\"\n}}\n", + runs.len(), + runs[0].0, + runs[1].0, + runs[2].0, + median, + times[0], + times[2], + dead, + shrink, + peak, + std::env::consts::OS, + std::env::consts::ARCH, + rustc_version(), + ); + let path = measure_path(); + if std::env::var_os("UPDATE_MEASURE").is_some() { + std::fs::write(&path, json.as_bytes()).unwrap(); + } else { + let committed = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; run UPDATE_MEASURE=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix", + path.display() + ) + }); + // Shape-valid on any machine: three runs, median inside its own range. + assert!(committed.contains("\"runs\": 3"), "{committed}"); + assert!(committed.contains("\"medianMicros\""), "{committed}"); + assert!(committed.contains("\"minMicros\""), "{committed}"); + assert!(committed.contains("\"maxMicros\""), "{committed}"); + } +} + +fn rustc_version() -> String { + std::process::Command::new("rustc") + .arg("--version") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|text| text.trim().to_owned()) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +const ARMS: [(&str, fn()); 11] = [ + ("arm_checkpoint_busy", arm_checkpoint_busy), + ("arm_sidecar_at_install", arm_sidecar_at_install), + ("arm_status_phases", arm_status_phases), + ("arm_error_codes", arm_error_codes), + ("arm_seal", arm_seal), + ("arm_digest_row_after_collection", arm_digest_row_after_collection), + ("arm_mount_and_compaction_exclude_each_other", arm_mount_and_compaction_exclude_each_other), + ("arm_mount_during_compaction_is_busy", arm_mount_during_compaction_is_busy), + ("arm_commit_during_rebuild", arm_commit_during_rebuild), + ("arm_random_store_shapes", arm_random_store_shapes), + ("arm_measurement", arm_measurement), +]; + +// ---- the acceptance matrix ------------------------------------------------ + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Backend { + Both, + Sqlite, + Pg, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Status { + /// An arm in this binary runs the SQLite half. + Executed, + /// A named existing test or fixture runs it. + Covered, + /// Some SQLite arms are named; the rest is named as missing. + Partial, + /// PostgreSQL-only (or the PostgreSQL half): T12.3b. + Deferred, + /// SQLite-applicable with no named evidence yet. + Gap, +} + +struct Row { + id: &'static str, + backend: Backend, + tier: &'static str, + status: Status, + /// `repo/path.rs#test_fn`, `repo/path.json`, or `runner#arm_name`. + evidence: &'static [&'static str], + note: &'static str, +} + +const MATRIX: [Row; 60] = [ + Row { id: "AC-01", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_verify", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle"], + note: "The SQLite section 9 crash matrix through named failpoints; a real kill -9 of a separate daemon process is T12.2's in-process stop points, not run here." }, + Row { id: "AC-02", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "The SQLite arm runs through the suite; the converted 1.0 Store arm and the PostgreSQL backend matrix are deferred to T12.3b." }, + Row { id: "AC-03", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_wal_switch", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_install"], + note: "SQLite mark/build/verify/install and WAL-switch stop points; lost COMMIT and power-loss arms belong to the PostgreSQL/fsync halves." }, + Row { id: "AC-04", backend: Backend::Both, tier: "E3", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "runner#arm_random_store_shapes"], + note: "Deterministic closure fixture plus a seeded six-history reachability property arm; the full random fork/commit/merge/publish/removal generator is not built here." }, + Row { id: "AC-05", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "runner#arm_random_store_shapes"], + note: "Full doctor (including the diagnostic scan) on the compacted Store, also in the property arm." }, + Row { id: "AC-06", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_digest_row_after_collection"], + note: "A Save's `file_digest` row named a root the rebuild collected; the row goes with its root and recording the same content again succeeds. The PostgreSQL half is T12.3b." }, + Row { id: "AC-07", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_mount_and_compaction_exclude_each_other", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction"], + note: "SQLite half executed: bypass gives `StaleStore`, a mount blocks compaction (`MountsActive`, naming the holder) and a live compaction refuses a mount and a second compaction (`BusyOp`). The open one-shot SDK transaction arm is PostgreSQL, T12.3b." }, + Row { id: "AC-08", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner"], + note: "One winner; the loser sees MountsActive, and a removed Branch is Missing." }, + Row { id: "AC-09", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_removed_branch_is_hidden_and_the_epoch_bumps_once", + "engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "api/crates/layerfs-api/tests/gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches"], + note: "Reserved main, absent Branch and holder details are asserted; project-side name collisions are in AC-30." }, + Row { id: "AC-10", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position"], + note: "Removed/UpToDate codes and the frontier; 2.0 frees no bytes for the removed path." }, + Row { id: "AC-11", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount"], + note: "Non-FF whiteouts, the fast-forward control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute. A merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed." }, + Row { id: "AC-12", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit"], + note: "A removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept, and only an ordinary edit conflict on it is reported." }, + Row { id: "AC-13", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit"], + note: "The earlier Commit on an unmerged Branch is hidden by the marker; a Commit after the frontier does not inherit the path." }, + Row { id: "AC-14", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes"], + note: "A Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount; its next Commit records the deletion." }, + Row { id: "AC-15", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Retention of the only-reaching-path object is proven; marker hiding in reads/listings and derived whiteouts are not in one named test." }, + Row { id: "AC-16", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "T12.1 pins a retained retired Branch name and a collected unreferenced one; the branch.remove-then-rebuild arm is exercised through those markers." }, + Row { id: "AC-17", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/kept_objects.rs#a_delta_base_is_enqueued_and_reported"], + note: "Depth-2 chain copied, pack ids preserved." }, + Row { id: "AC-18", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Collected digest row gone, retained one kept." }, + Row { id: "AC-19", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers", + "engine/crates/layerfs-persistence/tests/path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position"], + note: "Path-grammar refusals and the marker frontier are named; the full form table (symlink/hardlink, >4096 bytes, >256 components) is not one test." }, + Row { id: "AC-20", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date", + "engine/crates/layerfs-persistence/tests/path_removal.rs#an_unknown_project_is_missing"], + note: "UpToDate and never-existing paths are named; the re-creation and other-project arms are partial." }, + Row { id: "AC-21", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back", + "engine/crates/layerfs-persistence/tests/epoch_fence.rs#a_concurrent_commit_and_removal_have_one_winner"], + note: "One-shot writes racing removals roll back as Conflict; the merge-racing-remove_file arm is not named." }, + Row { id: "AC-22", backend: Backend::Both, tier: "E3", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible", + "engine/crates/layerfs-persistence/tests/epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back"], + note: "A deterministic barrier starts a merge and a `remove_file` together for three rounds: the loser sees the marker or gets `Conflict` and retries with the whiteout, and a tree derived afterwards never shows the path. The PostgreSQL half is T12.3b." }, + Row { id: "AC-23", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/project_delete.rs#deleting_a_project_hides_it_and_marks_once"], + note: "Deleted-project rows and markers are collected by the rebuild; a dedicated path_removed cleanup arm is not named." }, + Row { id: "AC-24", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order", + "engine/crates/layerfs-persistence/tests/gc_backfill.rs#existing_serials_are_kept_and_new_ones_start_above_the_maximum"], + note: "Serials backfilled, next_history_serial never falls." }, + Row { id: "AC-25", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Retained branch_retired/layer_stack_retired markers survive; collected rows take their markers." }, + Row { id: "AC-26", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/project_delete.rs#retry_classifies_retired_live_and_absent", + "engine/crates/layerfs-persistence/tests/project_delete.rs#deleting_a_project_hides_it_and_marks_once"], + note: "Hidden, entry present, retry returns the stored deleted_ms." }, + Row { id: "AC-27", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/project_delete.rs#paging_skips_a_hidden_project", + "engine/crates/layerfs-persistence/tests/project_delete.rs#every_layer_stack_lookup_carries_the_marker_filter"], + note: "Stale home entries stay out of list/lookup." }, + Row { id: "AC-28", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase C cleanup and its failpoints: T12.3b." }, + Row { id: "AC-29", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved", + "engine/crates/layerfs-persistence/tests/admission.rs#a_mount_blocks_all_three_removals_until_released"], + note: "Owner liveness and Unresolved settlement; machine_stopped and the other-home intent are partial." }, + Row { id: "AC-30", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "api/crates/layerfs-api/tests/gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect"], + note: "Holder detail and refusal codes are named; the full exact-message matrix (AC-55) is partial." }, + Row { id: "AC-31", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["api/crates/layerfs-api/tests/gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log", + "api/crates/layerfs-api/tests/gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt"], + note: "Byte-count path arguments and the keyed trace hash are asserted." }, + Row { id: "AC-32", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved"], + note: "Live/uncertain ownership is named; live-but-unreachable, machine_stopped and another home's intent are partial." }, + Row { id: "AC-33", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_checkpoint_busy", "runner#arm_sidecar_at_install"], + note: "A live reader makes wal_checkpoint(TRUNCATE) Busy with no artifact and no op; a -mj* super-journal at install is Conflict with no writes." }, + Row { id: "AC-34", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active"], + note: "A committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains. The failed-delete sub-arm is not inducible on SQLite: the storage schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register too; the explicit `Cleanup` report is covered by the seam's code path, not executed here. The PostgreSQL half is T12.3b." }, + Row { id: "AC-35", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL advisory-key refcount: T12.3b." }, + Row { id: "AC-36", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["control/crates/layerfs-control/tests/project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry", + "control/crates/layerfs-store/tests/project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry", + "control/crates/layerfs-store/tests/project_delete.rs#an_absent_stack_finishes_a_stale_entry"], + note: "A lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent. The PostgreSQL half is T12.3b." }, + Row { id: "AC-37", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/gc_trace.rs#the_redacted_set_covers_every_path_bearing_call", + "api/crates/layerfs-api/tests/gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt"], + note: "Redaction and the keyed hash are named; salt backfill on an upgraded Store is partial." }, + Row { id: "AC-38", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle"], + note: "The SQLite artifact set is removed exactly; the PostgreSQL schema/name arms are T12.3b." }, + Row { id: "AC-39", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order"], + note: "One project_epoch row per stack, gc_op not copied, next_history_serial never lowered." }, + Row { id: "AC-40", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/kept_objects.rs#the_digest_is_stable_and_moves_with_a_head"], + note: "SQLite live_heads_digest is identical before/after and moves with a head; PostgreSQL agreement is deferred." }, + Row { id: "AC-41", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase M mark transfer: T12.3b." }, + Row { id: "AC-42", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL reserved pack/Phase B: T12.3b." }, + Row { id: "AC-43", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase C cleanup resume: T12.3b." }, + Row { id: "AC-44", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL privileges and lost COMMIT reply: T12.3b." }, + Row { id: "AC-45", backend: Backend::Sqlite, tier: "E1", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#group_rows_indexed_layout_opens_compacts_and_takes_none", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#acquisition_layout_opens_compacts_and_clears_working_rows", + "engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs#explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order"], + note: "Monolithic/GroupRows/GroupRowsIndexed and the acquisition variants compact; working rows are cleared." }, + Row { id: "AC-46", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_sidecar_at_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction"], + note: "Sealed-state/sidecar change at install gives Conflict with no writes; a replaced Store gives StaleStore. The in-flight-transaction exclusion is a documented lane limit, not probed." }, + Row { id: "AC-47", backend: Backend::Sqlite, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Crash leaves the full set, the next compact removes it, none leaves no artifact, a stray name is Exists." }, + Row { id: "AC-48", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#an_empty_store_takes_the_none_path", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_capacity_failure_leaves_the_store_unchanged"], + note: "9/10/30 percent, empty-store zeroReason, capacity abort; verify-failure and re-refine arms are partial." }, + Row { id: "AC-49", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_mount_during_compaction_is_busy", + "runner#arm_commit_during_rebuild", + "engine/crates/layerfs-persistence/tests/admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner"], + note: "SQLite half executed: a mount during compact is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild gets `StaleStore` or lands, and its retry is never lost. The PG Phase A/Phase C arms are T12.3b." }, + Row { id: "AC-50", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["api/crates/layerfs-api/tests/calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover", + "api/crates/layerfs-api/tests/gc_calls.rs#the_four_gc_calls_take_ids_only"], + note: "Folder scope refuses the four calls; the admin profile is the only allowance." }, + Row { id: "AC-51", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_status_phases", + "control/crates/layerfs-store/tests/compact_report_status.rs#status_projects_the_sidecar_phase_and_state", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal"], + note: "The op sidecar records mark/build/verify/install (runner); T10 projects it through store.status, and the named control test asserts the phase, state and phases list." }, + Row { id: "AC-52", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json", + "control/crates/layerfs-store/tests/compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape"], + note: "Every section 11 field pinned with units and sampling point (golden) and rendered by the T10 wire-shape test; markBytes is a metric, not a result field." }, + Row { id: "AC-53", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/project_export.rs#export_omits_a_path_hidden_by_remove_file", + "api/crates/layerfs-api/tests/gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers"], + note: "One hidden-path export case is named; the full read-surface table (read/stat/list/glob/diff/revision/edit/grep/blame/cp/apply) is not executed." }, + Row { id: "AC-54", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["control/crates/layerfs-store/tests/restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install", + "control/crates/layerfs-store/tests/restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it"], + note: "Restore install guards and restored records are named; restore before/after a removal and the stale-entry listing arm are partial. The restore-window arm needs a lock-capable filesystem (ext4/xfs/btrfs), so on this Sandbox it reports Unsupported while the other two cases pass." }, + Row { id: "AC-55", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_error_codes", "runner#arm_checkpoint_busy", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted"], + note: "Exact SQLite texts asserted: Busy, Conflict(\"Store sidecar at install\"), Exists(\"unrecorded compaction artifact\"), Capacity(\"disk space\"); the full engine/API message matrix stays partial." }, + Row { id: "AC-56", backend: Backend::Both, tier: "E3", status: Status::Partial, + + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json"], + note: "The literal SQLite vector is pinned (absent head 33 zero bytes); the shared PostgreSQL literal and the three-run E3 shape are deferred." }, + Row { id: "AC-57", backend: Backend::Both, tier: "E3", status: Status::Partial, + evidence: &["runner#arm_measurement", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json", + "engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs#capacity_measure", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json"], + note: "The tiny three-run arm is median 28.2 ms. T-G61 measured a real 9.987 GB Store with 3 runs per arm: `none` at 8.88% dead (median 149.318 s) and `rebuild` at 10.00% dead (median 246.371 s, mark 149.089 s / build 94.317 s / verify 0.312 s / install 1.916 s), plus one WAL/FULL rebuild at 284.078 s; profile, host and toolchain are declared in the fixture, and the warm page cache is declared below. The section 13 50 GB arms, the 30% arm, each removal type, the plain `VACUUM INTO` comparison and PostgreSQL timings remain deferred/[E]." }, + Row { id: "AC-58", backend: Backend::Sqlite, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy"], + note: "none deletes the scratch and leaves no .compact; rebuild keeps the visited scratch through the build; step 2 writes only on rebuild." }, + Row { id: "AC-59", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_checkpoint_busy", "runner#arm_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale"], + note: "busy != 0 refuses Busy and the WAL is never unlinked; a successful seal records (size, mtime, counter) as fenced by the StaleStore arm." }, + Row { id: "AC-60", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL synchronous standby handling: T12.3b." }, +]; + +// ---- report --------------------------------------------------------------- + +fn evidence_label(entry: &str) -> String { + if let Some(arm) = entry.strip_prefix("runner#") { + return format!("this runner #{arm}"); + } + let (file, function) = entry.split_once('#').unwrap_or((entry, "")); + let base = file.rsplit('/').next().unwrap_or(file); + // T-G61's opt-in capacity suite is labelled with its ticket in the matrix + // evidence; its measured numbers are in the notes and run-environment page. + if base == "sqlite_compact_capacity.rs" { + return format!("T-G61 {base}#{function}"); + } + if function.is_empty() { + base.to_owned() + } else { + format!("{base}#{function}") + } +} + +fn backend_label(backend: Backend) -> &'static str { + match backend { + Backend::Both => "both", + Backend::Sqlite => "SQLite", + Backend::Pg => "PG", + } +} + +fn status_label(status: Status) -> &'static str { + match status { + Status::Executed => "executed", + Status::Covered => "covered", + Status::Partial => "partial", + Status::Deferred => "deferred (T12.3b)", + Status::Gap => "gap", + } +} + +/// The crash suite's test count and its real-process SIGKILL arms, read from +/// its source so the report's T-G28/T-G60 note cannot drift when an arm is +/// added: the in-process `crash_at!` arms, the `sigkill_at!` arms and the +/// crash child itself are every test the suite declares on Linux. +fn crash_suite_counts() -> (usize, usize) { + let path = + repo_root().join("engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; the crash suite the report counts", + path.display() + ) + }); + let sigkill_arms = text.matches("sigkill_at!(").count(); + let total = text.matches("crash_at!(").count() + + sigkill_arms + + text.matches("fn sigkill_at_failpoint(").count(); + (total, sigkill_arms) +} + +fn generate_report() -> String { + use std::fmt::Write as _; + let count = |status: Status| MATRIX.iter().filter(|row| row.status == status).count(); + let both = MATRIX.iter().filter(|row| row.backend == Backend::Both).count(); + let sqlite = MATRIX + .iter() + .filter(|row| row.backend == Backend::Sqlite) + .count(); + let pg = MATRIX.iter().filter(|row| row.backend == Backend::Pg).count(); + let mut out = String::new(); + writeln!(out, "# T12.3a: SQLite acceptance-matrix runner report").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "> Generated by `cargo test -p layerfs-persistence --test sqlite_compact_matrix`." + ) + .unwrap(); + writeln!( + out, + "> The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) section 12 and" + ) + .unwrap(); + writeln!( + out, + "> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) AC-01..AC-60 win; this" + ) + .unwrap(); + writeln!( + out, + "> page records evidence and adds no contract. Regenerate with" + ) + .unwrap(); + writeln!( + out, + "> `UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix`." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Scope and counts").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- Rows: {} (AC-01..AC-60); backend split: both {both}, SQLite-only {sqlite}, PG-only {pg}.", + MATRIX.len() + ) + .unwrap(); + writeln!( + out, + "- Executed here: {}; covered by a named existing test/fixture: {}; partial: {}; deferred to T12.3b: {}; SQLite gaps: {}.", + count(Status::Executed), + count(Status::Covered), + count(Status::Partial), + count(Status::Deferred), + count(Status::Gap) + ) + .unwrap(); + let (crash_tests, crash_sigkill_arms) = crash_suite_counts(); + writeln!( + out, + "- **T-G28/T-G60 follow-up.** T-G28 later added the real-process SIGKILL arms" + ) + .unwrap(); + writeln!( + out, + " and T-G60 extended them to mark, copy, verify and rename, so" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_crash` now has {crash_tests} tests ({crash_sigkill_arms} real-process SIGKILL arms and the" + ) + .unwrap(); + writeln!( + out, + " child placeholder), not the 13 recorded below. Those arms still kill a" + ) + .unwrap(); + writeln!( + out, + " re-executed test child, so T-G65 added" + ) + .unwrap(); + writeln!( + out, + " `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs`: 8 SIGKILL" + ) + .unwrap(); + writeln!( + out, + " arms on the built `layerfs store compact` process at mark, build, verify and" + ) + .unwrap(); + writeln!( + out, + " install, so AC-01 is `covered` (the matrix now reads covered 30 / partial" + ) + .unwrap(); + writeln!( + out, + " 16). This page keeps the T12.3c run's original counts." + ) + .unwrap(); + writeln!( + out, + "- A `covered`/`partial` row names a file and test function that this runner" + ) + .unwrap(); + writeln!( + out, + " asserts exists; an `executed` row names an arm this binary really runs." + ) + .unwrap(); + writeln!( + out, + "- A cross-backend row records its SQLite half only; every PostgreSQL half" + ) + .unwrap(); + writeln!( + out, + " and the PostgreSQL-only rows are deferred to T12.3b by name." + ) + .unwrap(); + writeln!( + out, + "- **T-G61 capacity follow-up.** A separate opt-in test," + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_capacity.rs#capacity_measure` (ignored by default), measured a" + ) + .unwrap(); + writeln!( + out, + " real **9.987 GB** Store: `none` at 8.88% dead (three runs, median 149.318 s)" + ) + .unwrap(); + writeln!( + out, + " and `rebuild` at 10.00% dead (three runs, median 246.371 s; mark 149.089 s," + ) + .unwrap(); + writeln!( + out, + " build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink" + ) + .unwrap(); + writeln!( + out, + " 1.017 GB, peak temp 9.144 GB), plus one WAL/FULL rebuild (284.078 s). The" + ) + .unwrap(); + writeln!( + out, + " numbers are in `tests/fixtures/sqlite_compact_capacity_measure.json`; the" + ) + .unwrap(); + writeln!( + out, + " 50 GB arms and the 30% arm remain [E]." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Matrix (SQLite half)").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "| Case | Backend | Matrix tier | Disposition | Evidence | Note |" + ) + .unwrap(); + writeln!(out, "|---|---|---|---|---|---|").unwrap(); + for row in &MATRIX { + let evidence = if row.evidence.is_empty() { + "-".to_owned() + } else { + row.evidence + .iter() + .map(|entry| evidence_label(entry)) + .collect::>() + .join("; ") + }; + writeln!( + out, + "| {} | {} | {} | {} | {} | {} |", + row.id, + backend_label(row.backend), + row.tier, + status_label(row.status), + evidence, + row.note + ) + .unwrap(); + } + writeln!(out).unwrap(); + writeln!(out, "## Executed in this runner").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- AC-33 / AC-59: `arm_checkpoint_busy` - a live reader makes" + ) + .unwrap(); + writeln!( + out, + " `wal_checkpoint(TRUNCATE)` report busy, the compaction refuses `Busy`," + ) + .unwrap(); + writeln!( + out, + " leaves no `.compact`, `.compact-visited` or `.compact-op`, and the WAL is" + ) + .unwrap(); + writeln!( + out, + " never unlinked; the next run rebuilds at `G -> G+1`." + ) + .unwrap(); + writeln!( + out, + "- AC-33 / AC-46: `arm_sidecar_at_install` - a `-mj*` super-journal placed" + ) + .unwrap(); + writeln!( + out, + " at install is `Conflict(\"Store sidecar at install\")` with no artifact," + ) + .unwrap(); + writeln!( + out, + " no op and an unchanged generation." + ) + .unwrap(); + writeln!( + out, + "- AC-51: `arm_status_phases` - stop points at mark, build, verify and install" + ) + .unwrap(); + writeln!( + out, + " record the same phase in `.compact-op`; every phase is observed." + ) + .unwrap(); + writeln!( + out, + "- AC-55: `arm_error_codes` - exact `Exists(\"unrecorded compaction artifact\")` and" + ) + .unwrap(); + writeln!( + out, + " `Capacity(\"disk space\")` texts, with no artifact after the capacity abort." + ) + .unwrap(); + writeln!( + out, + "- AC-59: `arm_seal` - a successful seal leaves no sidecar, reports the main-file" + ) + .unwrap(); + writeln!( + out, + " bytes and a SQLite version, and the file reopens." + ) + .unwrap(); + writeln!( + out, + "- AC-04 / AC-05: `arm_random_store_shapes` - a seeded property arm over six" + ) + .unwrap(); + writeln!( + out, + " reachability shapes: every live object stays readable, every wholly dead one" + ) + .unwrap(); + writeln!( + out, + " is collected, the retained-closure digest is unchanged and the doctor is clean." + ) + .unwrap(); + writeln!( + out, + "- AC-06: `arm_digest_row_after_collection` - a `file_digest` row named a" + ) + .unwrap(); + writeln!( + out, + " root the rebuild collected, so the row goes; recording the same content" + ) + .unwrap(); + writeln!( + out, + " again succeeds and points at the new root." + ) + .unwrap(); + writeln!( + out, + "- AC-07 / AC-49: `arm_mount_and_compaction_exclude_each_other` - a live mount" + ) + .unwrap(); + writeln!( + out, + " makes compaction `MountsActive` naming it, and a live compaction makes a" + ) + .unwrap(); + writeln!( + out, + " mount and a second compaction `BusyOp`; the refused runs leave no op." + ) + .unwrap(); + writeln!( + out, + "- AC-49: `arm_mount_during_compaction_is_busy` - the stopped run's live" + ) + .unwrap(); + writeln!( + out, + " compaction intent refuses a mount with `BusyOp`, and the next run adopts" + ) + .unwrap(); + writeln!( + out, + " the leftover and rebuilds." + ) + .unwrap(); + writeln!( + out, + "- AC-49: `arm_commit_during_rebuild` - a Commit started inside the install" + ) + .unwrap(); + writeln!( + out, + " fence is `StaleStore` or lands on `G + 1`; a retry is never lost." + ) + .unwrap(); + writeln!( + out, + "- AC-57: `arm_measurement` - three runs of the same rebuild with median and" + ) + .unwrap(); + writeln!( + out, + " range in `engine/crates/layerfs-persistence/tests/fixtures/" + ) + .unwrap(); + writeln!( + out, + " sqlite_compact_measure.json` (refresh with `UPDATE_MEASURE=1`). The separate" + ) + .unwrap(); + writeln!( + out, + " T-G61 `sqlite_compact_capacity.rs#capacity_measure` measures a real 9.987 GB" + ) + .unwrap(); + writeln!( + out, + " Store (three `none` + three `rebuild` runs, one WAL/FULL control) into" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_capacity_measure.json`; it is opt-in because the fixture is" + ) + .unwrap(); + writeln!( + out, + " ~10 GB and the ignored test is not part of a normal suite." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Deferred to T12.3b (PostgreSQL)").unwrap(); + writeln!(out).unwrap(); + for row in MATRIX.iter().filter(|row| row.status == Status::Deferred) { + writeln!(out, "- {} - {}", row.id, row.note).unwrap(); + } + writeln!( + out, + "- The PostgreSQL half of every `both` row (digest agreement, artifact" + ) + .unwrap(); + writeln!( + out, + " names, Phase M/B/C, standby, lost reply, 50 GB arms)." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## SQLite gaps (not claimed by this runner)").unwrap(); + writeln!(out).unwrap(); + if MATRIX.iter().all(|row| row.status != Status::Gap) { + writeln!( + out, + "- none; every SQLite gap this runner can execute is executed or covered." + ) + .unwrap(); + } else { + for row in MATRIX.iter().filter(|row| row.status == Status::Gap) { + writeln!(out, "- {} - {}", row.id, row.note).unwrap(); + } + } + writeln!(out).unwrap(); + writeln!(out, "## E3 arms").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- AC-04: the T12.1 closure fixture (deterministic) plus the seeded" + ) + .unwrap(); + writeln!( + out, + " reachability property arm; the full random history generator is deferred." + ) + .unwrap(); + writeln!( + out, + "- AC-22: a deterministic barrier starts a merge and a `remove_file` together" + ) + .unwrap(); + writeln!( + out, + " for three rounds; the loser retries and a derived tree never shows the path." + ) + .unwrap(); + writeln!( + out, + "- AC-56: the SQLite literal vector is pinned by the T12.1 golden; the shared" + ) + .unwrap(); + writeln!(out, " PostgreSQL literal is deferred.").unwrap(); + writeln!( + out, + "- AC-57: three-run measurement arm executed on this Sandbox (small fixture)," + ) + .unwrap(); + writeln!( + out, + " plus T-G61's real 9.987 GB capacity fixture with three runs per arm" + ) + .unwrap(); + writeln!( + out, + " (`none` 149.318 s median, `rebuild` 246.371 s median, phase sums recorded)." + ) + .unwrap(); + writeln!( + out, + " Its 50 GB projection, the 30% arm, PG `ACCESS EXCLUSIVE` duration and the" + ) + .unwrap(); + writeln!( + out, + " plain `VACUUM INTO` comparison are deferred/[E]." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Status and report projection").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "T10 (merged from main after this branch forked) projects `.compact-op`" + ) + .unwrap(); + writeln!( + out, + "through `store.status`: `op {{id, name, phase, started, uncertain}}`, the" + ) + .unwrap(); + writeln!( + out, + "busy/unresolved `state` and the backend `phases`. The named control tests" + ) + .unwrap(); + writeln!( + out, + "in AC-51 and AC-52 assert that projection and the snake_case section 11" + ) + .unwrap(); + writeln!( + out, + "render; the runner still asserts the recorded SQLite phases directly, so" + ) + .unwrap(); + writeln!( + out, + "AC-51 does not depend on the projection alone." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Run environment").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- T-G61 capacity run (separate opt-in test): Docker Sandbox `g61sbx`" + ) + .unwrap(); + writeln!( + out, + " (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 overlay, 4.0 TB free), rustc" + ) + .unwrap(); + writeln!( + out, + " 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF) plus one" + ) + .unwrap(); + writeln!( + out, + " `Durable` control, `CARGO_TARGET_DIR=/var/tmp/target-g61`, fixture under" + ) + .unwrap(); + writeln!( + out, + " `/var/tmp/g61big`; the test ran 1678 s. Caches: the ~10 GB Store fits in the" + ) + .unwrap(); + writeln!( + out, + " 135 GB page cache and no cache drop ran between the three runs of an arm;" + ) + .unwrap(); + writeln!( + out, + " the mark is CPU-bound at ~62 MB/s, not cache-bound." + ) + .unwrap(); + writeln!( + out, + "- Docker Sandbox `t123csbx` (Linux x86_64, tmpfs `/tmp`, overlayfs" + ) + .unwrap(); + writeln!( + out, + " `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable`" + ) + .unwrap(); + writeln!( + out, + " (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`." + ) + .unwrap(); + writeln!( + out, + "- Verified here: `sqlite_compact_matrix` (1 test); the persistence suites" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact` (20), `sqlite_compact_crash` (13)," + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_fixtures` (1), `admission` (5), `path_removal` (3)," + ) + .unwrap(); + writeln!( + out, + " `branch_removal` (3), `project_delete` (5), `epoch_fence` (7)," + ) + .unwrap(); + writeln!( + out, + " `gc_backfill` (4), `kept_objects` (5); the Store-service path suites" + ) + .unwrap(); + writeln!( + out, + " `sdk_path_merge_removal` (5), `sdk_mount_bind_failure` (1)," + ) + .unwrap(); + writeln!( + out, + " `sdk_path_whiteouts` (7), `sdk_path_hidden_semantics` (2)," + ) + .unwrap(); + writeln!( + out, + " `sdk_path_removal` (2); and `layerfs-control --features postgresql" + ) + .unwrap(); + writeln!( + out, + " project_delete_worker` (5), `layerfs-store --features postgresql" + ) + .unwrap(); + writeln!( + out, + " project_delete` (15) and `compact_report_status` (3)." + ) + .unwrap(); + writeln!( + out, + " All of those pass on the merged branch." + ) + .unwrap(); + writeln!( + out, + "- The control `restore_install` arm that opens a Store needs ext4/xfs/btrfs;" + ) + .unwrap(); + writeln!( + out, + " this Sandbox has no such direct filesystem, so that one case reports" + ) + .unwrap(); + writeln!( + out, + " `Unsupported` while its other two cases pass. It is an environment limit," + ) + .unwrap(); + writeln!( + out, + " not a matrix result; the cited API tests are validated by source name only" + ) + .unwrap(); + writeln!( + out, + " in this run." + ) + .unwrap(); + out +} + +fn validate_registry(executed: &BTreeSet) { + assert_eq!(MATRIX.len(), 60, "AC-01..AC-60 are 60 rows"); + for (index, row) in MATRIX.iter().enumerate() { + let expected = format!("AC-{:02}", index + 1); + assert_eq!(row.id, expected, "rows are in id order"); + if row.status == Status::Deferred { + assert!( + row.backend == Backend::Pg || row.note.contains("PostgreSQL"), + "{} is deferred but not PostgreSQL-only", + row.id + ); + continue; + } + if row.status == Status::Gap { + assert!( + row.evidence.is_empty(), + "{} is a gap but names evidence", + row.id + ); + continue; + } + assert!(!row.evidence.is_empty(), "{} has no evidence", row.id); + for entry in row.evidence { + if let Some(arm) = entry.strip_prefix("runner#") { + assert!(executed.contains(arm), "{} names unexecuted arm {arm}", row.id); + continue; + } + let (file, function) = entry.split_once('#').unwrap_or((entry, "")); + let path = repo_root().join(file); + assert!(path.exists(), "{}: evidence file missing: {file}", row.id); + if !function.is_empty() { + let text = std::fs::read_to_string(&path).unwrap(); + assert!( + text.contains(function), + "{}: {file} does not name {function}", + row.id + ); + } + } + if row.status == Status::Executed { + assert!( + row.evidence.iter().any(|entry| entry.starts_with("runner#")), + "{} is executed but names no arm", + row.id + ); + } + } +} + +#[test] +fn the_sqlite_half_of_the_acceptance_matrix_is_executed_and_reported() { + let mut executed = BTreeSet::new(); + for (name, arm) in ARMS { + arm(); + executed.insert(name.to_owned()); + } + validate_registry(&executed); + + let report = generate_report(); + let path = report_path(); + if std::env::var_os("UPDATE_REPORT").is_some() { + std::fs::write(&path, report.as_bytes()).unwrap(); + } else { + let committed = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; run UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix", + path.display() + ) + }); + assert_eq!( + committed, report, + "the report is stale; rerun with UPDATE_REPORT=1" + ); + } + + let count = |status: Status| MATRIX.iter().filter(|row| row.status == status).count(); + println!( + "T12.3a SQLite matrix: {} rows | executed {} | covered {} | partial {} | deferred {} | gap {}", + MATRIX.len(), + count(Status::Executed), + count(Status::Covered), + count(Status::Partial), + count(Status::Deferred), + count(Status::Gap) + ); +} diff --git a/engine/crates/layerfs-persistence/tests/support/metrics.rs b/engine/crates/layerfs-persistence/tests/support/metrics.rs new file mode 100644 index 0000000000..e1024b545d --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/support/metrics.rs @@ -0,0 +1,91 @@ +#![allow(dead_code)] +//! Shared metric assertions for the compaction captures. +//! +//! `compact_metrics` (SQLite §7) and `postgresql_compact` (PostgreSQL §8) +//! both read a `CompactMetricsCapture`, so the closed label dimensions and +//! values are checked once here for both backends. + +use layerfs_persistence::compact_metrics::CompactMetrics; + +/// The value of one sample, or `None` when this run did not emit it. +pub fn metric(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> Option { + metrics + .samples() + .into_iter() + .find(|sample| { + sample.name == name + && labels + .iter() + .all(|(key, value)| sample.label(key) == Some(*value)) + }) + .map(|sample| sample.value) +} + +/// The value of one sample the run must emit. +pub fn value(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> u64 { + metric(metrics, name, labels).unwrap_or_else(|| { + panic!( + "no sample {name} {labels:?}; emitted: {:?}", + metrics.samples() + ) + }) +} + +/// Bounded cardinality: every label key is a closed dimension and every value +/// is one the enums name, never a Store name, path, operation id or artifact. +pub fn assert_bounded(metrics: &CompactMetrics) { + let keys = [ + "outcome", + "code", + "kind", + "reason", + "state", + "objective", + "result", + "phase", + "statistic", + "which", + "part", + ]; + let values = [ + "rebuild", + "none", + "failed", + "uncertain", + "busy", + "unresolved", + "mark", + "build", + "verify", + "install", + "pg_mark", + "pg_build", + "pg_swap", + "pg_cleanup", + "run", + "leftover", + "count", + "sum", + "max", + "before", + "after", + "bound", + "refined", + "dead_in_retained_packs", + "delta_bases", + "inode_blocks", + "residue_free", + "generation", + "pass", + "fail", + ]; + for sample in metrics.samples() { + for (key, label) in &sample.labels { + assert!(keys.contains(key), "non-closed label key {key}"); + assert!( + values.contains(label), + "non-closed label value {label} for {key}" + ); + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/support/mod.rs b/engine/crates/layerfs-persistence/tests/support/mod.rs index cc9912afde..3a68d87e6f 100644 --- a/engine/crates/layerfs-persistence/tests/support/mod.rs +++ b/engine/crates/layerfs-persistence/tests/support/mod.rs @@ -141,5 +141,7 @@ pub mod sqlite { pub mod doctor; +pub mod metrics; + #[cfg(feature = "postgresql")] pub mod pg; diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index b57dc3c6a7..d3d581edca 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -129,21 +129,21 @@ fn mounts_detail(state: &layerfs_history::MountHolders) -> SdkValue { holder .workspace .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ( "sandbox", holder .sandbox .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ( "machine", holder .machine .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ("op", SdkValue::text(&holder.op)), ]) @@ -163,10 +163,7 @@ fn op_detail(state: &layerfs_history::OpConflict) -> SdkValue { ), ( "phase", - state - .phase - .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + state.phase.as_ref().map_or(SdkValue::Null, SdkValue::text), ), ("op", SdkValue::text(&state.op)), ("uncertain", SdkValue::Bool(state.uncertain)), diff --git a/engine/crates/layerfs-store-service/src/methods/file/blame.rs b/engine/crates/layerfs-store-service/src/methods/file/blame.rs index 1c9a8dcaf4..176f29da1c 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/blame.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/blame.rs @@ -17,8 +17,10 @@ pub fn blame(service: &StoreService, args: &SdkValue) -> Answer { &snap, &path, &seen, - line(args, "fromLine", 1), - line(args, "lineCount", 2000), + blame::Window { + from: line(args, "fromLine", 1), + count: line(args, "lineCount", 2000), + }, &mut blame::Limits { versions: crate::read::query::MAXIMUM_QUERY_WORK, bytes: 64 * 1024 * 1024, diff --git a/engine/crates/layerfs-store-service/src/read/attribution.rs b/engine/crates/layerfs-store-service/src/read/attribution.rs index e08f7df31a..fc67983235 100644 --- a/engine/crates/layerfs-store-service/src/read/attribution.rs +++ b/engine/crates/layerfs-store-service/src/read/attribution.rs @@ -61,8 +61,18 @@ fn owners( versions: budget.work, bytes: budget.bytes, }; - let page = - blame::blame(store, reader, snap, path, visibility, Ok(from), Ok(2000), &mut limits)?; + let page = blame::blame( + store, + reader, + snap, + path, + visibility, + blame::Window { + from: Ok(from), + count: Ok(2000), + }, + &mut limits, + )?; budget.work = limits.versions; budget.bytes = limits.bytes; exact &= !page.get_bool("comparisonCoarse") diff --git a/engine/crates/layerfs-store-service/src/read/blame.rs b/engine/crates/layerfs-store-service/src/read/blame.rs index a0c1ba9bd1..4ca272f6e9 100644 --- a/engine/crates/layerfs-store-service/src/read/blame.rs +++ b/engine/crates/layerfs-store-service/src/read/blame.rs @@ -129,6 +129,12 @@ pub struct Limits { pub bytes: u64, } +/// The line window a blame page covers. +pub struct Window { + pub from: Arg, + pub count: Arg, +} + /// The origin Commit of each line of the regular file at `path`, from line /// `from` on, along persisted first parents. pub fn blame( @@ -137,8 +143,7 @@ pub fn blame( snap: &Snap, path: &str, visibility: &Visibility, - from: Arg, - count: Arg, + window: Window, limits: &mut Limits, ) -> Answer { let tip = snapshot::existing(reader, snap, path)?; @@ -152,8 +157,8 @@ pub fn blame( SdkValue::Null, )); } - let from = from?; - let count = count?; + let from = window.from?; + let count = window.count?; if count == 0 || count > window::LINES as u64 { return Err(args::invalid("lineCount must be in 1..=2000")); } diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs index f388a1b1a5..906659b46b 100644 --- a/engine/crates/layerfs-store-service/src/read/visibility.rs +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -66,10 +66,9 @@ impl Visibility { /// `load` for a stack id and an explicit `history_serial`, for reads that /// walk several nodes and therefore ask `hidden_at` per node. pub fn for_stack(store: &Arc, stack: LayerStackId, serial: u64) -> Self { - let markers = match store.published(|history| history.path_removals(stack)) { - Ok(markers) => markers, - Err(_) => Vec::new(), - }; + let markers = store + .published(|history| history.path_removals(stack)) + .unwrap_or_default(); Self { serial, markers } } diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index 36925dcd95..683cdfdbe9 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -223,7 +223,7 @@ fn change( let tree = TreeTable::open(&*reader, FilesystemRootId(base.effective_root)) .map_err(args::content)?; let removed = |path: &str| -> Result, SdkRefusal> { - let exists = snapshot::entry(&*reader, &tree, path)?.is_some(); + let exists = snapshot::entry(&reader, &tree, path)?.is_some(); Ok(exists .then(|| visibility.hidden(path)) .flatten() diff --git a/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs b/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs new file mode 100644 index 0000000000..07a8a7a982 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs @@ -0,0 +1,95 @@ +//! T12.3c: §12 AC-34, the mount bind-failure path over the SQLite Store +//! service. A committed mount intent whose bind fails after the admission is +//! deleted in the error path, so no `MountsActive` remains. The failed-delete +//! arm is not inducible on a SQLite Store: the storage schema guard rejects a +//! trigger on `gc_intent`, and an outside lock held across the bind would +//! refuse the register too, so the row names that clause as uncovered. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::SdkValue; +use layerfs_history::{BranchId, HistoryName, LayerStackId, WorkspaceId}; +use layerfs_store_service::store::{BindError, BindPhase, BindRequest}; +use support::Fixture; + +/// The installed fixture's project and a Branch `broken` whose stack profile +/// no longer matches its root: the bind's root check must fail after the +/// admission intent commits. The Commit itself is service-written, so every +/// history identity stays valid; only the stack row's profile changes. +fn break_root(fixture: &Fixture) -> BranchId { + let stack = LayerStackId::from_authority([23; 16]); + let branch = BranchId::named(stack, &HistoryName::new("broken").unwrap()); + fixture.fork("broken"); + fixture.apply("broken", vec![support::write("/b", "b\n")]); + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + let changed = connection + .execute( + "UPDATE layer_stack SET profile_id = ?1 WHERE layer_stack_id = ?2", + rusqlite::params![[0x42u8; 32].to_vec(), stack.as_slice().to_vec()], + ) + .unwrap(); + assert_eq!(changed, 1, "the fixture has one LayerStack"); + branch +} + +fn bind( + fixture: &Fixture, + branch: BranchId, + id: u8, +) -> Result<(), layerfs_store_service::store::BindRefusal> { + fixture + .service + .store + .bind( + fixture.owner.client(), + BindRequest { + workspace: WorkspaceId::from_authority([id; 32]).unwrap(), + branch, + }, + ) + .map(|_| ()) +} + +/// The committed admission intents the Store holds. +fn intents(fixture: &Fixture) -> Vec> { + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + let mut statement = connection.prepare("SELECT op_id FROM gc_intent").unwrap(); + statement + .query_map([], |row| row.get(0)) + .unwrap() + .map(|row| row.unwrap()) + .collect() +} + +#[test] +fn a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active() { + let fixture = Fixture::new("t123c-bind", |source| { + std::fs::write(source.join("a"), "a\n").unwrap(); + 1 + }); + let branch = break_root(&fixture); + + // The admission intent commits, the root check then fails: the refusal + // names the root phase and the error path deleted the intent. + let refusal = bind(&fixture, branch, 71).expect_err("the root is not a filesystem root"); + assert_eq!(refusal.phase, BindPhase::Root, "{refusal:?}"); + assert!( + matches!(refusal.error, BindError::Content { .. }), + "{refusal:?}" + ); + assert!( + intents(&fixture).is_empty(), + "a failed bind leaves no committed intent" + ); + // No `MountsActive` remains: the removal is admitted. + fixture.call( + "project.remove_file", + vec![("path", SdkValue::text("/a"))], + ); + + fixture.cleanup(); +} diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs b/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs new file mode 100644 index 0000000000..36c4fe95a9 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs @@ -0,0 +1,290 @@ +//! T12.3c: `project.remove_file` in derived trees (#52 §12 rows AC-11..AC-14 +//! and AC-22). A merge of at-or-below heads materializes the whiteout, a +//! post-frontier re-creation wins, a fast-forward merge creates no tree, a +//! publish of an at-or-below Commit yields a hidden Layer whose mount +//! whiteouts, an unmerged Branch that existed before the frontier hides but +//! does not inherit, and a merge started together with the removal never +//! leaves the path visible. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::fs; +use support::{write, Change, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn try_remove(fixture: &Fixture, path: &str) -> Result { + fixture.try_call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, branch: &str, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text(branch)), ("path", text(path))], + ) +} + +/// The text of a present file. +fn visible(fixture: &Fixture, branch: &str, path: &str) -> String { + let value = + read(fixture, branch, path).unwrap_or_else(|refusal| panic!("{branch}{path}: {refusal:?}")); + value + .get_text("text") + .unwrap_or_else(|| panic!("{branch}{path} has no text: {value:?}")) + .to_owned() +} + +/// The path is hidden by the marker at this node: the read refuses `Removed` +/// and names the frontier. +fn hidden(fixture: &Fixture, branch: &str, path: &str) { + let refusal = + read(fixture, branch, path).expect_err(&format!("{branch}{path} should be hidden")); + assert_eq!(refusal.code, "Removed", "{branch}{path}: {refusal:?}"); + let paths = match refusal.detail.get("paths") { + Some(SdkValue::List(paths)) => paths, + other => panic!("Removed detail without paths: {other:?}"), + }; + assert_eq!(paths.len(), 1, "{paths:?}"); + assert_eq!(paths[0].get_text("path"), Some(path), "{paths:?}"); + assert!( + paths[0].get_int("frontier_serial").is_some(), + "{paths:?}" + ); +} + +/// The path is absent from the committed tree: above the frontier the read +/// is `NoSuchPath`, never `Removed`. +fn absent(fixture: &Fixture, branch: &str, path: &str) { + let refusal = + read(fixture, branch, path).expect_err(&format!("{branch}{path} should be absent")); + assert_eq!(refusal.code, "NoSuchPath", "{branch}{path}: {refusal:?}"); +} + +/// Rows of one history table in the installed Store. +fn rows(fixture: &Fixture, table: &str) -> i64 { + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + connection + .query_row(&format!("SELECT count(*) FROM \"{table}\""), [], |row| { + row.get(0) + }) + .unwrap() +} + +fn merge(fixture: &Fixture, left: &str, right: &str, name: &str) -> Result { + fixture.try_call( + "branch.merge", + vec![ + ("branch", text(left)), + ("with", text(right)), + ("name", text(name)), + ], + ) +} + +#[test] +fn a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden() { + let fixture = Fixture::new("t123c-ff", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("left"); + fixture.apply("left", vec![write("/l", "l\n")]); + // `fast` starts at left's Commit and commits one child: left contains no + // `fast` node, while `fast` contains every `left` node. + fixture.call( + "branch.fork", + vec![("name", text("fast")), ("from", text("left"))], + ); + fixture.apply("fast", vec![write("/r", "r\n")]); + remove(&fixture, "/a"); + let (commits, layers) = (rows(&fixture, "commit"), rows(&fixture, "layer")); + + // One side contains the other: a fast-forward Branch at the newer node. + let merged = merge(&fixture, "left", "fast", "merged").expect("a fast-forward merge"); + assert!(merged.get_bool("fastForward"), "{merged:?}"); + assert_eq!( + rows(&fixture, "commit"), + commits, + "a fast-forward merge writes no new Commit" + ); + assert_eq!(rows(&fixture, "layer"), layers, "and no new Layer"); + + // The node it points at is at or below the frontier: the marker hides it + // without any whiteout, and the other files stay visible. + hidden(&fixture, "merged", "/a"); + assert_eq!(visible(&fixture, "merged", "/l"), "l\n"); + assert_eq!(visible(&fixture, "merged", "/r"), "r\n"); + fixture.cleanup(); +} + +#[test] +fn publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount() { + let fixture = Fixture::new("t123c-publish", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("work"); + fixture.apply("work", vec![write("/w", "w\n")]); + remove(&fixture, "/a"); + + // `work`'s Commit is at or below the frontier: publishing republishes its + // tree as a Layer at the same position, so the marker hides it on main. + let published = fixture.call("branch.publish", vec![("branch", text("work"))]); + assert_eq!(published.get_text("kind"), Some("Added"), "{published:?}"); + hidden(&fixture, "main", "/a"); + assert_eq!(visible(&fixture, "main", "/w"), "w\n"); + + // A Branch forked from main is layer-headed (no head Commit) and its base + // Layer is the published, at-or-below one: the mount whiteouts /a, and the + // next Commit records the deletion rather than inheriting it. + fixture.fork("layer"); + fixture.workspace_commit("layer", &[Change::Create("/n", b"n")], 41, || {}); + absent(&fixture, "layer", "/a"); + assert_eq!(visible(&fixture, "layer", "/n"), "n"); + assert_eq!(visible(&fixture, "layer", "/b"), "b\n"); + fixture.cleanup(); +} + +#[test] +fn an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit() { + let fixture = Fixture::new("t123c-unmerged", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + // The Branch and its Commit exist before the marker. + fixture.fork("old"); + fixture.apply("old", vec![write("/x", "x\n")]); + remove(&fixture, "/a"); + + // The earlier Commit's path is hidden by the marker. + hidden(&fixture, "old", "/a"); + assert_eq!(visible(&fixture, "old", "/x"), "x\n"); + + // A Commit after the frontier on that unmerged Branch does not inherit + // it: the derived tree records the absence. + fixture.apply("old", vec![write("/y", "y\n")]); + absent(&fixture, "old", "/a"); + assert_eq!(visible(&fixture, "old", "/x"), "x\n"); + assert_eq!(visible(&fixture, "old", "/y"), "y\n"); + fixture.cleanup(); +} + +#[test] +fn a_recreated_removed_path_still_conflicts_as_an_ordinary_edit() { + let fixture = Fixture::new("t123c-recreate-conflict", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + 1 + }); + // `side` stays at the base node (at or below the frontier). + fixture.fork("side"); + fixture.apply("side", vec![write("/s", "s\n")]); + remove(&fixture, "/a"); + + // Two Branches re-create /a after the frontier; both are above it, so the + // marker no longer applies and this is an ordinary edit conflict. + fixture.fork("one"); + fixture.apply("one", vec![write("/a", "one\n")]); + fixture.fork("two"); + fixture.apply("two", vec![write("/a", "two\n")]); + assert_eq!(visible(&fixture, "one", "/a"), "one\n"); + assert_eq!(visible(&fixture, "two", "/a"), "two\n"); + + let refusal = merge(&fixture, "one", "two", "conflicted").expect_err("an ordinary edit conflict"); + assert_eq!(refusal.code, "Conflict", "{refusal:?}"); + let conflicts = match refusal.detail.get("conflicts") { + Some(SdkValue::List(conflicts)) => conflicts, + other => panic!("Conflict detail without conflicts: {other:?}"), + }; + let paths: Vec<&str> = conflicts + .iter() + .map(|conflict| conflict.get_text("path").unwrap_or("")) + .collect(); + assert_eq!(paths, ["/a"], "{conflicts:?}"); + // A conflicted merge creates nothing. + let missing = fixture + .try_call( + "fs.read", + vec![("branch", text("conflicted")), ("path", text("/a"))], + ) + .expect_err("the conflicted merge created no Branch"); + assert_eq!(missing.code, "NoSuchBranch", "{missing:?}"); + fixture.cleanup(); +} + +#[test] +fn a_merge_and_a_removal_started_together_never_leave_the_path_visible() { + let fixture = Fixture::new("t123c-race", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + for round in 0..3u8 { + let left = format!("left-{round}"); + let right = format!("right-{round}"); + let merged = format!("merged-{round}"); + fixture.fork(&left); + fixture.fork(&right); + fixture.apply(&left, vec![write(&format!("/l-{round}"), "l\n")]); + fixture.apply(&right, vec![write(&format!("/r-{round}"), "r\n")]); + + // A deterministic barrier starts the merge and the removal together. + let barrier = std::sync::Barrier::new(2); + let (merge_result, remove_result) = std::thread::scope(|scope| { + let merge_barrier = &barrier; + let remove_barrier = &barrier; + let merging = scope.spawn(|| { + merge_barrier.wait(); + merge(&fixture, &left, &right, &merged) + }); + let removing = scope.spawn(|| { + remove_barrier.wait(); + try_remove(&fixture, "/a") + }); + (merging.join().unwrap(), removing.join().unwrap()) + }); + + // The loser retries: a removal refused by the live merge (or its + // operation) is repeated after it, and a merge refused by the epoch + // fence is repeated with the marker in place. + if let Err(refusal) = &remove_result { + assert!( + matches!(refusal.code.as_str(), "MountsActive" | "Busy" | "Unresolved"), + "round {round}: {refusal:?}" + ); + remove(&fixture, "/a"); + } + if let Err(refusal) = &merge_result { + assert_eq!(refusal.code, "Conflict", "round {round}: {refusal:?}"); + merge(&fixture, &left, &right, &merged).expect("the retried merge"); + } + + // Neither order may leave the removed path visible on the merge. + match read(&fixture, &merged, "/a") { + Ok(value) => panic!("round {round}: /a is visible: {value:?}"), + Err(refusal) => assert!( + matches!(refusal.code.as_str(), "Removed" | "NoSuchPath"), + "round {round}: {refusal:?}" + ), + } + // A tree derived above the frontier records the absence. + fixture.apply(&merged, vec![write(&format!("/z-{round}"), "z\n")]); + absent(&fixture, &merged, "/a"); + assert_eq!(visible(&fixture, &merged, &format!("/l-{round}")), "l\n"); + assert_eq!(visible(&fixture, &merged, &format!("/r-{round}")), "r\n"); + } + fixture.cleanup(); +} diff --git a/tools/ci/test_store_ext4.sh b/tools/ci/test_store_ext4.sh index 7497dbf955..f70e03652c 100755 --- a/tools/ci/test_store_ext4.sh +++ b/tools/ci/test_store_ext4.sh @@ -42,7 +42,13 @@ # LAYERFS_TEST_CARGO_TARGET_DIR # default $LAYERFS_TEST_TMPDIR/target # LAYERFS_TEST_POSTGRES_URL libpq connection string; enables the postgresql -# feature on every suite and the PostgreSQL cases +# feature on every suite and the PostgreSQL cases. +# Full-mode diagnostics need amcheck created in +# that database (see testing.md); the harness +# warns when psql can see that it is missing. +# LAYERFS_TEST_EXPECT_AMCHECK +# assert a Pass relation summary in +# postgresql_diagnostic (optional) # Arguments after `--` are appended to every `cargo test` (e.g. -- --nocapture). # # Exit status: 0 every suite passed; 1 at least one failed; 2 wrong command line @@ -286,6 +292,53 @@ run_cargo() { fi } +# ---- The PostgreSQL prerequisites ----------------------------------------- +# +# The full-mode doctor calls amcheck's verify_heapam and bt_index_check on +# every heap and ready B-tree relation, and the PostgreSQL compaction suites +# assert a clean full audit. A database without amcheck reports +# store.database.relations=Unknown, so those tests fail with a diagnostic +# finding, not an environment refusal. The engine never installs the +# extension: warn, never fail, so the SQLite-only lane and a deliberately +# amcheck-free server stay usable. psql is optional too; without it the check +# is skipped with a clear message. +postgresql_extension_count() { + psql "$LAYERFS_TEST_POSTGRES_URL" -X -A -t -v ON_ERROR_STOP=1 \ + -c "SELECT count(*) FROM pg_catalog.pg_extension WHERE extname='$1'" 2>/dev/null +} + +warn_about_postgresql_prerequisites() { + if ! command -v psql >/dev/null 2>&1; then + printf '%s\n' \ + "test_store_ext4: LAYERFS_TEST_POSTGRES_URL is set but psql is not on PATH;" \ + " the amcheck precondition was not checked. Full-mode diagnostics report" \ + " store.database.relations=Unknown without it. Create it out of band:" \ + " CREATE EXTENSION IF NOT EXISTS amcheck;" >&2 + return 0 + fi + local extension count + for extension in amcheck pgstattuple; do + if ! count=$(postgresql_extension_count "$extension"); then + printf 'test_store_ext4: cannot query LAYERFS_TEST_POSTGRES_URL; the %s precondition was not checked.\n' "$extension" >&2 + return 0 + fi + if [ "${count:-0}" -gt 0 ]; then + continue + fi + if [ "$extension" = amcheck ]; then + printf '%s\n' \ + "test_store_ext4: warning: the PostgreSQL test database has no amcheck extension." \ + " The full-mode doctor reports store.database.relations=Unknown and the" \ + " PostgreSQL compaction suites fail their clean-audit assertions (T-G14)." \ + " Create it in that database, then re-run:" \ + " psql \"\$LAYERFS_TEST_POSTGRES_URL\" -c 'CREATE EXTENSION IF NOT EXISTS amcheck;'" >&2 + else + printf 'test_store_ext4: note: %s is not installed (companion extension; amcheck is the required one).\n' "$extension" >&2 + fi + done + return 0 +} + printf '%s\n' \ "Store test harness" \ " repository: $root" \ @@ -297,6 +350,10 @@ printf '%s\n' \ " mode: $mode" \ " PostgreSQL: $([ -n "${LAYERFS_TEST_POSTGRES_URL:-}" ] && echo 'on (LAYERFS_TEST_POSTGRES_URL set)' || echo 'off (LAYERFS_TEST_POSTGRES_URL unset; PostgreSQL cases print NOT_RUN)')" +if [ -n "${LAYERFS_TEST_POSTGRES_URL:-}" ] && [ "$list_only" -eq 0 ]; then + warn_about_postgresql_prerequisites +fi + if [ "$mode" = quick ]; then run_cargo layerfs-persistence --test heads --test sqlite_publication run_cargo layerfs-store-service --test kernel_guard --test commit_pipeline