From 9c846c1ada109b8325f8d7b08f0a05096b860294 Mon Sep 17 00:00:00 2001 From: rain Date: Sun, 15 Nov 2026 23:55:01 +0000 Subject: [PATCH] feat(gc): report, API/MCP, acceptance and docs (#50-#52) --- .../references/methods/project.md | 47 +- .github/workflows/store-ext4.yml | 203 ++ Cargo.lock | 1 + api/README.md | 2 +- api/crates/layerfs-api/src/calls/mod.rs | 2 +- api/crates/layerfs-api/src/lib.rs | 2 +- api/crates/layerfs-api/src/project/mod.rs | 4 +- api/crates/layerfs-api/src/project/remove.rs | 24 - api/crates/layerfs-api/tests/calls_scope.rs | 1 - api/crates/layerfs-api/tests/calls_table.rs | 21 +- api/crates/layerfs-api/tests/gc_calls.rs | 27 + .../layerfs-api/tests/project_export.rs | 25 +- .../layerfs-api/tests/project_family.rs | 183 +- .../layerfs-api/tests/project_init_worker.rs | 11 +- .../layerfs-api/tests/store_projects.rs | 13 +- api/crates/layerfs-api/tests/store_stats.rs | 60 +- api/crates/layerfs-cli/tests/gc.rs | 11 + .../tests/sqlite_compact_process_kill.rs | 426 ++++ api/mcp/src/profiles.js | 18 +- api/mcp/src/tools.js | 5 + api/mcp/test/profiles.test.mjs | 14 +- api/mcp/test/workflow.test.mjs | 7 +- api/node/README.md | 2 +- api/node/test/branch-file/inventory.mjs | 4 +- api/node/test/calls.mjs | 4 +- api/python/test/test_calls.py | 2 +- api/scripts/gen-coverage.mjs | 5 +- .../crates/layerfs-control/src/inventory.rs | 12 - control/crates/layerfs-control/src/project.rs | 15 +- control/crates/layerfs-control/src/store.rs | 1 - .../layerfs-control/src/worker_server.rs | 14 +- .../layerfs-control/tests/in_use_detail.rs | 38 +- .../crates/layerfs-control/tests/inventory.rs | 7 - .../tests/project_delete_worker.rs | 114 +- .../tests/store_transfer_recovery.rs | 1 - .../layerfs-sandbox/src/daemon/channel.rs | 13 + .../layerfs-sandbox/src/daemon/internal.rs | 11 +- .../src/daemon/workspace/mount.rs | 14 +- .../src/daemon/workspace/recovery.rs | 27 +- .../crates/layerfs-sandbox/src/registry.rs | 4 +- .../layerfs-sandbox/src/workspace/serve.rs | 13 +- control/crates/layerfs-store/Cargo.toml | 6 + control/crates/layerfs-store/src/compact.rs | 74 +- .../layerfs-store/src/postgresql/connect.rs | 24 + .../crates/layerfs-store/src/project/add.rs | 28 +- .../layerfs-store/src/project/delete.rs | 27 +- .../layerfs-store/src/project/family.rs | 63 +- .../crates/layerfs-store/src/project/frame.rs | 38 +- .../crates/layerfs-store/src/project/mod.rs | 1 - .../layerfs-store/src/project/remove.rs | 68 - control/crates/layerfs-store/src/related.rs | 4 +- control/crates/layerfs-store/src/report.rs | 2 +- control/crates/layerfs-store/src/status.rs | 24 +- .../tests/compact_report_status.rs | 26 + .../layerfs-store/tests/postgresql_status.rs | 405 ++++ .../tests/project_delete_store.rs | 2 +- .../layerfs-store/tests/project_frames.rs | 31 +- .../layerfs-store/tests/restore_install.rs | 1 - .../layerfs-history/src/contract/catalog.rs | 20 + .../src/encoding/delta/select.rs | 179 +- .../crates/layerfs-storage/src/save/pooled.rs | 13 +- .../issues/project-41/gc-acceptance-matrix.md | 219 +- .../issues/project-41/gc-capacity-model.md | 122 ++ .../project-41/gc-implementation-report.md | 485 ++++ core/docs/issues/project-41/gc-runbook.md | 69 +- .../issues/project-41/gc-t10-report-status.md | 90 +- .../gc-t12.3a-sqlite-matrix-report.md | 203 ++ .../gc-t12.3b-postgresql-acceptance.md | 150 ++ .../issues/project-41/gc-t8-sqlite-rebuild.md | 23 +- .../project-41/gc-t9-postgresql-rebuild.md | 6 + docs/architecture/api/README.md | 4 +- docs/architecture/api/calls/README.md | 4 +- docs/architecture/api/calls/coverage.md | 39 +- docs/architecture/api/front-ends/mcp.md | 17 +- docs/architecture/api/sdk/README.md | 16 +- docs/architecture/api/sdk/branch.md | 28 +- docs/architecture/api/sdk/project.md | 78 +- docs/architecture/api/sdk/store.md | 46 +- docs/architecture/engine/README.md | 2 +- docs/architecture/engine/store/README.md | 8 +- docs/architecture/engine/store/postgresql.md | 7 +- docs/architecture/engine/store/testing.md | 295 +++ docs/architecture/flows/call.md | 2 +- docs/architecture/glossary.md | 2 +- engine/crates/layerfs-bridge/src/control.rs | 2 +- .../layerfs-bridge/src/control_request.rs | 67 +- .../layerfs-bridge/src/control_types.rs | 53 + .../layerfs-bridge/tests/control_records.rs | 13 + .../src/application/diagnostics.rs | 2 + .../src/application/retained.rs | 2 + .../layerfs-daemon/src/control/operations.rs | 38 +- .../layerfs-daemon/tests/mount_holders.rs | 127 ++ .../queries/history/gc_intent_all_branch.sql | 2 +- .../queries/history/gc_intent_all_project.sql | 2 +- .../queries/history/gc_intent_ops_branch.sql | 2 +- .../queries/history/gc_intent_ops_project.sql | 2 +- .../src/backend/postgresql/access.rs | 89 +- .../src/backend/postgresql/compact.rs | 155 +- .../src/backend/postgresql/status.rs | 184 ++ .../src/backend/sqlite/compact.rs | 359 ++- .../src/compact_metrics.rs | 581 +++++ .../src/history/admission.rs | 17 +- .../src/history/catalog.rs | 13 + .../layerfs-persistence/src/history/mod.rs | 4 +- engine/crates/layerfs-persistence/src/lib.rs | 7 +- .../crates/layerfs-persistence/src/report.rs | 5 + .../tests/compact_metrics.rs | 371 ++++ .../tests/concurrent_first_installs.rs | 7 + .../tests/diagnostic_history.rs | 6 +- .../sqlite_compact_capacity_measure.json | 89 + .../sqlite_compact_capacity_measure_50gb.json | 74 + .../tests/fixtures/sqlite_compact_golden.json | 2 +- .../fixtures/sqlite_compact_measure.json | 16 + .../layerfs-persistence/tests/gc_backfill.rs | 2 +- .../tests/pg_doctor/conninfo.rs | 242 ++ .../tests/pg_doctor/mod.rs | 15 +- .../tests/postgresql_commit_round_trips.rs | 5 +- .../tests/postgresql_compact.rs | 1098 +++++++++- .../tests/postgresql_compact_fixtures.rs | 1790 +++++++++++++++ .../tests/postgresql_diagnostic.rs | 45 +- .../tests/postgresql_doctor_access.rs | 9 +- .../tests/postgresql_doctor_physical.rs | 46 +- .../tests/postgresql_gc_fence.rs | 532 +++++ .../tests/postgresql_gc_mark.rs | 387 ++++ .../tests/postgresql_narrow_locations.rs | 21 +- .../tests/postgresql_status.rs | 272 +++ .../tests/sqlite_compact.rs | 2 +- .../tests/sqlite_compact_capacity.rs | 902 ++++++++ .../tests/sqlite_compact_crash.rs | 450 +++- .../tests/sqlite_compact_fixtures.rs | 1 - .../tests/sqlite_compact_matrix.rs | 1952 +++++++++++++++++ .../tests/support/metrics.rs | 91 + .../layerfs-persistence/tests/support/mod.rs | 5 + .../layerfs-persistence/tests/support/pg.rs | 281 +++ .../crates/layerfs-project/src/store/init.rs | 11 +- .../crates/layerfs-project/src/store/live.rs | 20 +- .../layerfs-project/tests/init_postgresql.rs | 19 +- .../src/history/admission.rs | 11 +- .../src/history/branch.rs | 2 +- .../src/history/lookup.rs | 14 +- .../src/history/saved.rs | 4 +- .../src/methods/file/blame.rs | 6 +- .../src/operator/files.rs | 2 +- .../src/operator/inspect.rs | 14 +- .../src/read/attribution.rs | 14 +- .../layerfs-store-service/src/read/blame.rs | 13 +- .../src/read/visibility.rs | 7 +- .../layerfs-store-service/src/store/bind.rs | 49 +- .../layerfs-store-service/src/write/files.rs | 2 +- .../layerfs-store-service/tests/admission.rs | 3 + .../tests/golden/sdk_methods.txt | 6 +- .../tests/sdk_mount_bind_failure.rs | 95 + .../tests/sdk_path_merge_removal.rs | 290 +++ .../tests/sdk_path_whiteouts.rs | 56 +- tools/ci/test_store_ext4.sh | 384 ++++ 155 files changed, 14985 insertions(+), 835 deletions(-) create mode 100644 .github/workflows/store-ext4.yml delete mode 100644 api/crates/layerfs-api/src/project/remove.rs create mode 100644 api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs delete mode 100644 control/crates/layerfs-store/src/project/remove.rs create mode 100644 control/crates/layerfs-store/tests/postgresql_status.rs create mode 100644 core/docs/issues/project-41/gc-capacity-model.md create mode 100644 core/docs/issues/project-41/gc-implementation-report.md create mode 100644 core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md create mode 100644 core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md create mode 100644 docs/architecture/engine/store/testing.md create mode 100644 engine/crates/layerfs-daemon/tests/mount_holders.rs create mode 100644 engine/crates/layerfs-persistence/src/compact_metrics.rs create mode 100644 engine/crates/layerfs-persistence/tests/compact_metrics.rs create mode 100644 engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json create mode 100644 engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json create mode 100644 engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json create mode 100644 engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_gc_fence.rs create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_gc_mark.rs create mode 100644 engine/crates/layerfs-persistence/tests/postgresql_status.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs create mode 100644 engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs create mode 100644 engine/crates/layerfs-persistence/tests/support/metrics.rs create mode 100644 engine/crates/layerfs-persistence/tests/support/pg.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs create mode 100644 engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs create mode 100755 tools/ci/test_store_ext4.sh diff --git a/.agents/skills/layerfs-js-sdk/references/methods/project.md b/.agents/skills/layerfs-js-sdk/references/methods/project.md index bbd29a11b0..59e6ea6a6c 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/project.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/project.md @@ -13,7 +13,8 @@ project.init(project, folder, { timeout, on_progress }) project.list(store?, { page_size, page_pos }) project.get(project) project.export(project, destination, { branch, on_progress }) -project.remove(project) +project.delete(project) +project.remove_file(project, path) ``` ## `project.init` @@ -119,23 +120,41 @@ The project's record: its Store, the Store's kind and machine. {"project":"shop/api","store":"shop","name":"api","kind":"sqlite","machine":"native","created":1793574061265} ``` -## `project.remove` +## `project.delete` -`project.remove(project)` +`project.delete(project)` -Removes one project from its Store. Its Branches, Commits and `main` go with -it; there is no undo. The live result: +Retires one project in its Store and removes its home entry (spec §4). The +project disappears from `project.list` and `project.get`; its Layers, +Branches, Commits and stored content stay until a later `store.compact`, and +the Store stays addressable. The live result: ```json -{"stack":"3104401f11fb…","packs_kept":true,"project":"shop/api"} +{"kind":"Removed","project":"shop/api","deleted_ms":1793574061265,"home_entry_removed":true} ``` -`packs_kept: true`: the project's records are gone, but its stored content -stays in the Store file. Removing a project from a PostgreSQL Store is -`Unsupported` in this build. While a Sandbox of the project is recorded it is -refused `InUse`; `detail.related` lists each as `{ kind: "sandbox", name }` -(close them first). To delete a whole Store with every project in it, use -`store.remove`. +The Store marker is the commit point. If the home entry cannot be removed the +call still succeeds with `home_entry_removed: false` and keeps a durable +intent, so the same call retried from the same home reuses `deleted_ms` and +finishes the entry. A mount of the project is refused `MountsActive`, another +user or operation `Busy`/`Unresolved`. -An agent working on a project never needs `remove`; it is for whoever owns the -home. +## `project.remove_file` + +`project.remove_file(project, path)` + +Hides one absolute project path from every node at or below its marker +frontier (spec §4.3). It writes a marker only; no bytes are freed. The path is +normalized (one leading `/` stripped) and must be a canonical absolute POSIX +path. The live result: + +```json +{"kind":"Removed","project":"shop/api","path":"docs/old.md","frontier_serial":42,"removed_ms":1793574061265} +``` + +`kind` is `UpToDate` when no project node is above this path's marker +frontier. Later reads and stats of the path are refused `Removed`; a Commit +after the frontier that writes the path again is visible. + +An agent working on a project never needs `delete` or `remove_file`; they are +for whoever owns the home. diff --git a/.github/workflows/store-ext4.yml b/.github/workflows/store-ext4.yml new file mode 100644 index 0000000000..cffa8038de --- /dev/null +++ b/.github/workflows/store-ext4.yml @@ -0,0 +1,203 @@ +# Store lane: the Store suites, on a filesystem that carries SQLite's locks. +# +# A Store opens only on a directory that is private, owned by the test user and +# on a local filesystem whose locks reach every opener: ext4, xfs or btrfs on +# Linux (engine/crates/layerfs-store-service/src/kernel_guard.rs). The Store +# suites keep their Stores in TMPDIR and in $CARGO_TARGET_DIR/tmp, so both must +# be on such a filesystem, and an overlay checkout or a tmpfs /tmp is refused. +# tools/ci/test_store_ext4.sh refuses any other directory before cargo starts +# and points TMPDIR, CARGO_TARGET_DIR and CARGO_HOME at the one it is given; +# this lane only provides that directory. See +# docs/architecture/engine/store/testing.md. +# +# It adds no test selection of its own: every target it runs is already in the +# push or nightly lane chosen by tools/ci_tests.py. It runs them under the +# environment contract their guard requires. +# +# Reports only, like the other lanes (AGENTS.md §4): never a required check. +# The runner's /mnt is a separate ext4 data disk while /tmp is tmpfs, so the +# prepare step asserts the filesystem: a runner image whose /mnt is not ext4 +# fails there, with its own message, instead of as a guard refusal inside a +# suite. +name: store-ext4 + +on: + push: + branches: [main] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + pull_request: + # off for pull requests into layerfs_v2_0_0, like the other lanes + branches-ignore: [layerfs_v2_0_0] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + schedule: + # After nightly.yml's 04:23 UTC run; the two lanes never share runner time. + - cron: "47 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: store-ext4-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + # CI needs no debugger: smaller objects, much shorter links. + CARGO_PROFILE_TEST_DEBUG: "0" + # Link the test binaries with mold instead of GNU ld, as ci.yml's test job + # does. Scoped to this target, so the repository's aarch64 rustflags + # (.cargo/config.toml) stay. + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C link-arg=-fuse-ld=mold" + # The harness requires this and validates it before cargo starts. /mnt is the + # runner's ext4 data disk; never /tmp, which is tmpfs there. + LAYERFS_TEST_TMPDIR: /mnt/layerfs-store-tests + +jobs: + quick: + name: store ext4 (--quick) + # The push lane: one or two targets per suite, across the four suites. + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + # /mnt belongs to root on the runner image and the tests run as the + # runner, so hand the directory over before the harness probes it. + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # No Swatinem/rust-cache here: the harness moves CARGO_HOME and + # CARGO_TARGET_DIR onto LAYERFS_TEST_TMPDIR, so the checkout-scoped cache + # the other lanes restore is never read. The cold build is the price of + # running the suites where their guard admits them. + - name: Run the Store suites (--quick) + run: tools/ci/test_store_ext4.sh --quick + + full-postgresql: + name: store ext4 (--full, PostgreSQL 17) + # The scheduled and manual lane: every Store suite, with the PostgreSQL + # cases on. A manual dispatch runs this job only, which is the full sweep. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + # A libpq connection string, not a URL (testing.md). A service + # container's published port is 127.0.0.1 on the runner. The server is + # scratch -- each PostgreSQL test creates and drops its own schema -- and + # lives only for this job. Setting this adds `--features postgresql` to + # every suite in the harness. + LAYERFS_TEST_POSTGRES_URL: host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=disable + # The full-mode doctor calls amcheck's verify_heapam and bt_index_check; + # the step below creates it in this database. The physical doctor's two + # extra servers (an amcheck-free database and a physical standby) are + # not available in this job; see testing.md. Set so the diagnostic test + # asserts relation coverage is Pass instead of only printing it. + LAYERFS_TEST_EXPECT_AMCHECK: "1" + services: + postgres: + # PostgreSQL 17 is the supported version for these cases: the schema + # needs 16 or newer for STORAGE PLAIN, and the suites are qualified on + # 17.x (docs/architecture/engine/store/testing.md). + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + # Trust authentication: a throwaway server, reachable from this job's + # steps only, and the tests create and drop their own schemas. + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=postgres --dbname=postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # The service health check gates this job on the server answering. The + # engine never installs extensions: a full-mode audit calls amcheck's + # verify_heapam and bt_index_check and reports every relation as + # Unknown when the extension is absent, while the compaction suite + # asserts a clean full audit. A fresh cluster therefore needs these + # contrib extensions before any suite connects (T-G14). The server is + # scratch, and this is the one database every suite reaches. + - name: Create the PostgreSQL extensions + run: | + set -euo pipefail + # The runner image does not ship psql; the distro client reaches the + # PostgreSQL 17 service container. + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends postgresql-client + psql --version + psql "$LAYERFS_TEST_POSTGRES_URL" -X -v ON_ERROR_STOP=1 <<'SQL' + CREATE EXTENSION IF NOT EXISTS amcheck; + CREATE EXTENSION IF NOT EXISTS pgstattuple; + SQL + # Fail here, with a readable list, rather than as five Unknown + # relation checks inside a compaction suite. + psql "$LAYERFS_TEST_POSTGRES_URL" -X -A -t -v ON_ERROR_STOP=1 \ + -c "SELECT extname FROM pg_catalog.pg_extension WHERE extname IN ('amcheck','pgstattuple') ORDER BY extname" + # The suites own their schemas; the harness reports the mode it took. + - name: Run the Store suites (--full) + run: tools/ci/test_store_ext4.sh --full diff --git a/Cargo.lock b/Cargo.lock index 3ba6ad4ac3..94c8fbe9fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -999,6 +999,7 @@ dependencies = [ "layerfs-telemetry", "libc", "nix", + "postgres", ] [[package]] diff --git a/api/README.md b/api/README.md index c7329db066..ae3d5d2bc8 100644 --- a/api/README.md +++ b/api/README.md @@ -10,7 +10,7 @@ CLI package `@layerfs/cli` (in [`cli/`](cli)), the Python package (in the private stdio MCP server [`mcp/`](mcp) (`@layerfs/mcp`: one generated tool per call, `work` and `admin` profiles). The design is in [docs/architecture/api](../docs/architecture/api/README.md). The package is the -library's call table as ten families of functions, 73 calls in all: +library's call table as ten families of functions, 76 calls in all: | Family | Calls | | --- | --- | diff --git a/api/crates/layerfs-api/src/calls/mod.rs b/api/crates/layerfs-api/src/calls/mod.rs index ef74ed4009..e90eae9b74 100644 --- a/api/crates/layerfs-api/src/calls/mod.rs +++ b/api/crates/layerfs-api/src/calls/mod.rs @@ -2,7 +2,7 @@ //! table's shape, argument checks, folder scope, result keys and the one //! dispatcher with its call log line (`render`, #659); moved back from //! `layerfs-control` by layerfs-project#107. -//! The 70 entries in ten families live beside it, one folder per family and +//! The 76 entries in ten families live beside it, one folder per family and //! one file per call; each function here that needs them takes that list. mod check; mod context; diff --git a/api/crates/layerfs-api/src/lib.rs b/api/crates/layerfs-api/src/lib.rs index 0f67fadce2..83c1d4676b 100644 --- a/api/crates/layerfs-api/src/lib.rs +++ b/api/crates/layerfs-api/src/lib.rs @@ -71,7 +71,7 @@ pub use layerfs_sandbox::{ BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi, }; -/// The ten families, in the order help lists them: 77 calls, the same +/// The ten families, in the order help lists them: 76 calls, the same /// from the `layerfs` binary, Node and Python. pub fn families() -> Families { &[ diff --git a/api/crates/layerfs-api/src/project/mod.rs b/api/crates/layerfs-api/src/project/mod.rs index 7a65fd924f..caee164cb7 100644 --- a/api/crates/layerfs-api/src/project/mod.rs +++ b/api/crates/layerfs-api/src/project/mod.rs @@ -1,10 +1,9 @@ -//! The `project` family: 8 calls, one file each. +//! The `project` family: 7 calls, one file each. mod delete; mod export; mod get; mod init; mod list; -mod remove; mod remove_file; mod stats; @@ -20,6 +19,5 @@ pub(crate) fn calls() -> Calls { export::CALL, delete::CALL, remove_file::CALL, - remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/project/remove.rs b/api/crates/layerfs-api/src/project/remove.rs deleted file mode 100644 index c464f1a441..0000000000 --- a/api/crates/layerfs-api/src/project/remove.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! `project.remove`: remove one project from its Store. -use crate::calls::table::Kind::Project; -use crate::calls::table::{a, call, Call}; -use crate::calls::Context; -use layerfs_bridge::control::SdkValue; -use layerfs_home::names::ProjectRef; -use layerfs_home::outcome::Outcome; - -/// Its table entry. -pub(crate) const CALL: Call = call( - "project", - "remove", - &[a("project", Project)], - &[], - "remove one project from its Store", - run, -); - -/// Runs `project.remove`. -fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { - let project = ProjectRef::parse(args.get_text("project").unwrap_or(""))?; - let (home, events) = context.parts(); - layerfs_control::project::remove(home, events, &project) -} diff --git a/api/crates/layerfs-api/tests/calls_scope.rs b/api/crates/layerfs-api/tests/calls_scope.rs index 5b324435e0..28d19e6e6d 100644 --- a/api/crates/layerfs-api/tests/calls_scope.rs +++ b/api/crates/layerfs-api/tests/calls_scope.rs @@ -114,7 +114,6 @@ fn a_scope_refuses_every_call_it_does_not_cover() { let uncovered: &[(&str, &[(&str, SdkValue)])] = &[ ("store.remove", &[("name", t("shop"))]), ("store.list", &[]), - ("project.remove", &[("project", t("shop/api"))]), ( "branch.remove", &[("project", t("shop/api")), ("branch", t("main"))], diff --git a/api/crates/layerfs-api/tests/calls_table.rs b/api/crates/layerfs-api/tests/calls_table.rs index 2ba77897cd..0f329039b2 100644 --- a/api/crates/layerfs-api/tests/calls_table.rs +++ b/api/crates/layerfs-api/tests/calls_table.rs @@ -1,4 +1,4 @@ -//! The call table and its dispatcher (#533 step 2): 77 calls in ten +//! The call table and its dispatcher (#533 step 2): 76 calls in ten //! families, arguments checked before any effect, one test per family that //! a valid call reaches its family function and a wrong one is refused. use layerfs_api::{families, json, keys, size, Call, Kind}; @@ -51,9 +51,9 @@ fn code(name: &str, args: SdkValue) -> String { } #[test] -fn the_table_has_ten_families_and_77_calls() { +fn the_table_has_ten_families_and_76_calls() { let all: Vec<&Call> = layerfs_api::calls(); - assert_eq!(all.len(), 77); + assert_eq!(all.len(), 76); // each family holds only its own calls, and has at least one for family in families() { assert!(!family.calls().is_empty(), "{}", family.name); @@ -66,7 +66,7 @@ fn the_table_has_ten_families_and_77_calls() { let mut names: Vec<_> = all.iter().map(|c| c.name()).collect(); names.sort(); names.dedup(); - assert_eq!(names.len(), 77, "call names are unique"); + assert_eq!(names.len(), 76, "call names are unique"); // an optional positional is only ever followed by optional ones (D7) for c in &all { let first_optional = c.args.iter().position(|a| a.optional); @@ -75,7 +75,7 @@ fn the_table_has_ten_families_and_77_calls() { } } let counts: Vec = families().iter().map(|f| f.calls().len()).collect(); - assert_eq!(counts, [8, 9, 3, 15, 4, 6, 12, 13, 5, 2]); + assert_eq!(counts, [7, 9, 3, 15, 4, 6, 12, 13, 5, 2]); } #[test] @@ -98,6 +98,17 @@ fn unknown_calls_and_arguments_are_refused_before_any_effect() { .starts_with("InvalidInput")); } +#[test] +fn project_remove_is_retired() { + // spec 4: the hard delete leaves the table; a caller gets `InvalidInput` + // naming it, never a silent hard delete + assert!(layerfs_api::find("project.remove").is_none()); + assert_eq!( + code("project.remove", map(&[("project", t("shop/api"))])), + "InvalidInput: no call named `project.remove`" + ); +} + #[test] fn project_family() { assert!(code("project.get", map(&[("project", t("api"))])) diff --git a/api/crates/layerfs-api/tests/gc_calls.rs b/api/crates/layerfs-api/tests/gc_calls.rs index c7d37f5d24..ad390c8eab 100644 --- a/api/crates/layerfs-api/tests/gc_calls.rs +++ b/api/crates/layerfs-api/tests/gc_calls.rs @@ -246,6 +246,33 @@ fn project_delete_reports_removal_and_retires_the_entry() { let _ = std::fs::remove_dir_all(folder); } +/// A project name a `project.delete` retired is still held: a second +/// `project.init` is `NameTaken` with the `deleted` holder, refused before +/// the folder is scanned. +#[test] +fn project_init_after_delete_is_nametaken_deleted() { + let (home, folder) = project(); + ok(&home, "project.delete", &[p()]); + let refusal = layerfs_api::call( + Some(&home), + "project.init", + map(&[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ]), + &mut |_| {}, + ) + .unwrap_err(); + assert_eq!(refusal.code, "NameTaken", "{}", refusal.message); + let holder = refusal.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("deleted")); + assert_eq!(refusal.detail.get_text("source"), Some("store")); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + #[test] fn store_compact_refuses_an_unknown_store_and_reports_a_run() { let home = tempdir("compact-none"); diff --git a/api/crates/layerfs-api/tests/project_export.rs b/api/crates/layerfs-api/tests/project_export.rs index 0db2299303..07a79b982c 100644 --- a/api/crates/layerfs-api/tests/project_export.rs +++ b/api/crates/layerfs-api/tests/project_export.rs @@ -1283,7 +1283,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { // 2. During an in-flight `project.export`: // - same-Store `file.read` and a second concurrent `project.export` succeed; // - exclusive Store operations (`store.remove`, `store.push`) are refused `InUse`; - // - `project.remove` drops the catalog stack while packs stay (`packs_kept: true`), + // - `project.delete` retires the Stack (its marker keeps packs/objects), // leaving the in-flight pinned export intact while refusing later exports. let first_dest = area.join("first"); let second_dest = area.join("second"); @@ -1336,10 +1336,14 @@ fn shared_store_lease_and_concurrent_remove_admission() { .unwrap_err(); assert_eq!(push_code, "InUse", "{push_msg}"); - // `project.remove` drops the LayerStack while keeping packs/objects; - // the already-pinned export continues without corruption. - let removed = ok(&home, "project.remove", &[("project", t("shop/app"))]); - assert_eq!(removed.get("packs_kept"), Some(&SdkValue::Bool(true))); + // `project.delete` retires the LayerStack and removes the home + // entry; the already-pinned export continues without corruption. + let removed = ok(&home, "project.delete", &[("project", t("shop/app"))]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!( + removed.get("home_entry_removed"), + Some(&SdkValue::Bool(true)) + ); } }, ) @@ -1350,7 +1354,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { assert_same_tree(&source, &second_dest); let _ = std::fs::remove_file(&other_lease); - // After `project.remove`, a new export is refused `NoSuchProject` + // After `project.delete`, a new export is refused `NoSuchProject` let after_rm_dest = area.join("after-rm"); let (code, _) = call( &home, @@ -1364,12 +1368,13 @@ fn shared_store_lease_and_concurrent_remove_admission() { assert_eq!(code, "NoSuchProject"); assert!(!after_rm_dest.exists()); - // Re-init `shop/app` and verify Store-level `Busy` and `Unresolved` admission + // Re-init a new project (`shop/app` stays a retired name) and verify + // Store-level `Busy` and `Unresolved` admission. ok( &home, "project.init", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("folder", t(&source.display().to_string())), ], ); @@ -1394,7 +1399,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { &home, "project.export", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("destination", t(&busy_dest.display().to_string())), ], ) @@ -1419,7 +1424,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { &home, "project.export", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("destination", t(&busy_dest.display().to_string())), ], ) diff --git a/api/crates/layerfs-api/tests/project_family.rs b/api/crates/layerfs-api/tests/project_family.rs index fc4ba41bc0..3039230e5a 100644 --- a/api/crates/layerfs-api/tests/project_family.rs +++ b/api/crates/layerfs-api/tests/project_family.rs @@ -3,7 +3,7 @@ //! it, the Store pushed to a worker machine and read there, then the project //! removed. use layerfs_api::home::{self, paths, write_record}; -use layerfs_bridge::control::SdkValue; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::net::TcpListener; use std::path::{Path, PathBuf}; @@ -335,7 +335,15 @@ fn init_files_branches_move_and_remove() { ); assert_eq!(content(&read), "newer\n", "read through the worker"); - ok(&home, "project.remove", &[p()]); + // `project.delete` retires the project in the Store under `box` and + // removes both entries; its history stays until a compaction. + let deleted = ok(&home, "project.delete", &[p()]); + assert_eq!(deleted.get_text("kind"), Some("Removed")); + assert_eq!(deleted.get_text("project"), Some("shop/api")); + assert_eq!( + deleted.get("home_entry_removed"), + Some(&SdkValue::Bool(true)) + ); assert_eq!( call(&home, "project.get", &[p()]).unwrap_err().0, "NoSuchProject" @@ -389,3 +397,174 @@ fn a_private_home_may_be_passable_and_nothing_more() { std::fs::remove_dir_all(home).unwrap(); let _ = std::fs::remove_dir_all(folder); } + +fn refused(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkRefusal { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).expect_err("the call is refused") +} + +fn private_home(tag: &str) -> PathBuf { + let home = tempdir(tag); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + home +} + +/// A `project.init` name collision is the typed `NameTaken`: the home record +/// first, and the Store's own holder when the home has no record for it. +#[test] +fn project_init_name_collision_reports_holder_and_source() { + let home = private_home("name-taken"); + let folder = tempdir("name-taken-folder"); + std::fs::write(folder.join("README"), b"held\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + + let home_taken = refused( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + assert_eq!(home_taken.code, "NameTaken", "{}", home_taken.message); + let holder = home_taken.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project")); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(home_taken.detail.get_text("source"), Some("home")); + + // A home that no longer records the project meets the Store's own holder. + std::fs::remove_dir_all(paths::project_dir(&home, "shop", "api")).unwrap(); + let store_taken = refused( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + assert_eq!(store_taken.code, "NameTaken", "{}", store_taken.message); + let holder = store_taken.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project")); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(store_taken.detail.get_text("source"), Some("store")); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +/// A Layer keeps the name of the Branch it was published from after that +/// Branch is retired: provenance rendering is not a caller-facing lookup. +#[test] +fn a_retired_source_branch_still_renders_on_its_layers() { + let home = private_home("retired-source"); + let folder = tempdir("retired-source-folder"); + std::fs::write(folder.join("base.txt"), b"base\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + ok( + &home, + "branch.fork", + &[ + ("project", t("shop/api")), + ("name", t("work")), + ("from_branch", t("main")), + ], + ); + ok( + &home, + "file.write", + &[ + ("project", t("shop/api")), + ("branch", t("work")), + ("path", t("/new.txt")), + ("content", t("new\n")), + ("message", t("new")), + ], + ); + ok( + &home, + "branch.publish", + &[("project", t("shop/api")), ("branch", t("work"))], + ); + + let main = ok( + &home, + "commit.list", + &[("project", t("shop/api")), ("branch", t("main"))], + ); + let SdkValue::List(layers) = main.get("items").expect("items") else { + panic!("{main:?}") + }; + let id = layers[0].get_text("id").expect("layer id").to_owned(); + assert_eq!(layers[0].get_text("source_branch"), Some("work"), "{main:?}"); + + ok( + &home, + "branch.remove", + &[("project", t("shop/api")), ("branch", t("work"))], + ); + + // `layer.list` still renders the provenance ... + let main = ok( + &home, + "commit.list", + &[("project", t("shop/api")), ("branch", t("main"))], + ); + let SdkValue::List(layers) = main.get("items").expect("items") else { + panic!("{main:?}") + }; + assert_eq!( + layers[0].get_text("source_branch"), + Some("work"), + "layer.list: {main:?}" + ); + // ... and so does `layer.get` ... + let got = ok( + &home, + "commit.get", + &[ + ("project", t("shop/api")), + ("branch", t("main")), + ("commit", t(&id)), + ], + ); + assert_eq!( + got.get_text("source_branch"), + Some("work"), + "layer.get: {got:?}" + ); + // ... while a caller-facing lookup of the retired Branch stays refused. + let (code, message) = call( + &home, + "branch.get", + &[("project", t("shop/api")), ("branch", t("work"))], + ) + .unwrap_err(); + assert_eq!(code, "NoSuchBranch", "{message}"); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} diff --git a/api/crates/layerfs-api/tests/project_init_worker.rs b/api/crates/layerfs-api/tests/project_init_worker.rs index d0f3900c4b..07b805b32d 100644 --- a/api/crates/layerfs-api/tests/project_init_worker.rs +++ b/api/crates/layerfs-api/tests/project_init_worker.rs @@ -231,14 +231,19 @@ fn init_into_a_worker_s_store_copies_the_caller_s_folder() { let stat = store_call(&home, "file.stat", "shop/web", "/link"); assert_eq!(stat.get_text("type"), Some("symlink"), "{stat:?}"); - let again = call( - &home, + let again = layerfs_api::call( + Some(&home), "project.init", init_args("shop/web", &folder), &mut |_| {}, ) .unwrap_err(); - assert_eq!(again.0, "Exists", "{}", again.1); + assert_eq!(again.code, "NameTaken", "{}", again.message); + let holder = again.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project"), "{again:?}"); + assert_eq!(holder.get_text("name"), Some("web")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(again.detail.get_text("source"), Some("home")); open_up(&folder); let (missing, plain) = (missing.parent().unwrap(), plain.parent().unwrap()); diff --git a/api/crates/layerfs-api/tests/store_projects.rs b/api/crates/layerfs-api/tests/store_projects.rs index 8961aaf5f8..9f3875af36 100644 --- a/api/crates/layerfs-api/tests/store_projects.rs +++ b/api/crates/layerfs-api/tests/store_projects.rs @@ -7,7 +7,10 @@ use layerfs_content::{ file::FileView, filesystem::{FilesystemRead, FilesystemRootId, LogicalPath}, }; -use layerfs_history::{BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId}; +use layerfs_history::{ + error::{HolderKind, HolderState, NameSource}, + BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId, +}; use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; @@ -124,7 +127,13 @@ fn an_empty_store_holds_two_projects_and_refuses_a_taken_name() { assert_ne!(shop.initialized.root, api.initialized.root); assert_eq!(shop.branch.effective_root, shop.initialized.root); let taken = add(&path, source(&directory, "again", b"x"), "shop", 30); - assert!(matches!(taken, Err(InitError::Exists)), "{taken:?}"); + let Err(InitError::NameTaken(state)) = taken else { + panic!("a held project name is NameTaken: {taken:?}"); + }; + assert_eq!(state.holder.kind, HolderKind::Project); + assert_eq!(state.holder.name, "shop"); + assert_eq!(state.holder.state, HolderState::Live); + assert_eq!(state.source, NameSource::Store); let handles = SqliteHandles::open_writable(store(&path), BINDING, CURSOR).unwrap(); let names: Vec<_> = projects(&handles.history) diff --git a/api/crates/layerfs-api/tests/store_stats.rs b/api/crates/layerfs-api/tests/store_stats.rs index dcb2fceded..54cfc0f3a5 100644 --- a/api/crates/layerfs-api/tests/store_stats.rs +++ b/api/crates/layerfs-api/tests/store_stats.rs @@ -26,28 +26,31 @@ fn call(home: &Path, method: &str, fields: &[(&str, SdkValue)]) -> SdkValue { layerfs_api::call(Some(home), method, args, &mut |_| {}) .unwrap_or_else(|r| panic!("{method}: {r:?}")) } -fn check(home: &Path, kind: &str) { +fn check(home: &Path, kind: &str, test: &str) { let stats = || call(home, "store.stats", &[("name", t("stats"))]); let empty = stats(); assert_eq!(empty.get_text("kind"), Some(kind)); - assert_eq!(empty.get_int("history_logical"), Some(0)); + // a previous project's bytes stay after its soft delete, so later runs + // measure from the Store's current history + let base = empty.get_int("history_logical").unwrap(); assert_eq!( empty.get("free") == Some(&SdkValue::Null), kind == "postgresql" ); assert!(empty.get("dedupe").is_none()); - let source = folder(&format!("input-{kind}")); + let name = format!("stats/{test}"); + let source = folder(&format!("input-{kind}-{test}")); std::fs::write(source.join("file"), "hello").unwrap(); call( home, "project.init", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("folder", t(source.to_str().unwrap())), ], ); let initial = stats(); - assert_eq!(initial.get_int("history_logical"), Some(5)); + assert_eq!(initial.get_int("history_logical"), Some(base + 5)); assert!(initial.get_int("canonical").unwrap() > 0); assert!(initial.get_int("stored").unwrap() > 0); assert!(initial.get_int("file").unwrap() > 0); @@ -55,7 +58,7 @@ fn check(home: &Path, kind: &str) { home, "branch.fork", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("name", t("dev")), ("from_branch", t("main")), ], @@ -64,7 +67,7 @@ fn check(home: &Path, kind: &str) { home, "file.write", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("branch", t("dev")), ("path", t("/file")), ("content", t("abcdef")), @@ -73,30 +76,31 @@ fn check(home: &Path, kind: &str) { call( home, "branch.publish", - &[("project", t("stats/p")), ("branch", t("dev"))], + &[("project", t(&name)), ("branch", t("dev"))], ); let published = stats(); assert_eq!( published.get_int("history_logical"), - Some(17), + Some(base + 17), "Commit and Layer count their snapshots" ); - if kind == "postgresql" { - let refused = layerfs_api::call( - Some(home), - "project.remove", - SdkValue::map([("project", t("stats/p"))]), - &mut |_| {}, - ) - .unwrap_err(); - assert_eq!(refused.code, "Unsupported"); - } else { - call(home, "project.remove", &[("project", t("stats/p"))]); - let removed = stats(); - assert_eq!(removed.get_int("history_logical"), Some(0)); - assert_eq!(removed.get("canonical"), published.get("canonical")); - assert_eq!(removed.get("stored"), published.get("stored")); - } + // `project.delete` retires the Stack and removes the home entry; the + // marker is the commit point, so the history bytes stay until a + // `store.compact` collects them and the counters do not drop. + call(home, "project.delete", &[("project", t(&name))]); + let removed = stats(); + assert_eq!(removed.get_int("history_logical"), Some(base + 17)); + assert_eq!(removed.get("canonical"), published.get("canonical")); + assert_eq!(removed.get("stored"), published.get("stored")); + let gone = layerfs_api::call( + Some(home), + "project.get", + SdkValue::map([("project", t(&name))]), + &mut |_| {}, + ) + .unwrap_err(); + assert_eq!(gone.code, "NoSuchProject"); + assert!(!paths::project_record(home, "stats", test).exists()); layerfs_api::close_idle(std::time::Duration::ZERO); std::fs::remove_dir_all(source).unwrap(); } @@ -105,7 +109,7 @@ fn check(home: &Path, kind: &str) { fn native_sqlite_and_worker_statistics_use_the_same_counters() { let home = folder("sqlite-home"); call(&home, "store.create", &[("name", t("stats"))]); - check(&home, "sqlite"); + check(&home, "sqlite", "p"); let root = folder("worker-root"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap().to_string(); @@ -126,7 +130,7 @@ fn native_sqlite_and_worker_statistics_use_the_same_counters() { "store.push", &[("name", t("stats")), ("machine", t("box"))], ); - check(&home, "sqlite"); + check(&home, "sqlite", "w"); // A stopped worker refuses rather than starting anything. layerfs_api::close_idle(std::time::Duration::ZERO); std::fs::remove_file(paths::worker_record(&home, "box")).unwrap(); @@ -159,6 +163,6 @@ fn postgresql_statistics_have_null_free_and_count_publications() { ("synchronous_commit", t("off")), ], ); - check(&home, "postgresql"); + check(&home, "postgresql", "p"); std::fs::remove_dir_all(home).unwrap(); } diff --git a/api/crates/layerfs-cli/tests/gc.rs b/api/crates/layerfs-cli/tests/gc.rs index 36d04aeb68..0b2096f0d6 100644 --- a/api/crates/layerfs-cli/tests/gc.rs +++ b/api/crates/layerfs-cli/tests/gc.rs @@ -36,6 +36,17 @@ const CALLS: &[(&[&str], &str, &str)] = &[ ), ]; +#[test] +fn project_remove_is_a_usage_error() { + let home = tempdir("gc-remove"); + let (code, out, err) = run_code(&home, &["project", "remove", "shop/api"]); + assert_eq!(code, Some(2), "{out} {err}"); + assert!(err.contains("no call `project remove`"), "{err}"); + assert!(out.is_empty(), "{out}"); + assert!(!home.join("store").exists(), "no call reached a home"); + std::fs::remove_dir_all(home).unwrap(); +} + #[test] fn help_lists_each_gc_call_with_its_usage() { let home = tempdir("gc-help"); diff --git a/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs new file mode 100644 index 0000000000..9e53c90749 --- /dev/null +++ b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs @@ -0,0 +1,426 @@ +//! AC-01 (§12 crash injection, §9): `kill -9` the **real process** that +//! carries `store.compact`, not a re-execution of this test binary. +//! +//! The in-process matrix (`engine/crates/layerfs-persistence/tests/ +//! sqlite_compact_crash.rs`) stops at a failpoint inside the test process, and +//! its `real_process` arm re-executes the *test binary*, so the killed process +//! is still a test child. This test starts the built `layerfs` command with +//! `store compact`, waits through the SQLite engine's Unix-socket gate until +//! that process stands inside the run, and `SIGKILL`s it there. Per #352 +//! F-020 no daemon serves Store calls: the caller's process performs the +//! compaction, so this is the process the §9 crash describes. (It is still not +//! a long-lived `layerfs worker` serving a machine's Store.) +//! +//! Every arm reopens the Store through the CLI and asserts the §9 crash state: +//! wholly old at `G` or wholly new at `G + 1`, the exact artifact set, the +//! sidecars a killed session leaves, the `store.status` projection (phase, +//! `uncertain`, `unresolved`), the next `store.compact` settling the leftover, +//! the retained layer still readable, and a clean `doctor`. The `G` baseline +//! is the §3 `none` run before the dead pack exists, so the crash arms start +//! from a Store with exactly one wholly dead pack. +//! +//! Linux only: the arm asserts `SIGKILL` delivery through +//! `ExitStatusExt::signal`. A macOS build has no gate in the engine, so the +//! hook is inert there by construction. +#![cfg(target_os = "linux")] +mod common; + +use common::{layerfs, run, run_with_stdin, tempdir, text, LAYERFS}; +use layerfs_api::json; +use layerfs_bridge::control::SdkValue; +use std::io::Read; +use std::os::unix::net::UnixListener; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +/// The engine's gate: a run that reaches the named failpoint connects to this +/// socket and blocks there. Both variables are read by the production +/// `store.compact` path only when the gate is present. +const GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +const FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// A short socket path: the sandbox's own root is longer than `SUN_LEN`. +fn gate_socket() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + PathBuf::from(format!( + "/tmp/layerfs-g65-gate-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) +} + +/// One crash point: the failpoint, the phase `store.status` must project, +/// whether the rebuild is already installed at `G + 1`, and which staged +/// artifacts the kill leaves beside the Store. +#[derive(Clone, Copy, Debug)] +struct Arm { + failpoint: &'static str, + phase: &'static str, + installed: bool, + target: bool, + visited: bool, + /// A kill before the seal ends the process with the writer session's WAL + /// on disk (`-wal`, `-shm`); every later point has closed it. + live_sidecars: bool, +} + +/// `layerfs ` on `home`, parsing the one-line JSON result. +fn call(home: &Path, words: &[&str]) -> SdkValue { + let out = run(&mut layerfs(home, words)); + assert_eq!( + out.status.code(), + Some(0), + "{words:?}: {}", + text(&out.stderr) + ); + json::parse(text(&out.stdout).trim()).unwrap_or_else(|error| { + panic!("{words:?}: {error:?}: {}", text(&out.stdout)); + }) +} + +/// `store.compact s`, the call whose process the crash arms kill. +fn compact(home: &Path) -> SdkValue { + call(home, &["store", "compact", "s"]) +} + +fn generations(report: &SdkValue) -> (u64, u64) { + let generation = report.get("generation").expect("the report's generation"); + let before = generation.get_int("before").expect("generation.before"); + let after = generation.get_int("after").expect("generation.after"); + (before as u64, after as u64) +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +/// Kills `store.compact` for real: waits until the process is stopped inside +/// the run at `failpoint`, then `SIGKILL`s it. The gate keeps the wait from +/// being a sleep, and a process that ends early (a refusal) fails the arm with +/// its own stderr. +fn crash_and_kill(home: &Path, failpoint: &'static str) { + let gate = gate_socket(); + let _ = std::fs::remove_file(&gate); + let listener = UnixListener::bind(&gate).expect("the crash gate socket"); + listener + .set_nonblocking(true) + .expect("a nonblocking crash gate"); + let mut child = Command::new(LAYERFS) + .args(["store", "compact", "s"]) + .env("LAYERFS_HOME", home) + .env(GATE_ENV, &gate) + .env(FAILPOINT_ENV, failpoint) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("the real store.compact process"); + + let deadline = Instant::now() + Duration::from_secs(60); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the child's status") { + let mut stderr = String::new(); + if let Some(mut pipe) = child.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!("store.compact ended before the gate ({failpoint}): {status}: {stderr}"); + } + assert!( + Instant::now() < deadline, + "store.compact never reached the gate ({failpoint})" + ); + std::thread::yield_now(); + } + Err(error) => panic!("crash gate accept: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(60))) + .expect("a bounded gate read"); + let mut ready = [0u8; 1]; + let mut reader = &stream; + reader + .read_exact(&mut ready) + .expect("the child reached the crash gate"); + assert_eq!(ready[0], b'R', "the child stopped at the crash gate"); + + // The real §9 crash: SIGKILL the process that performs the compaction. + child.kill().expect("SIGKILL the store.compact process"); + let status = child.wait().expect("the killed process exits"); + assert_eq!( + status.signal(), + Some(9), + "the store.compact process was SIGKILLed: {status:?}" + ); + drop(stream); + let _ = std::fs::remove_file(&gate); +} + +/// The fixture: a Store, one project, a Branch whose commit writes a large +/// blob, and then a `§3 none` baseline that names `G` without moving it. +fn fixture(tag: &str) -> (PathBuf, PathBuf, u64) { + let home = tempdir(&format!("g65-home-{tag}")); + let source = tempdir(&format!("g65-src-{tag}")); + std::fs::write(source.join("a.txt"), b"the retained bytes\n").unwrap(); + call(&home, &["store", "create", "s"]); + call(&home, &["project", "init", "s/p", source.to_str().unwrap()]); + call(&home, &["branch", "fork", "s/p", "dev", "main"]); + // A large blob so its pack is wholly dead once `dev` is soft-removed, and + // large enough that the rebuild threshold (`dead >= 10% of total`) holds. + let big: Vec = (0..256 * 1024).map(|at| (at % 251) as u8).collect(); + let written = run_with_stdin( + &mut layerfs(&home, &["file", "write", "s/p", "dev", "/big"]), + &big, + ); + assert!(written.status.success(), "{}", text(&written.stderr)); + + // Nothing is dead yet: §3 `none`, so the generation does not move. This + // names `G`, the generation every crash arm starts from. + let baseline = compact(&home); + assert_eq!(baseline.get_text("path"), Some("none"), "{baseline:?}"); + let (before, after) = generations(&baseline); + assert_eq!(before, after, "a `none` run moves no generation"); + + // One wholly dead pack: the Branch that wrote `/big` is soft-removed and + // its objects become unreachable; the next run must reclaim them. + call(&home, &["branch", "remove", "s/p", "dev"]); + (home, source, before) +} + +fn exercise(arm: Arm) { + let (home, source, generation) = fixture(&arm.failpoint.to_lowercase()); + let store = home.join("store/s/store.sqlite"); + assert!(store.is_file(), "{}", store.display()); + + crash_and_kill(&home, arm.failpoint); + + // §9 crash state, read before any handle opens the Store: the exact + // artifact set beside the live file, and the sidecars the kill leaves. + assert_eq!( + with_suffix(&store, ".compact").exists(), + arm.target, + "staged Store at {}", + arm.failpoint + ); + assert_eq!( + with_suffix(&store, ".compact-visited").exists(), + arm.visited, + "visited scratch at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, ".compact-op").exists(), + "op record at {}", + arm.failpoint + ); + if arm.live_sidecars { + assert!( + with_suffix(&store, "-wal").exists(), + "the killed session's WAL at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, "-shm").exists(), + "the killed session's shared memory at {}", + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + if arm.live_sidecars && suffix != "-journal" { + continue; + } + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} beside the Store at {}", + arm.failpoint + ); + assert!( + !with_suffix(&with_suffix(&store, ".compact"), suffix).exists(), + "sidecar {suffix} beside the staged Store at {}", + arm.failpoint + ); + } + + // `store.status` (§11) projects the recorded run: the phase it stood in, + // an uncertain outcome, and an unresolved state because its owner died. + let observed = call(&home, &["store", "status", "s"]); + let op = observed.get("op").expect("the op projection"); + assert!(op.get("id").is_some(), "{observed:?}"); + assert_eq!(op.get_text("phase"), Some(arm.phase), "{observed:?}"); + assert_eq!( + op.get("uncertain"), + Some(&SdkValue::Bool(true)), + "{observed:?}" + ); + assert_eq!( + observed.get_text("state"), + Some("unresolved"), + "{observed:?}" + ); + + // The next run judges the leftover: a rebuild from `G` before install, a + // settle at `G + 1` after it, and no residue either way. + let recovered = compact(&home); + if arm.installed { + assert_eq!(recovered.get_text("path"), Some("none"), "{recovered:?}"); + assert_eq!( + generations(&recovered), + (generation + 1, generation + 1), + "{recovered:?}" + ); + } else { + assert_eq!(recovered.get_text("path"), Some("rebuild"), "{recovered:?}"); + assert_eq!( + generations(&recovered), + (generation, generation + 1), + "{recovered:?}" + ); + } + let residue = recovered.get("residue").expect("the residue"); + assert_eq!(residue.get_int("bytes"), Some(0), "{recovered:?}"); + assert!( + matches!(residue.get("artifacts"), Some(SdkValue::List(items)) if items.is_empty()), + "{recovered:?}" + ); + + // Nothing is left beside the Store, and the op record is settled. + for path in [ + with_suffix(&store, ".compact"), + with_suffix(&store, ".compact-visited"), + with_suffix(&store, ".compact-op"), + ] { + assert!( + !path.exists(), + "{} left at {}", + path.display(), + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} left at {}", + arm.failpoint + ); + } + let settled = call(&home, &["store", "status", "s"]); + assert!( + settled.get("op").and_then(|op| op.get("id")).is_none(), + "{settled:?}" + ); + assert_ne!(settled.get_text("state"), Some("unresolved"), "{settled:?}"); + + // The retained layer still reads, and the home passes the doctor: the + // dead pack is collected while the live one is intact. + let stats = call(&home, &["project", "stats", "s/p"]); + assert!(stats.get_int("files").unwrap_or(0) >= 1, "{stats:?}"); + call(&home, &["file", "stat", "s/p", "main", "/a.txt"]); + let doctor = call(&home, &["doctor"]); + assert_eq!(doctor.get("ok"), Some(&SdkValue::Bool(true)), "{doctor:?}"); + + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&source).unwrap(); +} + +macro_rules! sigkill_at { + ($name:ident, $failpoint:literal, $phase:literal, $installed:literal, $target:literal, $visited:literal, $live_sidecars:literal) => { + #[test] + fn $name() { + exercise(Arm { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + live_sidecars: $live_sidecars, + }); + } + }; +} + +// The mark phase: the open session and admission intent at `BeforeSeal` (the +// kill leaves the writer's WAL sidecars), the sealed Store at `AfterSeal`. +sigkill_at!( + sigkill_before_seal, + "BeforeSeal", + "mark", + false, + false, + false, + true +); +sigkill_at!( + sigkill_after_seal, + "AfterSeal", + "mark", + false, + false, + false, + false +); +// The build: immediately before `VACUUM INTO` (SQLite runs that copy as one +// uninterruptible call), and after the staged Store is built. +sigkill_at!( + sigkill_before_vacuum, + "BeforeVacuum", + "build", + false, + false, + true, + false +); +sigkill_at!( + sigkill_after_build, + "AfterBuild", + "build", + false, + true, + true, + false +); +// The verification and the install boundary. +sigkill_at!( + sigkill_during_verify, + "DuringVerify", + "verify", + false, + true, + true, + false +); +sigkill_at!( + sigkill_before_install, + "BeforeInstall", + "verify", + false, + true, + true, + false +); +// Installed at `G + 1`: after the rename, and partway through the settle. +sigkill_at!( + sigkill_after_install, + "AfterInstall", + "install", + true, + false, + true, + false +); +sigkill_at!( + sigkill_during_settle, + "DuringSettle", + "install", + true, + false, + false, + false +); diff --git a/api/mcp/src/profiles.js b/api/mcp/src/profiles.js index 495a83bff4..cec2810f13 100644 --- a/api/mcp/src/profiles.js +++ b/api/mcp/src/profiles.js @@ -6,7 +6,8 @@ // Assignment rule (each call lands in exactly one bucket, none excluded): // work the families an agent works in (branch, commit, file, workspace) // plus the read-only project and Sandbox calls named below -// admin every other call: project create/remove, Sandbox create/close, +// admin every other call: project create/delete/remove_file, Branch +// remove, Sandbox create/close, // and the machine, store, backup and logs families // The `admin` launch profile exposes the union, so `admin` adds the // administrative calls to the `work` set. @@ -20,15 +21,15 @@ const WORK_FAMILIES = new Set(["branch", "commit", "file", "workspace"]); /** Read-only calls of families that are otherwise administration. */ const WORK_CALLS = new Set(["project.list", "project.get", "project.stats", "project.export", "sandbox.list", "sandbox.get"]); -/** Administrative calls of the project and sandbox families. */ -const ADMIN_CALLS = new Set(["project.init", "project.remove", "sandbox.create", "sandbox.close"]); +/** Administrative calls outside the admin families. */ +const ADMIN_CALLS = new Set(["project.init", "project.delete", "project.remove_file", "branch.remove", "sandbox.create", "sandbox.close"]); /** Families whose every call belongs to the `admin` profile. */ const ADMIN_FAMILIES = new Set(["machine", "store", "backup", "logs"]); /** Why a family's calls sit where they do; one line per family, in the doc. */ const WHY = { - project: "read a project's record or its entry counts, or export one (work); importing and removing a project are administration", + project: "read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration", branch: "work on Branches: list, read, fork, merge, publish and diff", commit: "read a Branch's Commit history", file: "read and change files in a Branch; each write is one explicit Commit", @@ -48,9 +49,11 @@ const nameOf = (spec) => `${spec.family}.${spec.call}`; */ export function bucketOf(spec) { const name = nameOf(spec); + // The admin exception list is checked before the work families: a retired + // entity in a work family (`branch.remove`) is still administration (§10). + if (ADMIN_CALLS.has(name)) return "admin"; if (WORK_CALLS.has(name)) return "work"; if (WORK_FAMILIES.has(spec.family)) return "work"; - if (ADMIN_CALLS.has(name)) return "admin"; if (ADMIN_FAMILIES.has(spec.family)) return "admin"; return null; } @@ -58,7 +61,10 @@ export function bucketOf(spec) { /** Why this call is in its bucket: a line for the generated documentation table. */ export function whyOf(spec) { if (spec.family === "project" && spec.call === "init") return "import a host folder into a Store; administration"; - if (spec.family === "project" && spec.call === "remove") return "remove a project from its Store; administration"; + if (spec.family === "project" && spec.call === "delete") return "retire a project in its Store and remove its home entry; administration; a folder scope refuses it"; + if (spec.family === "project" && spec.call === "remove_file") return "hide one absolute path from every node at or below its marker frontier; administration; a folder scope refuses it"; + if (spec.family === "branch" && spec.call === "remove") return "retire one Branch in its Store; administration; a folder scope refuses it"; + if (spec.family === "store" && spec.call === "compact") return "reclaim one sqlite Store's dead bytes; administration; a folder scope refuses it"; if (spec.family === "sandbox" && spec.call === "create") return "start a Sandbox process; administration"; if (spec.family === "sandbox" && spec.call === "close") return "stop a Sandbox; administration"; if (spec.family === "project" && spec.call === "export") return "export a Branch snapshot to a local folder under a launch-time --export-root; a folder scope refuses it"; diff --git a/api/mcp/src/tools.js b/api/mcp/src/tools.js index c8fc065b52..8f7635a9b0 100644 --- a/api/mcp/src/tools.js +++ b/api/mcp/src/tools.js @@ -31,6 +31,11 @@ const COMMON = { description: "a whole number; send a decimal string to keep one beyond ±(2^53-1) exact", }, bool: { type: "boolean" }, + image: { + type: "string", + minLength: 1, + description: "a sha256: image digest, a Docker image id, or a Docker reference such as postgres:17-bookworm", + }, size: { oneOf: [{ type: "integer" }, { type: "string", minLength: 1 }], description: "a size in bytes, or text such as \"512M\" or \"8G\"", diff --git a/api/mcp/test/profiles.test.mjs b/api/mcp/test/profiles.test.mjs index 39752837d6..fce4272a92 100644 --- a/api/mcp/test/profiles.test.mjs +++ b/api/mcp/test/profiles.test.mjs @@ -13,20 +13,28 @@ const nameOf = (spec) => `${spec.family}.${spec.call}`; test("every call is in exactly one bucket, and admin adds the work bucket", () => { assert.deepEqual(check(table), []); - assert.equal(table.length, 73); + assert.equal(table.length, 76); const buckets = table.map(bucketOf); assert.equal(buckets.filter((bucket) => bucket === "work").length, 38); - assert.equal(buckets.filter((bucket) => bucket === "admin").length, 35); + assert.equal(buckets.filter((bucket) => bucket === "admin").length, 38); assert.ok(buckets.every((bucket) => bucket === "work" || bucket === "admin")); const work = tools(table, "work").map(nameOf); const admin = tools(table, "admin").map(nameOf); assert.equal(work.length, 38); - assert.equal(admin.length, 73); + assert.equal(admin.length, 76); for (const name of work) assert.ok(admin.includes(name), `${name} is in work but not admin`); assert.throws(() => tools(table, "root"), /unknown profile/); }); +test("the admin exception list is checked before the work families", () => { + // `branch` is a work family, but retiring one is administration (§10) + const retire = { family: "branch", call: "remove" }; + assert.equal(bucketOf(retire), "admin"); + assert.ok(!tools([...table, retire], "work").includes(retire)); + assert.ok(tools([...table, retire], "admin").includes(retire)); +}); + test("every generated tool is a valid MCP tool with the table's arguments and options", () => { const generated = toolsOf(table); assert.equal(generated.length, table.length); diff --git a/api/mcp/test/workflow.test.mjs b/api/mcp/test/workflow.test.mjs index 6c26d40b46..9b5f2f8b28 100644 --- a/api/mcp/test/workflow.test.mjs +++ b/api/mcp/test/workflow.test.mjs @@ -92,6 +92,11 @@ test("a real workflow, refusals, progress, scope and the two profiles", async () server.client.callTool({ name: "not_a_tool", arguments: {} }), (error) => error.code === -32602 && /unknown tool/.test(error.message), ); + // retired in 2.0 (spec 4): the old tool name does not exist + await assert.rejects( + server.client.callTool({ name: "project_remove", arguments: { project: "shop/api" } }), + (error) => error.code === -32602 && /unknown tool/.test(error.message), + ); await assert.rejects( server.client.callTool({ name: "store_create", arguments: { name: 5 } }), (error) => error.code === -32602 && /name.*must be text/.test(error.message), @@ -122,7 +127,7 @@ test("a real workflow, refusals, progress, scope and the two profiles", async () for (const name of names("admin").filter((name) => !names("work").includes(name))) { assert.ok(!workNames.has(name), `${name} is an admin tool in the work profile`); } - for (const prefix of ["store_", "machine_", "backup_", "project_init", "project_remove", "sandbox_create", "sandbox_close"]) { + for (const prefix of ["store_", "machine_", "backup_", "project_init", "project_delete", "project_remove_file", "branch_remove", "sandbox_create", "sandbox_close"]) { assert.ok(![...workNames].some((name) => name.startsWith(prefix)), `${prefix} in the work profile`); } const scopedRead = await call(work.client, "file_read", { project: "shop/api", branch: "feat", path: "/sub/b.txt" }); diff --git a/api/node/README.md b/api/node/README.md index 44af7efcda..6a9d49edc9 100644 --- a/api/node/README.md +++ b/api/node/README.md @@ -2,7 +2,7 @@ The LayerFS SDK for Node: ten families — `project`, `branch`, `commit`, `file`, `sandbox`, `workspace`, `machine`, `store`, `backup`, `logs` — over one call -table of 73 calls into one native library. Each function makes one call; it +table of 76 calls into one native library. Each function makes one call; it does not retry, replay or commit on its own. Results are plain objects with snake_case keys, whole numbers are exact (`bigint` beyond `Number.MAX_SAFE_INTEGER`), bytes are `Buffer`s, and the TypeScript typings diff --git a/api/node/test/branch-file/inventory.mjs b/api/node/test/branch-file/inventory.mjs index 39debc4c41..a16f8d05c3 100644 --- a/api/node/test/branch-file/inventory.mjs +++ b/api/node/test/branch-file/inventory.mjs @@ -11,8 +11,8 @@ const root = fileURLToPath(new URL("../../../../", import.meta.url)); const command = process.env.LAYERFS_CLI ?? join(root, "target/debug/layerfs"); const docs = join(root, "docs/architecture/api/sdk"); -test("the SDK pages name all 73 current calls, one `### family.call` heading each", () => { - assert.equal(table.length, 73); +test("the SDK pages name all 76 current calls, one `### family.call` heading each", () => { + assert.equal(table.length, 76); const families = [...new Set(table.map((spec) => spec.family))]; assert.equal(families.length, 10); const index = readFileSync(join(docs, "README.md"), "utf8"); diff --git a/api/node/test/calls.mjs b/api/node/test/calls.mjs index f417bcab15..5eb15f49a5 100644 --- a/api/node/test/calls.mjs +++ b/api/node/test/calls.mjs @@ -14,9 +14,9 @@ const temp = (tag) => { return dir; }; -test("the table: ten families, 73 calls, each a function", () => { +test("the table: ten families, 76 calls, each a function", () => { assert.deepEqual(families, ["project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs"]); - assert.equal(table.length, 73); + assert.equal(table.length, 76); for (const spec of table) assert.equal(typeof layerfs[spec.family][spec.call], "function", `${spec.family}.${spec.call}`); }); diff --git a/api/python/test/test_calls.py b/api/python/test/test_calls.py index adeabfb50e..e8fc9793a3 100644 --- a/api/python/test/test_calls.py +++ b/api/python/test/test_calls.py @@ -340,7 +340,7 @@ def test_the_table(self): layerfs.FAMILIES, ["project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs"], ) - self.assertEqual(len(layerfs.TABLE), 73) + self.assertEqual(len(layerfs.TABLE), 76) for spec in layerfs.TABLE: self.assertTrue(callable(getattr(getattr(layerfs, spec["family"]), spec["call"]))) with self.assertRaises(layerfs.LayerFSError) as caught: diff --git a/api/scripts/gen-coverage.mjs b/api/scripts/gen-coverage.mjs index 03b36f7fca..55474981db 100644 --- a/api/scripts/gen-coverage.mjs +++ b/api/scripts/gen-coverage.mjs @@ -101,9 +101,12 @@ function usage(words) { /** What the command line of `spec` must be, by the table: a content or JSON positional is read from * stdin or `--from`, `on_progress` is `--progress`, the output functions are `--stream`. */ function expectedCli(spec) { + const options = spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)); + // the command frames a call with a stdout sink as events: `--events`, last + if (spec.options.some((o) => o.name === "stdout")) options.push("events"); return { positionals: spec.args.filter((a) => !piped(a)).map((a) => ({ name: a.name, optional: a.optional })), - options: spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)), + options, from: spec.args.some(piped), }; } diff --git a/control/crates/layerfs-control/src/inventory.rs b/control/crates/layerfs-control/src/inventory.rs index d84ff2afdb..38256c129c 100644 --- a/control/crates/layerfs-control/src/inventory.rs +++ b/control/crates/layerfs-control/src/inventory.rs @@ -216,15 +216,3 @@ pub fn store_idle(home: &Path, store: &str) -> Result<()> { held.extend(related("sandbox", &sandboxes)); Err(in_use(why.join("; "), held)) } - -/// `project.remove`: refused `InUse` while a Sandbox of `project` is -/// recorded; `detail.related` lists them as `{kind: "sandbox", name}`. -pub fn can_remove_project(home: &Path, project: &str) -> Result<()> { - let sandboxes = sandboxes_of_project(home, project)?; - let message = match sandboxes.as_slice() { - [] => return Ok(()), - [sandbox] => format!("Sandbox {sandbox} of {project} is running"), - many => format!("Sandboxes {} of {project} are running", many.join(", ")), - }; - Err(in_use(message, related("sandbox", &sandboxes))) -} diff --git a/control/crates/layerfs-control/src/project.rs b/control/crates/layerfs-control/src/project.rs index baee69b1eb..af30e384f6 100644 --- a/control/crates/layerfs-control/src/project.rs +++ b/control/crates/layerfs-control/src/project.rs @@ -1,20 +1,13 @@ -//! The `project` call that checks who uses a project, `project.remove` -//! (layerfs-project#105); the other `project` calls are `layerfs-store`'s -//! own. +//! The `project` call that retires a project in its Store and removes the +//! caller's home entry (`project.delete`, spec §4); the other `project` +//! calls are `layerfs-store`'s own. use layerfs_home::events::Events; use layerfs_home::names::ProjectRef; use layerfs_home::outcome::Outcome; use std::path::Path; -/// `project.remove`: a project's history dropped from its Store, once its -/// Sandboxes are closed. -pub fn remove(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { - layerfs_store::project::family::remove(home, events, project, crate::store::RELATED) -} - /// `project.delete(project)`: retires the project in its Store and removes -/// the caller home entry (spec §4). Add the API and MCP call-table entry in -/// T11; this is the service/control layer behind that dispatch. +/// the caller home entry (spec §4). pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { layerfs_store::project::delete::delete(home, events, project) } diff --git a/control/crates/layerfs-control/src/store.rs b/control/crates/layerfs-control/src/store.rs index 8b7dc2334f..4dd9903e85 100644 --- a/control/crates/layerfs-control/src/store.rs +++ b/control/crates/layerfs-control/src/store.rs @@ -27,7 +27,6 @@ pub fn compact(home: &Path, events: &mut Events<'_>, name: &StoreName) -> Outcom /// Who uses a Store or a project, read from the Sandbox records. pub const RELATED: Related = Related { store_idle: inventory::store_idle, - can_remove_project: inventory::can_remove_project, }; /// `store.remove`: a sqlite Store no Sandbox uses, with its file. diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index 3e7d2cf8bd..b768955acb 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -10,8 +10,8 @@ //! build sends nothing more and closes. Then a request `{ op, args }`, read with at most //! [`REQUEST_WAIT`] between bytes. `worker.ping` answers who it is; //! `store.call`, `store.file`, `store.reach`, `sandbox.host`, `project.stage`, -//! `project.init`, `project.export`, `project.remove`, `project.delete` and -//! `project.list` run one operation on this machine, +//! `project.init`, `project.export`, `project.delete` and `project.list` run +//! one operation on this machine, //! with `home` set to the worker's own root (the machine's data disk), //! never a path the caller sends. Answers are //! `{ value }` or `{ refused }`, after any `{ event }` frames. @@ -33,10 +33,10 @@ use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; use layerfs_store::project::frame::{ DeleteAdmitFrame, DeleteFrame, DeleteReleaseFrame, ExportFrame, InitFrame, ListingFrame, - RemoveFrame, StageFrame, + StageFrame, }; use layerfs_store::project::listing::list_projects; -use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; +use layerfs_store::project::{init::in_worker, staged}; use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -273,12 +273,6 @@ fn run( }, layerfs_sandbox::workspace::dispatch::call, ), - // drops one project from the Store under this worker's root (2.0): - // `{ store, project, store_record }` - "project.remove" => { - let root = home_of(Some(root))?; - remove_project(&root, &RemoveFrame::decode(&args)?) - } // retires one project in the Store under this worker root and // removes the worker own entry (2.0, T4.5b): `{ store, stack_id, // deleted_ms?, project?, op_id?, store_record? }` diff --git a/control/crates/layerfs-control/tests/in_use_detail.rs b/control/crates/layerfs-control/tests/in_use_detail.rs index e83e63c43c..be250e38ae 100644 --- a/control/crates/layerfs-control/tests/in_use_detail.rs +++ b/control/crates/layerfs-control/tests/in_use_detail.rs @@ -1,9 +1,9 @@ -//! `InUse` from `project.remove` and `store.disconnect` fills -//! `detail.related` with `{kind, name}` entries (#536 §10b refusal detail): -//! the project's Sandboxes, and another process's lease on the Store. +//! `InUse` from `store.disconnect` fills `detail.related` with +//! `{kind, name}` entries (#536 §10b refusal detail): another process's +//! lease on the Store. use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::events::Events; -use layerfs_home::home::{paths, private_dir, write_entity, write_record}; +use layerfs_home::home::{paths, private_dir, write_entity}; use std::path::PathBuf; fn t(s: &str) -> SdkValue { @@ -34,36 +34,6 @@ fn refused(run: impl FnOnce(&mut Events<'_>) -> layerfs_home::outcome::Outcome) run(&mut Events::new(&mut sink)).unwrap_err() } -#[test] -fn project_remove_names_the_project_s_sandboxes() { - let home = home("project"); - let store = SdkValue::map([ - ("name", t("shop")), - ("kind", t("sqlite")), - ("machine", t("native")), - ]); - write_entity(&paths::store_record(&home, "shop"), &store).unwrap(); - let project = SdkValue::map([("name", t("api"))]); - write_entity(&paths::project_record(&home, "shop", "api"), &project).unwrap(); - for name in ["sa", "sb"] { - let sandbox = SdkValue::map([ - ("v", SdkValue::Int(1)), - ("name", t(name)), - ("project", t("shop/api")), - ("machine", t("native")), - ]); - write_record(&paths::sandbox_record(&home, name), &sandbox).unwrap(); - } - let project = layerfs_home::names::ProjectRef::parse("shop/api").unwrap(); - let refusal = refused(|events| layerfs_control::project::remove(&home, events, &project)); - assert_eq!(refusal.code, "InUse", "{refusal:?}"); - let sandbox = |n: &str| ("sandbox".to_owned(), n.to_owned()); - assert_eq!(related(&refusal), [sandbox("sa"), sandbox("sb")]); - // nothing was removed - assert!(paths::project_record(&home, "shop", "api").is_file()); - let _ = std::fs::remove_dir_all(&home); -} - #[test] fn store_disconnect_names_another_process_s_lease() { let home = home("disconnect"); diff --git a/control/crates/layerfs-control/tests/inventory.rs b/control/crates/layerfs-control/tests/inventory.rs index d72f18ba2a..a2d8d91c4a 100644 --- a/control/crates/layerfs-control/tests/inventory.rs +++ b/control/crates/layerfs-control/tests/inventory.rs @@ -216,13 +216,6 @@ fn the_checks_refuse_in_use_with_the_same_texts() { }; assert_eq!(names(refusal.detail.get("related").unwrap()), ["a", "c"]); assert!(inventory::store_idle(&home, "pg").is_ok()); - let refusal = inventory::can_remove_project(&home, "docs/web").unwrap_err(); - assert_eq!(refusal.message, "Sandbox b of docs/web is running"); - assert_eq!(names(refusal.detail.get("related").unwrap()), ["b"]); - let refusal = inventory::can_remove_project(&home, "shop/api").unwrap_err(); - assert_eq!(refusal.message, "Sandboxes a, c of shop/api are running"); - assert_eq!(names(refusal.detail.get("related").unwrap()), ["a", "c"]); - assert!(inventory::can_remove_project(&home, "shop/web").is_ok()); let _ = std::fs::remove_dir_all(&home); } diff --git a/control/crates/layerfs-control/tests/project_delete_worker.rs b/control/crates/layerfs-control/tests/project_delete_worker.rs index c1ab487c20..6e301384a5 100644 --- a/control/crates/layerfs-control/tests/project_delete_worker.rs +++ b/control/crates/layerfs-control/tests/project_delete_worker.rs @@ -20,7 +20,7 @@ use layerfs_store::sqlite::create::create; use layerfs_store_service::history::lookup; use std::net::TcpListener; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; static NEXT: AtomicU64 = AtomicU64::new(0); @@ -153,7 +153,7 @@ fn the_worker_path_admits_before_the_sidecar_and_releases_after_the_marker() { store: StoreName::parse("shop").unwrap(), name: "app".to_owned(), source, - timeout_ms: None, + timeout_ms: Some(60_000), store_record: None, }) .unwrap(); @@ -298,10 +298,11 @@ fn the_worker_path_admits_before_the_sidecar_and_releases_after_the_marker() { } /// A fake worker on `listener`: answers its secret, answers each request with -/// `reply`, and shares back the requests it read. -fn fake_worker( +/// `reply`, and shares back the requests it read. A `None` answer drops the +/// connection without a reply, as a lost answer does. +fn fake_worker_with( listener: TcpListener, - reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, + reply: impl Fn(&SdkValue) -> Option + Send + 'static, ) -> Arc>> { let seen: Arc>> = Arc::default(); let requests = seen.clone(); @@ -325,12 +326,22 @@ fn fake_worker( }; let answer = reply(&request); requests.lock().unwrap().push(request); - let _ = write_frame(&mut stream, &answer); + if let Some(answer) = answer { + let _ = write_frame(&mut stream, &answer); + } } }); seen } +/// [`fake_worker_with`] of a worker that always answers. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Arc>> { + fake_worker_with(listener, move |request| Some(reply(request))) +} + /// The `op` of one request the fake worker read. fn op_of(request: &SdkValue) -> &str { request.get_text("op").unwrap_or("") @@ -548,3 +559,94 @@ fn hex_decode(text: &str) -> Vec { .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) .collect() } + +#[test] +fn a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([6; 16]); + let home = caller_home("fake-lost", &address, stack); + // The first marker attempt loses its answer; the second (the retry) + // retires the Stack with the sidecar's `deleted_ms`. + let attempts = Arc::new(AtomicU32::new(0)); + let count = attempts.clone(); + let seen = fake_worker_with(listener, move |request| match op_of(request) { + "project.delete.admit" => Some(admitted_for(stack)), + "project.delete" => { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + None + } else { + Some(removed_for(stack, 42)) + } + } + "project.delete.release" => Some(released_for(request)), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("the marker answer was lost"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + // The sidecar stays `pending`, naming the admission, and the entry stays + // for the retry: an unknown outcome is never reported clean. + let intent = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays"); + assert_eq!( + intent.state, + layerfs_store::project::delete::IntentState::Pending + ); + let op = intent.op.expect("the sidecar names its admission"); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let first = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), first); + assert_eq!(sent_op(&requests[2]), first); + assert_eq!(hex_decode(&first), op.to_vec()); + drop(requests); + + // The retry re-registers the same identity and finishes the entry. + let outcome = remove(&home).expect("the retry settles the unknown outcome"); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None, + "the retry unlinked the sidecar" + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release", + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let same: Vec<&str> = requests[3..].iter().map(sent_op_ref).collect(); + assert_eq!(same, [first.as_str(), first.as_str(), first.as_str()]); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The `op_id` a request's args carry, borrowed. +fn sent_op_ref(request: &SdkValue) -> &str { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") +} diff --git a/control/crates/layerfs-control/tests/store_transfer_recovery.rs b/control/crates/layerfs-control/tests/store_transfer_recovery.rs index c122aa6b8e..0dea8ba5d7 100644 --- a/control/crates/layerfs-control/tests/store_transfer_recovery.rs +++ b/control/crates/layerfs-control/tests/store_transfer_recovery.rs @@ -41,7 +41,6 @@ fn idle(_home: &Path, _name: &str) -> Result<(), SdkRefusal> { fn related() -> Related { Related { store_idle: idle, - can_remove_project: idle, } } diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index 49c161ef81..4aad4a659e 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -291,6 +291,19 @@ fn matches_reply(request: &Request, reply: &Reply) -> bool { Reply::Bound { token, binding }, ) => token.workspace == *workspace && binding.branch.name.as_str() == branch, (Request::MountNamed { .. }, Reply::SdkRefused(_)) => true, + ( + Request::MountPlaced { + workspace, branch, .. + }, + Reply::Bound { token, binding }, + ) => token.workspace == *workspace && binding.branch.id == *branch, + ( + Request::MountNamedPlaced { + workspace, branch, .. + }, + Reply::Bound { token, binding }, + ) => token.workspace == *workspace && binding.branch.name.as_str() == branch, + (Request::MountNamedPlaced { .. }, Reply::SdkRefused(_)) => true, ( Request::Commit(_), Reply::Committed(_) | Reply::MergeCommitted(_) | Reply::Conflict(_), diff --git a/control/crates/layerfs-sandbox/src/daemon/internal.rs b/control/crates/layerfs-sandbox/src/daemon/internal.rs index 3188e6d448..58b6a0ace2 100644 --- a/control/crates/layerfs-sandbox/src/daemon/internal.rs +++ b/control/crates/layerfs-sandbox/src/daemon/internal.rs @@ -2,7 +2,7 @@ pub use super::channel::{Control, ControlCause, ControlFailure, ControlPhase}; use crate::{BoundWorkspace, OperationFailure, WorkspaceApi}; use layerfs_bridge::{ - control::{CleanupObservation, ReadyMount, WorkspaceStatus, WorkspaceToken}, + control::{CleanupObservation, MountPlace, ReadyMount, WorkspaceStatus, WorkspaceToken}, duplex::Connection, }; use layerfs_history::{BranchId, WorkspaceId}; @@ -16,6 +16,7 @@ pub fn workspace(control: &mut Control) -> InternalWorkspaceApi<'_> { InternalWorkspaceApi(WorkspaceApi { control, observation_scope: None, + place: MountPlace::default(), }) } /// Applies explicit observation scope only to this borrow's filesystem calls. @@ -26,6 +27,7 @@ pub fn with_observations<'a>( InternalWorkspaceApi(WorkspaceApi { control, observation_scope: Some(scope), + place: MountPlace::default(), }) } impl<'a> Deref for InternalWorkspaceApi<'a> { @@ -40,6 +42,13 @@ impl DerefMut for InternalWorkspaceApi<'_> { } } impl InternalWorkspaceApi<'_> { + /// Names the Sandbox and machine a mount through this channel belongs + /// to; a mount then registers them beside its Workspace. + #[must_use] + pub fn placed(mut self, sandbox: Option<&str>, machine: Option<&str>) -> Self { + self.0.place = MountPlace::new(sandbox, machine); + self + } /// Original logical bind for component proofs; kernel mount still uses both steps. pub fn bind( &mut self, diff --git a/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs b/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs index 038cf2245f..9c5e86e90f 100644 --- a/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs +++ b/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs @@ -1,6 +1,6 @@ //! Logical binding and kernel mount acknowledgements. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::control::{ReadyMount, Reply, Request, WorkspaceToken}; +use layerfs_bridge::control::{MountPlace, ReadyMount, Reply, Request, WorkspaceToken}; use layerfs_history::{BranchId, BranchSnapshot, WorkspaceId}; /// Explicit filesystem operations on one caller-owned sequential control channel. @@ -8,6 +8,10 @@ use layerfs_history::{BranchId, BranchSnapshot, WorkspaceId}; pub struct WorkspaceApi<'a> { pub control: &'a mut Control, pub observation_scope: Option<&'a [u8; 32]>, + /// The Sandbox and machine a mount through this channel belongs to, + /// when its caller knows them: the daemon registers the holder with + /// them. Both absent when the caller knows neither. + pub place: MountPlace, } impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. @@ -21,8 +25,16 @@ impl<'a> WorkspaceApi<'a> { Self { control, observation_scope: None, + place: MountPlace::default(), } } + /// Names the Sandbox and machine a mount through this channel belongs + /// to; a mount then registers them beside its Workspace. + #[must_use] + pub fn placed(mut self, sandbox: Option<&str>, machine: Option<&str>) -> Self { + self.place = MountPlace::new(sandbox, machine); + self + } pub fn exchange(&mut self, request: Request) -> Result> { let request = match self.observation_scope { Some(scope) => Request::Observed { diff --git a/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs b/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs index 1c6836649b..25ab47ebce 100644 --- a/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs +++ b/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs @@ -12,7 +12,15 @@ impl WorkspaceApi<'_> { workspace: WorkspaceId, branch: BranchId, ) -> Result> { - let request = Request::Mount { workspace, branch }; + let request = if self.place.known() { + Request::MountPlaced { + workspace, + branch, + place: self.place.clone(), + } + } else { + Request::Mount { workspace, branch } + }; match self.exchange(request.clone())? { Reply::Bound { token, binding } => Ok(BoundWorkspace { token, binding }), reply => Err(OperationFailure::unexpected(request, reply)), @@ -27,10 +35,19 @@ impl WorkspaceApi<'_> { project: &str, branch: &str, ) -> Result> { - let request = Request::MountNamed { - workspace, - project: project.to_owned(), - branch: branch.to_owned(), + let request = if self.place.known() { + Request::MountNamedPlaced { + workspace, + project: project.to_owned(), + branch: branch.to_owned(), + place: self.place.clone(), + } + } else { + Request::MountNamed { + workspace, + project: project.to_owned(), + branch: branch.to_owned(), + } }; match self.exchange(request.clone())? { Reply::Bound { token, binding } => Ok(BoundWorkspace { token, binding }), diff --git a/control/crates/layerfs-sandbox/src/registry.rs b/control/crates/layerfs-sandbox/src/registry.rs index 6864944940..91d407a8d7 100644 --- a/control/crates/layerfs-sandbox/src/registry.rs +++ b/control/crates/layerfs-sandbox/src/registry.rs @@ -1,6 +1,6 @@ //! What this process owns: its running Inits and its live Sandboxes. A -//! project itself is only its home entry, so a project removed there can be -//! initialized again without a restart. +//! project itself is only its home entry, so a project deleted there can be +//! initialized again (under another name, until a compaction frees the name). //! //! A Sandbox is one daemon process (`runtime`: native, or namespaces with //! its init stage), known by its name. Its runtime runs diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index 457cfb1356..c878cde8c0 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -13,11 +13,11 @@ use crate::daemon::values; use crate::host::Host; use crate::internal::workspace; use crate::lifecycle::{operation, WORKSPACES}; -use crate::record::{DaemonRecord, WorkspaceRecord}; +use crate::record::{DaemonRecord, SandboxRecord, WorkspaceRecord}; use crate::registry::{sandbox, sandbox_live, Mount}; use crate::OperationCause; use crate::Unmounted; -use layerfs_bridge::control::{CommitControl, Reply, SdkRefusal, SdkValue}; +use layerfs_bridge::control::{CommitControl, MountPlace, Reply, SdkRefusal, SdkValue}; use layerfs_history::{BranchId, CommitId, WorkspaceId}; use layerfs_home::helpers::{lock, random, required}; use layerfs_home::names::WorkspaceName; @@ -102,7 +102,14 @@ pub fn mount(host: &Host, branch: Option<&str>, name: Option<&str>) -> Outcome { let id = WorkspaceId::from_authority(random()?).map_err(failed)?; let mounted = { let mut control = crate::daemon::trace::control(&state.controls)?; - workspace(&mut control).mount_named(id, (project, branch)) + // This Sandbox's record names the machine it runs on; the daemon + // that binds the mount registers both names beside its Workspace. + let machine = + SandboxRecord::read_in(&directory, sandbox)?.map(|record| record.machine().to_owned()); + let place = MountPlace::new(Some(state.id.as_str()), machine.as_deref()); + workspace(&mut control) + .placed(place.sandbox.as_deref(), place.machine.as_deref()) + .mount_named(id, (project, branch)) }; let mounted = mounted.map_err(|failure| match (&failure.bound, &failure.failure.cause) { // the daemon's lookup refusal, nothing bound: the call's answer diff --git a/control/crates/layerfs-store/Cargo.toml b/control/crates/layerfs-store/Cargo.toml index 0c2a6ad807..830dbf4cd3 100644 --- a/control/crates/layerfs-store/Cargo.toml +++ b/control/crates/layerfs-store/Cargo.toml @@ -31,3 +31,9 @@ postgresql = [ ] # https:// remotes: Cloudflare R2, MinIO behind TLS. tls = ["layerfs-bucket/tls"] + +# The control-level PostgreSQL `store.status` test drives a real server and +# inserts the rows a compaction leaves; this is the pinned client the +# persistence PostgreSQL feature already locks, no new version. +[dev-dependencies] +postgres = "=0.19.14" diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs index dab93bac77..0a20fc12c7 100644 --- a/control/crates/layerfs-store/src/compact.rs +++ b/control/crates/layerfs-store/src/compact.rs @@ -15,25 +15,71 @@ use crate::open_args::OpenArgs; use crate::place::{self, Executor}; #[cfg(feature = "postgresql")] use crate::postgresql; -use crate::record::{self, StoreRecord, StoreRef}; +use crate::record::{StoreRecord, StoreRef}; use layerfs_bridge::control::SdkRefusal; use layerfs_home::events::Events; use layerfs_home::home::paths; use layerfs_home::outcome::{failed, refused, Outcome}; +use layerfs_persistence::compact_metrics::{self, CompactRefusal}; #[cfg(feature = "postgresql")] use layerfs_persistence::PostgresHandles; use layerfs_persistence::{ - CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, + CompactFailpoint, CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, }; use layerfs_store_service::history::{admission, lookup}; use std::path::Path; +/// The T-G65 crash harness's gate: the real `layerfs store compact` process it +/// starts carries this socket, and the SQLite engine stops at the failpoint +/// below instead of reporting an unknown outcome. Ordinary callers set +/// neither variable, so a run carries no failpoint. +const CRASH_GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +/// The stop point the crash harness names; honored only beside the gate. +const CRASH_FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// The failpoint the crash harness named, or `None` in every ordinary run. +/// +/// A failpoint alone would stop the run in-process; the gate makes the stop +/// observable to the harness, which then `SIGKILL`s this process. Requiring +/// both keeps the hook inert for production callers and for builds without +/// the engine's Unix-socket gate. +#[cfg(unix)] +fn crash_failpoint() -> Option { + std::env::var_os(CRASH_GATE_ENV)?; + match std::env::var(CRASH_FAILPOINT_ENV).ok()?.as_str() { + "BeforeSeal" => Some(CompactFailpoint::BeforeSeal), + "AfterSeal" => Some(CompactFailpoint::AfterSeal), + "BeforeVacuum" => Some(CompactFailpoint::BeforeVacuum), + "AfterBuild" => Some(CompactFailpoint::AfterBuild), + "DuringVerify" => Some(CompactFailpoint::DuringVerify), + "BeforeInstall" => Some(CompactFailpoint::BeforeInstall), + "DuringRename" => Some(CompactFailpoint::DuringRename), + "AfterInstall" => Some(CompactFailpoint::AfterInstall), + "DuringSettle" => Some(CompactFailpoint::DuringSettle), + _ => None, + } +} + +/// No gate, no failpoint: a stop would hang the run in-process. +#[cfg(not(unix))] +fn crash_failpoint() -> Option { + None +} + /// `store.compact(name)`: reclaim one sqlite Store's dead bytes, answering /// the §11 report. pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { let home = home.to_path_buf(); - let (_lease, record) = lease::claim(&home, store)?; + // A live operation is `Busy`, a dead owner's record `Unresolved`; the + // refusal is a bounded metric label and never changes what is returned. + let (_lease, record) = match lease::claim(&home, store) { + Ok(claimed) => claimed, + Err(refusal) => { + note_refusal(&refusal); + return Err(refusal); + } + }; let store_ref = record.store_ref()?.clone(); if matches!(store_ref, StoreRef::Postgres { .. }) { return compact_postgresql(&record, &home, store); @@ -76,6 +122,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { owner_pid: Some(std::process::id()), owner_pid_start, owner_pid_boot, + failpoint: crash_failpoint(), ..CompactRequest::default() }) .map_err(lookup::history)?; @@ -85,7 +132,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { } /// `store.compact` of a PostgreSQL Store in a build without the `postgresql` -/// feature is `Unsupported`. +/// feature: no engine is linked, so the call is `Unsupported`. #[cfg(not(feature = "postgresql"))] fn compact_postgresql(_record: &StoreRecord, _home: &Path, _store: &str) -> Outcome { Err(refused( @@ -97,9 +144,12 @@ fn compact_postgresql(_record: &StoreRecord, _home: &Path, _store: &str) -> Outc /// The §8 PostgreSQL path: dial the Store's recorded URL, open its schema as /// this home's authority, and run Phase M/A/B/C. The Store is usable at /// G + 1 from the swap on; a lost reply is judged by the next call (§9). +/// +/// Only built with the `postgresql` feature; the stub above answers +/// `Unsupported` without it. #[cfg(feature = "postgresql")] fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome { - let url = record::url(record)?; + let url = crate::record::url(record)?; let id = identity::identity(&OpenArgs { home: Some(home.to_path_buf()), store: Some(store.to_owned()), @@ -139,12 +189,24 @@ fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome /// failure with its own text. fn open_refusal(error: layerfs_storage::port::PersistenceError) -> SdkRefusal { use layerfs_storage::port::PersistenceError; - match error { + let refusal = match error { PersistenceError::Busy => refused("Busy", "the Store is held by another writer"), PersistenceError::Refused { status } => refused( "Busy", format!("the Store refused the compaction: {status}"), ), other => failed(other), + }; + note_refusal(&refusal); + refusal +} + +/// Counts a Busy/Unresolved refusal of a call that never reached a run; the +/// measured SQLite run records the refusals it produces itself. +fn note_refusal(refusal: &SdkRefusal) { + match refusal.code.as_str() { + "Busy" => compact_metrics::record_refusal(CompactRefusal::Busy), + "Unresolved" => compact_metrics::record_refusal(CompactRefusal::Unresolved), + _ => {} } } diff --git a/control/crates/layerfs-store/src/postgresql/connect.rs b/control/crates/layerfs-store/src/postgresql/connect.rs index 67228c5731..f070ca35bc 100644 --- a/control/crates/layerfs-store/src/postgresql/connect.rs +++ b/control/crates/layerfs-store/src/postgresql/connect.rs @@ -313,3 +313,27 @@ pub fn status(record: &StoreRecord) -> SdkValue { crate::postgresql::open::status(&url, &[SCHEMA.to_owned()]) .unwrap_or_else(|refusal| SdkValue::map([("unavailable", SdkValue::Text(refusal.message))])) } + +/// The Store's compaction operation as `store.status` (T10.5) renders it: +/// `{id, name, phase, started, uncertain, state, phases}`, or null when the +/// Store has none. One status query reads the Store-side `gc_op` and the +/// compaction `gc_intent`; no Store session is opened, no schema is changed +/// and no artifact or scratch schema is read. An unreachable server or a +/// query failure is null here: the `server` field already reports reach. +pub fn compact_status(record: &StoreRecord) -> SdkValue { + #[cfg(feature = "postgresql")] + { + let Ok(url) = url(record) else { + return SdkValue::Null; + }; + match layerfs_persistence::postgresql_compact_status(&url, SCHEMA) { + Ok(Some(op)) => crate::report::compact_status(&op.status()), + _ => SdkValue::Null, + } + } + #[cfg(not(feature = "postgresql"))] + { + let _ = record; + SdkValue::Null + } +} diff --git a/control/crates/layerfs-store/src/project/add.rs b/control/crates/layerfs-store/src/project/add.rs index af7e24516f..5e4236eaf0 100644 --- a/control/crates/layerfs-store/src/project/add.rs +++ b/control/crates/layerfs-store/src/project/add.rs @@ -7,6 +7,7 @@ use crate::project::discovery::Project; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::{ProviderKind, StoreManifest, StoreProfile}; use layerfs_bridge::sdk_values::hex; +use layerfs_history::error::{HolderKind, HolderState, NameHolder, NameSource, NameTakenState}; use layerfs_history::{BranchId, HistoryName, LayerStackId}; use layerfs_home::helpers::random; use layerfs_home::home::paths; @@ -14,6 +15,7 @@ use layerfs_home::names::StoreName; use layerfs_home::outcome::{failed, refused, Outcome}; use layerfs_persistence::{PersistenceConfig, SealedStore, SqlitePersistenceProfile}; use layerfs_project::store::{AddRequest, InitError, ProjectApi}; +use layerfs_store_service::history::lookup; use layerfs_project::{InitCounters, ProjectError}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -75,7 +77,7 @@ pub fn make_entry(home: &Path, store: &str, name: &str) -> Result { let entry = layerfs_home::home::paths::project_dir(home, store, name); std::fs::create_dir(&entry).map_err(|error| { if error.kind() == std::io::ErrorKind::AlreadyExists { - refused("Exists", format!("project {name} exists in Store {store}")) + home_name_taken(store, name) } else { failed(error) } @@ -85,6 +87,20 @@ pub fn make_entry(home: &Path, store: &str, name: &str) -> Result { Ok(entry) } +/// A project name the caller's home already records: `NameTaken` with its +/// `home` source, decided before the Store is asked anything. +pub(crate) fn home_name_taken(store: &str, name: &str) -> SdkRefusal { + let state = NameTakenState::new( + NameHolder::new(HolderKind::Project, name, HolderState::Live), + NameSource::Home, + ); + lookup::refuse( + "NameTaken", + format!("project {name} exists in Store {store}"), + lookup::name_taken_detail(&state), + ) +} + /// `project.init`'s add of folder `source` as the new project `name` of /// Store `store` in `home` (the caller's, or a worker's root), whose file is /// `/store//store.sqlite`. @@ -143,10 +159,7 @@ pub fn add_project(request: &ProjectAdd) -> Outcome { } let home = discovery::home_of(Some(&request.home))?; if name == store && legacy(&home, store)? { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(home_name_taken(store, name)); } let entry = make_entry(&home, store, name)?; let _guard = discovery::guard(&entry)?; @@ -213,9 +226,10 @@ pub fn add_project(request: &ProjectAdd) -> Outcome { /// diagnosis. pub fn add_failure(error: InitError, entry: Option<&Path>, name: &str, store: &str) -> SdkRefusal { let refusal = match &error { - InitError::Exists => refused( - "Exists", + InitError::NameTaken(state) => lookup::refuse( + "NameTaken", format!("Store {store} already has a project named {name}"), + lookup::name_taken_detail(state), ), InitError::NoAcquisition => refused( "Unsupported", diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs index 12fb89b68b..49f5cda341 100644 --- a/control/crates/layerfs-store/src/project/delete.rs +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -18,7 +18,7 @@ //! //! The worker op is `project.delete`; its frame is //! `{store, stack_id, deleted_ms?, project?, op_id?, store_record?}`. The two -//! optional context fields are the same ones `project.remove` sends: the +//! optional context fields are the ones the retired `project.remove` call sent: the //! worker opens the Store under its own root with the caller's `store_record`, //! and the project name lets its reply and its own home entry name the //! project. The worker saves the legacy own-project identity, writes the @@ -330,6 +330,13 @@ fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool ]) } +/// The side effects and the project-scope admission one finishing attempt +/// runs against: together, so `mark_and_finish` stays a small call. +struct Finish<'a> { + effects: &'a mut dyn Effects, + admission: Admission, +} + /// The mutating part of `delete_with` after the project scope is admitted: /// the identity save, the sidecar, the marker and the entry removal (spec §4). fn mark_and_finish( @@ -339,9 +346,12 @@ fn mark_and_finish( stack: LayerStackId, intent: Option, mut deleted_ms: i64, - effects: &mut impl Effects, - admission: Admission, + finish: Finish<'_>, ) -> Outcome { + let Finish { + effects, + admission, + } = finish; effects.prepare()?; let mut current = match intent { Some(intent) => intent, @@ -466,7 +476,16 @@ pub fn delete_with( } }; let attempt = mark_and_finish( - home, store, project, stack, intent, deleted_ms, effects, admission, + home, + store, + project, + stack, + intent, + deleted_ms, + Finish { + effects: &mut *effects, + admission, + }, ); let released = effects.release(admission); match attempt { diff --git a/control/crates/layerfs-store/src/project/family.rs b/control/crates/layerfs-store/src/project/family.rs index af8d81619d..6b1cf8985b 100644 --- a/control/crates/layerfs-store/src/project/family.rs +++ b/control/crates/layerfs-store/src/project/family.rs @@ -5,22 +5,19 @@ //! `native` or a PostgreSQL Store, in the machine's worker for a sqlite //! Store on any other machine (#536 H11). The folder is always on the //! caller's host: for a worker this process copies it to the machine -//! first (H11a). Records in the home list projects; `remove` drops one -//! project's history from its Store. +//! first (H11a). Records in the home list projects. use crate::calls::Page; use crate::lease; use crate::open_args::OpenArgs; use crate::place::{self, Executor, Worker}; use crate::project::add::{add_project, ProjectAdd}; -use crate::project::discovery::home_of; -use crate::project::frame::{InitFrame, ListingFrame, ListingReply, RemoveFrame}; +use crate::project::frame::{InitFrame, ListingFrame, ListingReply}; use crate::project::postgres_init::PgProjectAdd; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; use crate::resolve; use crate::service; use crate::sqlite::family::progress; -use crate::Related; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_home::home::paths; @@ -331,10 +328,7 @@ fn init_on_worker( timeout, } = call; if paths::project_record(&home, store, name).exists() { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(crate::project::add::home_name_taken(store.as_str(), name)); } let total = send::measure(Path::new(folder), name)?; let copy = send::Copy { @@ -399,10 +393,7 @@ fn init_postgresql( let home = home.to_path_buf(); let path = paths::project_record(&home, store, name); if path.exists() { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(crate::project::add::home_name_taken(store.as_str(), name)); } let add = PgProjectAdd { add: ProjectAdd { @@ -426,49 +417,3 @@ fn init_postgresql( } Ok(out) } - -/// `project.remove(project)`: drops the project's Layers, Branches and -/// Commits from its Store in one write, then its record. Its Sandboxes must -/// be closed first; the other projects stay. -pub fn remove( - home: &Path, - events: &mut Events<'_>, - project: &ProjectRef, - related: Related, -) -> Outcome { - let text = project.to_string(); - let home = home.to_path_buf(); - let target = resolve::target(&home, &text)?; - target.record.unblocked()?; - (related.can_remove_project)(&home, &text)?; - let request = RemoveFrame { - store: project.store.clone(), - project: target.project.clone(), - store_record: Some(target.record.to_record().clone()), - }; - if let StoreRef::Postgres { .. } = target.store_ref { - return Err(refused( - "Unsupported", - "removing a project from a PostgreSQL Store is not built", - )); - } - let removed = match place::plan(&home, &target.store_ref)? { - Executor::Local => { - let _lease = lease::hold(&home, &target.store)?; - crate::project::remove::remove_project(&home_of(Some(&home))?, &request)? - } - Executor::Worker(mut worker) => { - let _lease = lease::hold(&home, &target.store)?; - worker.call("project.remove", request.encode(), &mut |e| events.event(e))? - } - }; - let entry = paths::project_dir(&home, &target.store, &target.project); - if entry.exists() { - std::fs::remove_dir_all(&entry).map_err(failed)?; - } - Ok(layerfs_home::home::with_field( - removed, - "project", - SdkValue::text(text), - )) -} diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 808fbd46e4..bcdfa67405 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,5 +1,5 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.list` (T4.5c), `project.stage`, +//! `project.list` (T4.5c), `project.stage`, //! `project.export`, `project.delete` (T4.5b) and its admission ops, each //! encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. @@ -30,18 +30,6 @@ pub struct InitFrame { pub store_record: Option, } -/// `project.remove`: drop project `project` from Store `store` (here, or in -/// its worker). -#[derive(Clone, Debug)] -pub struct RemoveFrame { - /// The Store. - pub store: StoreName, - /// The project's name in it. - pub project: String, - /// The Store's record. - pub store_record: Option, -} - /// `project.export` in a worker: stream Branch `branch` of project /// `project` of Store `store` back to the caller. #[derive(Clone, Debug)] @@ -146,30 +134,6 @@ impl InitFrame { } } -impl RemoveFrame { - /// The `project.remove` frame: `{ store, project, store_record }`. - pub fn encode(&self) -> SdkValue { - let mut fields = vec![ - ("store", SdkValue::text(self.store.as_str())), - ("project", SdkValue::text(&self.project)), - ]; - fields.extend(self.store_record.clone().map(|r| ("store_record", r))); - map(fields) - } - - /// A `project.remove` frame: no Store, no project, then a bad Store - /// name are refused. - pub fn decode(frame: &SdkValue) -> Result { - let store = required(frame.get_text("store"), "store")?; - let project = required(frame.get_text("project"), "project")?.to_owned(); - Ok(Self { - store: StoreName::parse(store)?, - project, - store_record: frame.get("store_record").cloned(), - }) - } -} - impl StageFrame { /// The `project.stage` frame: `{ store, name, staged, step, part? }`. pub fn encode(self) -> SdkValue { diff --git a/control/crates/layerfs-store/src/project/mod.rs b/control/crates/layerfs-store/src/project/mod.rs index 65597449ed..1b35d9d8ca 100644 --- a/control/crates/layerfs-store/src/project/mod.rs +++ b/control/crates/layerfs-store/src/project/mod.rs @@ -13,7 +13,6 @@ pub mod parts; pub mod postgres_init; #[doc(hidden)] pub mod record; -pub mod remove; pub mod send; pub mod staged; pub mod tree; diff --git a/control/crates/layerfs-store/src/project/remove.rs b/control/crates/layerfs-store/src/project/remove.rs deleted file mode 100644 index 4e5200e347..0000000000 --- a/control/crates/layerfs-store/src/project/remove.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! A project removed from its Store (`project.remove`, here or in the -//! Store's worker). -use crate::identity::{identity, legacy, save_record, store_dir}; -use crate::open_args::OpenArgs; -use crate::project::frame::RemoveFrame; -use layerfs_bridge::control::SdkValue; -use layerfs_bridge::sdk_values::hex; -use layerfs_home::home::paths; -use layerfs_home::outcome::{failed, refused, Outcome}; -use std::path::Path; - -/// `project.remove`'s work on the Store's machine (here, or in its worker -/// as the worker op `project.remove`), in `home` (the caller's, or the -/// worker's root; checked by the caller with [`home_of`](super::discovery::home_of)): -/// drops the project's stack (its Layers, Branches and Commits) from the -/// Store in one write, then its entry from the home. The other projects -/// stay. The entry of the project a Store was made by (its own Init) holds -/// the Store's identity, so a Store record is saved before that entry goes. -/// Its packs and objects stay in the Store; reclaiming them is A8 (#411). -/// The caller stops the project's Sandboxes and clients first. -pub fn remove_project(home: &Path, request: &RemoveFrame) -> Outcome { - let (store, name) = (request.store.as_str(), request.project.as_str()); - let open = OpenArgs { - home: Some(home.to_path_buf()), - store: Some(store.to_owned()), - project: Some(name.to_owned()), - store_path: Some(paths::store_file(home, store).display().to_string()), - store_record: request.store_record.clone(), - postgresql: None, - }; - let identity = identity(&open)?; - let own = name == store && legacy(home, store)?; - let entry = layerfs_home::home::paths::project_dir(home, store, name); - let stack = crate::service::with_service(&open, |service| { - let stacks = service - .store - .published(layerfs_project::store::projects) - .map_err(failed)?; - let Some(stack) = stacks.into_iter().find(|stack| stack.name.as_str() == name) else { - return Err(refused( - "NoSuchProject", - format!("Store {store} holds no project named {name}"), - )); - }; - service - .store - .history() - .remove_layerstack(stack.id) - .map_err(|error| { - failed(format!( - "removing project {name} from Store {store}: {error:?}" - )) - })?; - Ok(stack.id) - })?; - // the own project's entry carried the Store's identity until now - if own { - layerfs_home::home::private_dir(&store_dir(home, store)?)?; - save_record(home, store, &identity)?; - } - if entry.exists() { - std::fs::remove_dir_all(&entry).map_err(failed)?; - } - Ok(SdkValue::map([ - ("stack", SdkValue::text(hex(&stack.to_bytes()))), - ("packsKept", SdkValue::Bool(true)), - ])) -} diff --git a/control/crates/layerfs-store/src/related.rs b/control/crates/layerfs-store/src/related.rs index 56b471049a..06e2bc9fcb 100644 --- a/control/crates/layerfs-store/src/related.rs +++ b/control/crates/layerfs-store/src/related.rs @@ -8,13 +8,11 @@ use std::path::Path; /// An `InUse` check on the Store or project `name` in home `home`. type Check = fn(home: &Path, name: &str) -> Result<(), SdkRefusal>; -/// The `InUse` checks over a Store's and a project's users. +/// The `InUse` checks over a Store's users. #[derive(Clone, Copy)] pub struct Related { /// `store.push`, `store.pull`, `store.remove`, `store.disconnect` and /// `backup.restore`: refused while another live process holds the Store /// or a Sandbox of one of its projects is recorded. pub store_idle: Check, - /// `project.remove`: refused while a Sandbox of the project is recorded. - pub can_remove_project: Check, } diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs index a02dc621e6..cf6ad16ed8 100644 --- a/control/crates/layerfs-store/src/report.rs +++ b/control/crates/layerfs-store/src/report.rs @@ -28,7 +28,7 @@ pub fn compact_report(report: &CompactReport) -> SdkValue { .residue .artifacts .iter() - .map(|artifact| SdkValue::text(artifact)) + .map(SdkValue::text) .collect(), ), ), diff --git a/control/crates/layerfs-store/src/status.rs b/control/crates/layerfs-store/src/status.rs index 5656f73eb7..664f7d4a87 100644 --- a/control/crates/layerfs-store/src/status.rs +++ b/control/crates/layerfs-store/src/status.rs @@ -6,6 +6,7 @@ use crate::sqlite::family::{fields_map, record_of, step_on}; use crate::sqlite::transfer::step::StoreStep; use layerfs_bridge::control::SdkValue; use layerfs_home::outcome::Outcome; +use layerfs_persistence::compact_metrics::{self, CompactStatusState}; use std::path::Path; /// `store.status`: who holds it, the operation on it (`op`, as H6 shows it), @@ -15,20 +16,22 @@ use std::path::Path; /// /// While a compaction runs, or after a dead owner left one, `op` is that run's /// projection and `state` is `busy`/`unresolved` (§11); `phases` names the -/// backend's ordered phases. The compaction record is read from -/// `.compact-op` alone: the Store database and its artifacts are never -/// opened for it. +/// backend's ordered phases. A sqlite compaction record is read from +/// `.compact-op` alone, and a PostgreSQL one from the Store-side +/// `gc_op` with its `gc_intent` on one status query (T10.5): neither opens +/// the Store database, changes a schema, nor reads an artifact. pub fn status(home: &Path, name: &str) -> Outcome { let record = record_of(home, name)?; let (holders, _) = lease::read_holders(home, name)?; let mut down = None; let mut more = Vec::new(); let store_ref = record.store_ref()?; - let mut compaction = SdkValue::Null; + let compaction: SdkValue; if let StoreRef::Postgres { .. } = store_ref { more.push(("server", crate::postgresql::connect::status(&record))); - // A PostgreSQL compaction's op lives in the Store's `gc_op`; T9 reads - // it here with its rebuild. This path reads no file. + // T10.5: a PostgreSQL compaction's op is the Store's `gc_op` row read + // with its `gc_intent`; this path opens no Store and reads no file. + compaction = crate::postgresql::connect::compact_status(&record); } else { more.push(("machine", SdkValue::text(store_ref.machine()?))); // the file's size where it lives, starting nothing (P4), and the @@ -67,6 +70,15 @@ pub fn status(home: &Path, name: &str) -> Outcome { let state = compaction .get_text("state") .unwrap_or_else(|| record.state(down.is_some(), &holders).as_str()); + // The status path observes an unsettled run by its bounded state label, + // never by a Store name or phase path. + if compaction.get("id").is_some() { + match state { + "busy" => compact_metrics::record_status(CompactStatusState::Busy), + "unresolved" => compact_metrics::record_status(CompactStatusState::Unresolved), + _ => {} + } + } let mut fields = vec![ ("name", SdkValue::text(name)), ("kind", SdkValue::text(store_ref.kind().as_str())), diff --git a/control/crates/layerfs-store/tests/compact_report_status.rs b/control/crates/layerfs-store/tests/compact_report_status.rs index 88aa9e03da..56a7a28068 100644 --- a/control/crates/layerfs-store/tests/compact_report_status.rs +++ b/control/crates/layerfs-store/tests/compact_report_status.rs @@ -159,6 +159,32 @@ fn the_report_wire_is_the_snake_case_contract_shape() { assert!(wire.get("zeroReason").is_none(), "no camelCase key"); } +#[test] +fn status_emits_bounded_busy_and_unresolved_metrics() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + let capture = layerfs_persistence::compact_metrics::CompactMetricsCapture::start(); + // Idle: no compaction record, so no `compact.status` sample. + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + // A live owner is `busy`; a gone one is `unresolved`. + sidecar(&home, "build", std::process::id()); + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + sidecar(&home, "build", 99_999_999); + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + let metrics = capture.finish(); + let value = |state: &str| { + metrics + .samples() + .iter() + .find(|sample| sample.name == "compact.status" && sample.label("state") == Some(state)) + .map_or(0, |sample| sample.value) + }; + assert_eq!(value("busy"), 1); + assert_eq!(value("unresolved"), 1); + assert_eq!(value("idle"), 0, "an idle status is not a compaction op"); + std::fs::remove_dir_all(home).unwrap(); +} + #[test] fn status_projects_the_sidecar_phase_and_state() { let home = store_home(); diff --git a/control/crates/layerfs-store/tests/postgresql_status.rs b/control/crates/layerfs-store/tests/postgresql_status.rs new file mode 100644 index 0000000000..3d0d2e933b --- /dev/null +++ b/control/crates/layerfs-store/tests/postgresql_status.rs @@ -0,0 +1,405 @@ +//! T10.5: the PostgreSQL branch of `store.status`, through the control call. +//! +//! `store.status` of a PostgreSQL Store reads the Store-side `gc_op` row with +//! its compaction `gc_intent` in one status query, projects the operation +//! `{id, name, phase, started, uncertain}`, and separates a live owner +//! (`busy`) from a leftover (`unresolved`); `phases` is PostgreSQL's §8 order +//! `mark`, `build`, `swap`, `cleanup`. A Store with no row — never compacted, +//! or a run that settled and removed its row — is idle, with `op: null` and +//! `phases: null`. +//! +//! This is the control-level end-to-end case T10.5 was missing: the +//! persistence suite proves the reader, and this test drives the call the SDK +//! and CLI actually reach (`layerfs_store::sqlite::family::status`, the +//! kind-neutral `store.status` route) against a real PostgreSQL Store. The +//! rows a compaction leaves are written directly, so the test proves the +//! projection, not a failpoint: a live owner is a real non-idle backend +//! recorded in `pg_stat_activity`. +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` to a +//! libpq connection string or URL (docs/architecture/engine/store/testing.md). +//! The test creates and drops its own database, because `store.status` reads +//! the one schema a 2.0 PostgreSQL Store uses (`layerfs`); a shared schema +//! would collide with another run. Without the variable it reports NOT_RUN. +#![cfg(feature = "postgresql")] + +use layerfs_bridge::control::SdkValue; +use layerfs_history::HistoryCatalogConfig; +use layerfs_home::home; +use layerfs_home::names::StoreName; +use layerfs_persistence::{PostgresHandles, PostgresSelection}; +use layerfs_storage::StoragePolicy; +use layerfs_store::record::StoreRecord; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The §8 phases `store.status` names for a PostgreSQL compaction, in order. +const PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + +const BINDING: &[u8] = b"t105-control-postgresql-status"; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t105-control-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The URL of `database` on the same server as `source`: a URL's path is +/// replaced; a libpq key/value string gains a final `dbname`, which wins. +fn database_url(source: &str, database: &str) -> String { + if source.starts_with("postgres://") || source.starts_with("postgresql://") { + let (base, query) = match source.split_once('?') { + Some((base, query)) => (base, format!("?{query}")), + None => (source, String::new()), + }; + let scheme = base.find("://").expect("a PostgreSQL URL") + 3; + match base[scheme..].find('/').map(|at| scheme + at) { + Some(at) => format!("{}/{database}{query}", &base[..at]), + None => format!("{base}/{database}{query}"), + } + } else { + format!("{source} dbname={database}") + } +} + +/// One disposable database: `store.status` reads the fixed `layerfs` schema, +/// so a test run needs a database nothing else uses. +struct Database { + source: String, + name: String, +} + +impl Database { + fn create(source: &str) -> Self { + let name = format!( + "lfs_t105_ctl_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let mut admin = postgres::Client::connect(source, postgres::NoTls).unwrap(); + admin + .batch_execute(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .unwrap(); + admin + .batch_execute(&format!("CREATE DATABASE {name}")) + .unwrap(); + Self { + source: source.to_owned(), + name, + } + } + + fn url(&self) -> String { + database_url(&self.source, &self.name) + } +} + +impl Drop for Database { + fn drop(&mut self) { + if let Ok(mut admin) = postgres::Client::connect(&self.source, postgres::NoTls) { + let _ = admin.batch_execute(&format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + self.name + )); + } + } +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn op_id(byte: u8) -> String { + format!("op-{}", hex(&[byte; 16])) +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + match value { + SdkValue::List(items) => items + .iter() + .map(|item| { + if let SdkValue::Text(text) = item { + text.clone() + } else { + panic!("not text: {item:?}"); + } + }) + .collect(), + other => panic!("not a list: {other:?}"), + } +} + +/// What a compaction leaves in `gc_op`/`gc_intent`; `pid` and +/// `backend_start` name the owner the §9 rule tests for liveness. +struct OpRow<'a> { + op_id: [u8; 16], + phase: &'a str, + generation: i64, + started_ms: i64, + pid: i64, + backend_start: &'a str, +} + +fn insert(admin: &mut postgres::Client, row: &OpRow<'_>) { + let artifact = format!("lfs_gc_{}", hex(&row.op_id)); + admin + .execute( + "INSERT INTO layerfs.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,\ + started_ms,artifact,pid,backend_start) \ + VALUES (1,$1,$2,$3,$4,$5,$6,$7,$3,$8,$9)", + &[ + &row.generation, + &row.phase, + &artifact, + &11i64, + &12i64, + &vec![9u8; 32], + &row.started_ms, + &row.pid, + &row.backend_start, + ], + ) + .unwrap(); + admin + .execute( + "INSERT INTO layerfs.gc_intent \ + (op_id,kind,scope_kind,owner_runtime,owner_boot,state,uncertain,phase,\ + generation,artifact,updated_ms) \ + VALUES ($1,'compaction','store',$2,$3,'live',FALSE,$4,$5,$6,$7)", + &[ + &row.op_id.to_vec(), + &"runtime-t105", + &"boot-t105", + &row.phase, + &row.generation, + &artifact, + &1_700_000_000_000i64, + ], + ) + .unwrap(); +} + +fn clear(admin: &mut postgres::Client) { + admin + .batch_execute( + "DELETE FROM layerfs.gc_op; \ + DELETE FROM layerfs.gc_intent WHERE kind='compaction'", + ) + .unwrap(); +} + +/// Waits until the recorded owner backend is gone from `pg_stat_activity`, +/// as the persistence fixture does. +fn wait_gone(admin: &mut postgres::Client, pid: i32, start: &str) { + for _ in 0..400 { + let live: bool = admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2)", + &[&pid, &start], + ) + .unwrap() + .get(0); + if !live { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend never ended"); +} + +/// The real control entry point: `store.status` for the Store in `home`. +fn status(home: &Path) -> SdkValue { + let name = StoreName::parse("pgs").unwrap(); + layerfs_store::sqlite::family::status(home, &name).unwrap() +} + +#[test] +fn postgresql_store_status_projects_busy_leftover_and_idle() { + let Some(source) = url() else { + return; + }; + let database = Database::create(&source); + let url = database.url(); + let handles = PostgresHandles::create( + &PostgresSelection::new(&url, "layerfs"), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + + let home = tempdir("home"); + home::ensure(&home).unwrap(); + // `store.connect` records exactly this shape; `store.status` reads it. + StoreRecord::connected("pgs", &url, "layerfs", "off", None) + .write(&home) + .unwrap(); + let mut admin = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + + // Idle: no compaction row, so no op and no phases. + let idle = status(&home); + assert_eq!(idle.get_text("name"), Some("pgs")); + assert_eq!(idle.get_text("kind"), Some("postgresql")); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!( + idle.get("op"), + Some(&SdkValue::Null), + "an idle Store has no op" + ); + assert_eq!( + idle.get("phases"), + Some(&SdkValue::Null), + "an idle Store has no phases" + ); + assert!( + matches!(idle.get("server"), Some(SdkValue::Map(_))), + "the server status is reported beside the op" + ); + + // A live owner: an open, non-idle backend is what the §9 dead-owner rule + // treats as still running the op. + let mut owner = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + owner.batch_execute("BEGIN; SELECT 1").unwrap(); + let (pid, start): (i32, String) = { + let row = owner + .query_one( + "SELECT pg_backend_pid(),backend_start::text FROM pg_stat_activity \ + WHERE pid=pg_backend_pid()", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + let started_ms = 1_700_000_000_000i64; + let live = OpRow { + op_id: [0xab; 16], + phase: "build", + generation: 7, + started_ms, + pid: i64::from(pid), + backend_start: &start, + }; + insert(&mut admin, &live); + + let capture = layerfs_persistence::compact_metrics::CompactMetricsCapture::start(); + let busy = status(&home); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + op, + &SdkValue::map([ + ("id", SdkValue::text(op_id(0xab))), + ("name", SdkValue::text("store.compact")), + ("phase", SdkValue::text("build")), + ("started", SdkValue::Int(started_ms)), + ("uncertain", SdkValue::Bool(true)), + ]), + "the op is exactly the five §11 fields" + ); + assert_eq!(texts(busy.get("phases").unwrap()), PHASES); + assert!( + op.get("state").is_none() && op.get("phases").is_none(), + "state and phases sit beside the op" + ); + + // A gone owner leaves the same row: uncertain, and now unresolved. + owner.close().unwrap(); + wait_gone(&mut admin, pid, &start); + let leftover = status(&home); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("id").unwrap()), op_id(0xab)); + assert_eq!(op.get_text("name"), Some("store.compact")); + assert_eq!(op.get_text("phase"), Some("build")); + assert_eq!(op.get_int("started"), Some(started_ms)); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!(texts(leftover.get("phases").unwrap()), PHASES); + + // The two observed states emit the bounded `compact.status{state}` sample + // once each; idle emits none. + let metrics = capture.finish(); + let sample = |state: &str| { + metrics + .samples() + .iter() + .find(|sample| sample.name == "compact.status" && sample.label("state") == Some(state)) + .map_or(0, |sample| sample.value) + }; + assert_eq!(sample("busy"), 1); + assert_eq!(sample("unresolved"), 1); + assert_eq!(sample("idle"), 0, "an idle status is not a compaction op"); + + // Every recorded phase is projected under its §8 name; a dead owner is + // an unresolved leftover for each. + for (index, phase) in PHASES.iter().enumerate() { + clear(&mut admin); + let byte = u8::try_from(index).unwrap(); + insert( + &mut admin, + &OpRow { + op_id: [byte; 16], + phase, + generation: 7, + started_ms, + pid: 99_999_999, + backend_start: "2000-01-01 00:00:00+00", + }, + ); + let projected = status(&home); + assert_eq!(projected.get_text("state"), Some("unresolved")); + assert_eq!( + projected.get("op").unwrap().get_text("phase"), + Some(*phase), + "{phase}: the row records the phase" + ); + let expected_id = op_id(byte); + assert_eq!( + projected.get("op").unwrap().get_text("id"), + Some(expected_id.as_str()) + ); + assert_eq!(texts(projected.get("phases").unwrap()), PHASES); + } + + // A settled run removes the row: idle again, with no op and no phases. + clear(&mut admin); + let idle = status(&home); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + + drop(admin); + drop(handles); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-store/tests/project_delete_store.rs b/control/crates/layerfs-store/tests/project_delete_store.rs index 1d0e945a70..fec3091bbb 100644 --- a/control/crates/layerfs-store/tests/project_delete_store.rs +++ b/control/crates/layerfs-store/tests/project_delete_store.rs @@ -72,7 +72,7 @@ fn shop(tag: &str) -> Option<(PathBuf, OpenArgs, LayerStackId)> { store: StoreName::parse("shop").unwrap(), name: "app".to_owned(), source, - timeout_ms: None, + timeout_ms: Some(60_000), store_record: None, }) .unwrap(); diff --git a/control/crates/layerfs-store/tests/project_frames.rs b/control/crates/layerfs-store/tests/project_frames.rs index e200d7f041..2afe04cff9 100644 --- a/control/crates/layerfs-store/tests/project_frames.rs +++ b/control/crates/layerfs-store/tests/project_frames.rs @@ -3,7 +3,7 @@ //! refuses in the order it always did. use layerfs_bridge::control::SdkValue; use layerfs_home::names::StoreName; -use layerfs_store::project::frame::{InitFrame, RemoveFrame, StageFrame, StageStep}; +use layerfs_store::project::frame::{InitFrame, StageFrame, StageStep}; fn t(text: &str) -> SdkValue { SdkValue::text(text) @@ -45,26 +45,6 @@ fn each_frame_encodes_the_map_its_caller_built() { assert_eq!((decoded.name, decoded.timeout_ms), (init.name, Some(5))); assert_eq!(decoded.store_record, init.store_record); - let remove = RemoveFrame { - store: store(), - project: "web".to_owned(), - store_record: Some(record.clone()), - }; - let encoded = remove.encode(); - assert_eq!( - encoded, - map(vec![ - ("store", t("shop")), - ("project", t("web")), - ("store_record", record), - ]) - ); - let decoded = RemoveFrame::decode(&encoded).unwrap(); - assert_eq!( - (decoded.store, decoded.project), - (store(), "web".to_owned()) - ); - let stage = |step| StageFrame { store: "shop".to_owned(), name: "web".to_owned(), @@ -98,15 +78,6 @@ fn the_worker_refuses_a_frame_in_its_old_order() { let bare = InitFrame::decode(&map(vec![("store", t("shop")), ("staged", t(ID))])).unwrap(); assert_eq!((bare.name, bare.timeout_ms), (None, None)); - let remove = |fields| RemoveFrame::decode(&map(fields)).unwrap_err().message; - assert_eq!(remove(vec![]), "missing text argument `store`"); - // no project before a bad Store name - assert_eq!( - remove(vec![("store", t("../x"))]), - "missing text argument `project`" - ); - assert!(remove(vec![("store", t("../x")), ("project", t("web"))]).starts_with("store name")); - let stage = |fields| StageFrame::decode(map(fields)).unwrap_err().message; // a bad id before an unknown step assert_eq!( diff --git a/control/crates/layerfs-store/tests/restore_install.rs b/control/crates/layerfs-store/tests/restore_install.rs index 2364f2e8c3..688d62cd02 100644 --- a/control/crates/layerfs-store/tests/restore_install.rs +++ b/control/crates/layerfs-store/tests/restore_install.rs @@ -45,7 +45,6 @@ fn unused(home: &Path, name: &str) -> Result<(), SdkRefusal> { fn related() -> Related { Related { store_idle: unused, - can_remove_project: unused, } } diff --git a/core/crates/layerfs-history/src/contract/catalog.rs b/core/crates/layerfs-history/src/contract/catalog.rs index b543464d0f..da2364410c 100644 --- a/core/crates/layerfs-history/src/contract/catalog.rs +++ b/core/crates/layerfs-history/src/contract/catalog.rs @@ -92,6 +92,17 @@ pub trait HistoryCatalog: Send + Sync { /// Stacks in name order. fn layer_stacks(&self, page: &Page) -> HistoryResult>; + /// The typed `NameTaken` state for a LayerStack (project) name, live or + /// deleted, read without resolving the Stack (spec 4.4). The default + /// reports none; a provider that keeps the collision query overrides it, + /// so `project.init` refuses a held name before it scans anything. + fn layer_stack_name_taken( + &self, + _name: &HistoryName, + ) -> HistoryResult> { + Ok(None) + } + /// The stack named `name`, if any. A provider answers through its name /// index; this default pages through the stacks in name order. fn layer_stack_named(&self, name: &HistoryName) -> HistoryResult> { @@ -114,6 +125,15 @@ pub trait HistoryCatalog: Send + Sync { /// One Branch by identity. fn branch(&self, id: BranchId) -> HistoryResult>; + /// One Branch by identity without the visibility filter (spec 4.2). Only + /// rendering and identity keep a retired Branch's name this way; every + /// caller-facing lookup resolves through [`HistoryCatalog::branch`]. A + /// provider that keeps retired rows overrides this; the default reads the + /// filtered row. + fn branch_any(&self, id: BranchId) -> HistoryResult> { + self.branch(id) + } + /// One Branch by its name in a stack, through the name index: the /// stack's other Branches are not read. fn branch_named( diff --git a/core/crates/layerfs-storage/src/encoding/delta/select.rs b/core/crates/layerfs-storage/src/encoding/delta/select.rs index c293559691..65cadeb583 100644 --- a/core/crates/layerfs-storage/src/encoding/delta/select.rs +++ b/core/crates/layerfs-storage/src/encoding/delta/select.rs @@ -60,6 +60,9 @@ pub struct DeltaCounters { pub absent_candidates: u64, /// Candidates present but ineligible under the depth or role policy. pub ineligible_candidates: u64, + /// Candidates whose stored chain names the object being stored: a + /// concurrent writer's copy depends on it. Stored FULL by policy. + pub cyclic_candidates: u64, /// Candidates refused because the resulting chain would exceed its budget. pub work_exceeded: u64, /// Candidates whose chain already reached the role's depth cap. Their chain @@ -101,6 +104,7 @@ impl DeltaCounters { (&mut self.no_candidate, other.no_candidate), (&mut self.absent_candidates, other.absent_candidates), (&mut self.ineligible_candidates, other.ineligible_candidates), + (&mut self.cyclic_candidates, other.cyclic_candidates), (&mut self.work_exceeded, other.work_exceeded), (&mut self.depth_capped, other.depth_capped), (&mut self.rebased, other.rebased), @@ -154,6 +158,18 @@ pub struct ChainCost { pub encoded: u64, } +/// How a walk of one stored dependency chain ended. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChainWalk { + /// The object is stored and its chain costs this. + Cost(ChainCost), + /// The object, or a base it names, is not stored. + Absent, + /// The chain names the object the caller is storing: that object cannot be + /// its base without closing a dependency cycle. + Cycle, +} + /// Bounded per-save cache of dependency depths and chain costs. #[derive(Debug, Default)] pub struct DepthCache { @@ -205,8 +221,59 @@ impl DepthCache { connection: &dyn Source, workspace: &mut DecompressionWorkspace, id: ObjectId, - mut base_of: F, + base_of: F, ) -> StorageResult> + where + F: FnMut( + &dyn Source, + &mut DecompressionWorkspace, + &ObjectLocation, + ) -> StorageResult, + { + match self.walk(connection, workspace, id, None, base_of)? { + ChainWalk::Cost(cost) => Ok(Some(cost)), + ChainWalk::Absent => Ok(None), + // Without an object to avoid, a walk that names its own start is a + // cycle already stored. + ChainWalk::Cycle => Err(StorageError::Integrity("dependency cycle")), + } + } + + /// `id`'s chain cost, or [`ChainWalk::Cycle`] when the chain names `avoid` + /// (the object the caller is about to store against it). + /// + /// Two writers can store the same new object at once. One writer's copy can + /// name the other writer's object as its base, so the candidate a selection + /// is about to use can name the object that selection is storing. Storing + /// that candidate as the new object's base would close a dependency cycle, + /// so the caller stores the object whole instead. + pub fn cost_avoiding( + &mut self, + connection: &dyn Source, + workspace: &mut DecompressionWorkspace, + id: ObjectId, + avoid: ObjectId, + base_of: F, + ) -> StorageResult + where + F: FnMut( + &dyn Source, + &mut DecompressionWorkspace, + &ObjectLocation, + ) -> StorageResult, + { + self.walk(connection, workspace, id, Some(avoid), base_of) + } + + /// One iterative, bounded walk from `id` to the root of its stored chain. + fn walk( + &mut self, + connection: &dyn Source, + workspace: &mut DecompressionWorkspace, + id: ObjectId, + avoid: Option, + mut base_of: F, + ) -> StorageResult where F: FnMut( &dyn Source, @@ -226,9 +293,18 @@ impl DepthCache { if path.iter().any(|(id, _, _)| *id == current) { return Err(StorageError::Integrity("dependency cycle")); } - if let Some(cost) = self.costs.get(¤t).copied() { - cached_edge = 1; - break cost; + if Some(current) == avoid { + return Ok(ChainWalk::Cycle); + } + // An avoid walk reads the chain from storage: a cost cached + // earlier in this Save can come from a plan the first-wins + // registration later replaced, and only the stored edges decide + // whether this candidate names the object being stored. + if avoid.is_none() { + if let Some(cost) = self.costs.get(¤t).copied() { + cached_edge = 1; + break cost; + } } // Past the first element `current` is a base a stored record names. let location = if path.is_empty() { @@ -237,7 +313,7 @@ impl DepthCache { connection.base_location(current, i64::MAX)? }; let Some(location) = location else { - return Ok(None); + return Ok(ChainWalk::Absent); }; path.push((current, location.canonical_length as u64, 0)); // A chain of depth `d` has `d + 1` records, and the deepest supported @@ -252,6 +328,9 @@ impl DepthCache { last.2 = edge.width; } match edge.base { + // The record names the object being stored: its own copy may not + // be published yet, but the edge is already a cycle-to-be. + Some(base) if Some(base) == avoid => return Ok(ChainWalk::Cycle), Some(base) => current = base, None => { break ChainCost { @@ -291,7 +370,7 @@ impl DepthCache { self.record(*id, level); result = level; } - Ok(Some(result)) + Ok(ChainWalk::Cost(result)) } /// Records the depth and chain cost of one identity. @@ -542,13 +621,13 @@ pub fn select( input.counters.note_depth(role, 0); return Ok(full); } - Some(id) => match probe(input, id, role)? { - true => Some(id), + Some(candidate) => match probe(input, candidate, role, id)? { + true => Some(candidate), false => None, }, None => None, }, - _ => match acquisition(input, role, advisory)? { + _ => match acquisition(input, role, advisory, id)? { Some(id) => Some(id), // A payload no PREFIX record can beat asks the ring nothing: the // answer is FULL, as a trial would find (`prefix_cannot_win`). @@ -558,10 +637,10 @@ pub fn select( input.candidates.ensure_loaded(input.connection)?; let found = input.candidates.propose(id, &sig); match found { - Some(id) => match probe(input, id, role)? { + Some(proposal) => match probe(input, proposal, role, id)? { true => { - from_ring = Some((id, sig)); - Some(id) + from_ring = Some((proposal, sig)); + Some(proposal) } false => None, }, @@ -589,9 +668,21 @@ pub fn select( // A chunk's trial outcome is noted against the base it was offered, which // re-basing may replace (`LosingBases`). let offered = (role == ObjectRole::Chunk).then_some(base_id); - let base_cost = chain_cost(input, base_id)?; + let Some(base_cost) = chain_cost(input, base_id, id)? else { + // The candidate's chain names this object: a concurrent writer already + // stored a copy that depends on it, so using it as a base would close a + // cycle. Store FULL by policy (the concurrent copy keeps the object). + input.counters.cyclic_candidates = input.counters.cyclic_candidates.saturating_add(1); + if lane == PackLane::WholeFile { + let sig = prehashed.unwrap_or_else(|| signature(raw)); + input.candidates.ensure_loaded(input.connection)?; + input.candidates.insert(id, sig); + } + input.counters.note_depth(role, 0); + return Ok(full); + }; let Some((base_id, base_cost)) = - trial_base(input, base_id, base_cost, canonical.len() as u64, depth_cap)? + trial_base(input, base_id, base_cost, canonical.len() as u64, depth_cap, id)? else { return refused_by_budget(input, id, role, raw, prehashed, full); }; @@ -705,18 +796,29 @@ fn refused_by_budget( Ok(full) } -/// Depth and work cost of a stored candidate's chain, from its records' sizes. -fn chain_cost(input: &mut SelectInput<'_>, id: ObjectId) -> StorageResult { +/// Depth and work cost of a stored candidate's chain, from its records' sizes; +/// `None` when the chain names `avoid`, the object being stored. +fn chain_cost( + input: &mut SelectInput<'_>, + id: ObjectId, + avoid: ObjectId, +) -> StorageResult> { let mut bases = ChainBases::new(input.packs); let started = Instant::now(); - let cost = input.depths.cost_of( + let cost = input.depths.cost_avoiding( input.connection, input.decode, id, + avoid, |connection, workspace, location| bases.edge(connection, workspace, location), ); SaveProfile::charge(&mut input.profile.resolve.cost_ns, started); - cost?.ok_or(StorageError::Integrity("selected base is not stored")) + match cost? { + ChainWalk::Cost(cost) => Ok(Some(cost)), + // `probe` admitted the candidate as stored; a dangling base is corrupt. + ChainWalk::Absent => Err(StorageError::Integrity("selected base is not stored")), + ChainWalk::Cycle => Ok(None), + } } /// The base the one trial runs against, and its chain cost (#89). @@ -734,6 +836,7 @@ fn trial_base( cost: ChainCost, target: u64, depth_cap: u8, + avoid: ObjectId, ) -> StorageResult> { let capped = cost.depth >= depth_cap; if capped { @@ -748,7 +851,9 @@ fn trial_base( let root = chain_root(input.connection, input.packs, input.decode, id); SaveProfile::charge(&mut input.profile.resolve.cost_ns, started); let root = root?.object_id; - let root_cost = chain_cost(input, root)?; + let Some(root_cost) = chain_cost(input, root, avoid)? else { + return Ok(None); + }; if root_cost.depth >= depth_cap { return Ok(None); } @@ -804,8 +909,13 @@ fn note_stored(profile: &mut SaveProfile, record: &EncodedRecord) { /// Every rejected candidate is counted, whether it was absent or present but /// ineligible; neither ever becomes a failure of the operation. A candidate at /// the depth cap is eligible: selection re-bases it onto its chain root (#89). -fn probe(input: &mut SelectInput<'_>, id: ObjectId, role: ObjectRole) -> StorageResult { - if eligible(input, id, role)? { +fn probe( + input: &mut SelectInput<'_>, + id: ObjectId, + role: ObjectRole, + avoid: ObjectId, +) -> StorageResult { + if eligible(input, id, role, avoid)? { return Ok(true); } input.counters.ineligible_candidates = input.counters.ineligible_candidates.saturating_add(1); @@ -816,16 +926,22 @@ fn acquisition( input: &mut SelectInput<'_>, role: ObjectRole, advisory: &[ObjectId], + avoid: ObjectId, ) -> StorageResult> { for id in advisory { - if probe(input, *id, role)? { + if probe(input, *id, role, avoid)? { return Ok(Some(*id)); } } Ok(None) } -fn eligible(input: &mut SelectInput<'_>, id: ObjectId, role: ObjectRole) -> StorageResult { +fn eligible( + input: &mut SelectInput<'_>, + id: ObjectId, + role: ObjectRole, + avoid: ObjectId, +) -> StorageResult { let Some(location) = input.connection.location(id, i64::MAX)? else { input.counters.absent_candidates = input.counters.absent_candidates.saturating_add(1); return Ok(false); @@ -835,18 +951,25 @@ fn eligible(input: &mut SelectInput<'_>, id: ObjectId, role: ObjectRole) -> Stor } let mut bases = ChainBases::new(input.packs); let started = Instant::now(); - let depth = input.depths.depth_of( + let walk = input.depths.cost_avoiding( input.connection, input.decode, id, + avoid, |connection, workspace, location| bases.edge(connection, workspace, location), ); SaveProfile::charge(&mut input.profile.resolve.eligible_ns, started); - if depth?.is_none() { - input.counters.absent_candidates = input.counters.absent_candidates.saturating_add(1); - return Ok(false); + match walk? { + ChainWalk::Cost(_) => Ok(true), + ChainWalk::Absent => { + input.counters.absent_candidates = input.counters.absent_candidates.saturating_add(1); + Ok(false) + } + ChainWalk::Cycle => { + input.counters.cyclic_candidates = input.counters.cyclic_candidates.saturating_add(1); + Ok(false) + } } - Ok(true) } fn acquire(input: &mut SelectInput<'_>, id: ObjectId) -> StorageResult> { diff --git a/core/crates/layerfs-storage/src/save/pooled.rs b/core/crates/layerfs-storage/src/save/pooled.rs index f5b01ec28a..36a86ae156 100644 --- a/core/crates/layerfs-storage/src/save/pooled.rs +++ b/core/crates/layerfs-storage/src/save/pooled.rs @@ -9,6 +9,7 @@ use super::{source::WaveSource, state::State}; use crate::{ encoding::{ + delta::select::ChainWalk, pool::{delta, leaf}, EncodedRecord, }, @@ -57,7 +58,7 @@ impl State<'_> { ) -> StorageResult { let input = PoolingLeaf::decode(object.canonical())?; let mut known = BTreeMap::<[u8; INODE_VALUE_BYTES], u32>::new(); - let bases = self.pooled_bases(advisory, &mut known)?; + let bases = self.pooled_bases(object.id(), advisory, &mut known)?; // Values no base holds may still be ones this Save already assigned, // which is the common case while a whole tree is first saved. let mut candidates = BTreeSet::new(); @@ -180,6 +181,7 @@ impl State<'_> { /// another delta still gives its rows their ordinals. fn pooled_bases( &mut self, + target: ObjectId, advisory: &[ObjectId], known: &mut BTreeMap<[u8; INODE_VALUE_BYTES], u32>, ) -> StorageResult> { @@ -201,10 +203,11 @@ impl State<'_> { let reader = &mut self.pool; let groups = &mut self.groups; let packs = &mut self.packs; - let Some(cost) = self.depths.cost_of( + let walk = self.depths.cost_avoiding( &source, &mut self.decode, *id, + target, |source, workspace, location| { // The pool reader keeps the encoded account of a pooled chain // (`chain_encoded` below), so the walk measures no width. @@ -220,8 +223,10 @@ impl State<'_> { ) .map(|base| crate::encoding::delta::read::ChainEdge { base, width: 0 }) }, - )? - else { + )?; + let ChainWalk::Cost(cost) = walk else { + // A base whose chain names this leaf would close a cycle; an + // absent or dangling one cannot carry it. continue; }; let (body, canonical) = self.pool.leaf_parts_with_caches( diff --git a/core/docs/issues/project-41/gc-acceptance-matrix.md b/core/docs/issues/project-41/gc-acceptance-matrix.md index 70ad3d7a43..e462bf967e 100644 --- a/core/docs/issues/project-41/gc-acceptance-matrix.md +++ b/core/docs/issues/project-41/gc-acceptance-matrix.md @@ -1,14 +1,36 @@ # GC acceptance matrix: #52 evidence plan for the §12 cases > Draft matrix for the frozen `LayerFS-GC-spec.md` §12 (the spec wins; this -> page adds no contract). Main layer `32d78561a`. AC-01..AC-57 are the §12 -> bullets one-to-one, in spec order; AC-58..AC-60 pin sub-steps of §7/§8 that -> §12 folds into broader cases. Each row's expected result restates spec text -> only; a mismatch is a spec bug, not a matrix change. +> page adds no contract). The plan was written at main layer `32d78561a`; +> statuses below are re-checked on main `32ae51ca` (T-G75) and reconciled +> against the reports published on main: T12.3c at `3282f9f6865` regenerated +> [`gc-t12.3a-sqlite-matrix-report.md`](gc-t12.3a-sqlite-matrix-report.md) +> (executed 8 / covered 29 / partial 16 / deferred 7 / SQLite gaps 0), beside +> T12.3a/T12.3b/T9/T10/T-G6a; T-G28 added the real-process SIGKILL arms and +> moved AC-01 to `partial`; T-G60 extended them to the mark, copy, verify and +> rename points; T-G38 and T-G54 supplied the real PostgreSQL +> reply-loss and restricted-role privilege arms and moved AC-44 to `covered`; +> T-G65 killed the real `layerfs store compact` process itself at the mark, +> build, verify and install points and moved AC-01 back to `covered`. +> The final counts are executed 8 / covered 30 / partial 16 / deferred 6; +> T-G75 re-ran the SQLite suites from `32ae51ca` and re-checked all 60 rows. +> AC-01..AC-57 are the §12 bullets one-to-one, in +> spec order; AC-58..AC-60 pin sub-steps of §7/§8 that §12 folds into broader +> cases. Each row's expected result restates spec text only; a mismatch is a +> spec bug, not a matrix change. > -> Status vocabulary: `open` (no §12-conforming evidence recorded yet), -> `pass`, `fail`, `waived`. §13 says nothing has been measured: every E3 row -> starts `open`. Tiers are this page's evidence scale, not contract. +> Status vocabulary: `executed` (a T12.3c runner arm really executes the +> case), `covered` (a named existing test or fixture covers it), `partial` +> (some named arms proven; the evidence cell says what remains), `deferred` (a +> PostgreSQL-only row the SQLite runner leaves to T12.3b by name), `pass` (the +> named arm passed in the recorded run), `pending` (no §12-conforming evidence +> recorded yet; none remain). The dispositions +> are the SQLite half; the PostgreSQL half is T12.3b and its evidence stays +> named below. T-G66 measured the 50 GB `none` and `rebuild` arms (one run each); +> §13's measurement caveat now applies to the 30% dead and 50 GB-live arms, and +> an E3 row records the three-run or property evidence that exists and stays +> `partial` where it does not. +> Tiers are this page's evidence scale, not contract. ## Evidence tiers (E0-E3) @@ -21,68 +43,123 @@ ## Matrix -| Case ID | Contract (spec §) | Backend | Fixture / command | Expected result | Tier | Status | -|---|---|---|---|---|---|---| -| AC-01 | §12 Crash injection; §9 | both | Store with one wholly dead pack; `kill -9` the daemon at mark, build, verify, before and after install; next `store.compact` | Store is wholly old or wholly new; the leftover is judged by §9; generation and digest asserted | E2 | open | -| AC-02 | §12 Backend matrix; §4.1 | both | Run every removal, admission, report and name case on SQLite and PostgreSQL; open a converted 1.0 Store on both | Each case's §12 result holds on both backends; the additive step adds the §4.1 tables on both | E2 | open | -| AC-03 | §12 Crash matrix; §9 | both | Named failpoints: SQLite mark, build, verify, install, WAL switch, artifact removal, op settle; PG mark, build, swap, cleanup, scratch drop; plus lost COMMIT, power loss (fsync/rename) and unresolved op/lease takeover | §9 state, generation, digest and closure hold after resume; only a live owner blocks; an unreachable machine reports unknown, never clean | E2 | open | -| AC-04 | §12 No live object lost; §6.1-§6.2 | both | Property test over random fork, commit, merge, publish, `branch.remove`, `project.delete` and `remove_file` histories; compare closure sets before and after | No live object reaches a collected row; the retained closure is unchanged | E3 | open | -| AC-05 | §12 Independent walk; §9 | both | After compaction, walk the new Store with `diagnostic/scan.rs` and run the full doctor on the fixtures (#155 D-23) | Walk and doctor pass on the compacted Store; no dangling reference | E2 | open | -| AC-06 | §12 Regression; §6.4 | both | After compaction, Save bytes whose `file_digest` row named a collected root | The Save succeeds; no stale digest row blocks it | E2 | open | -| AC-07 | §12 Fences; §10 | both | A connection that bypassed admission; a second compaction; a remote PG daemon holding a mount; a one-shot SDK transaction open at swap time | Bypass gets `StaleStore`; second compaction gets `Busy`; the mount blocks compaction and the matching removal, naming itself; the open transaction makes the swap abandon | E2 | open | -| AC-08 | §12 Admission races; §5.1 | both | Start a mount and a removal of the same Branch together | Never both succeed; the loser gets `MountsActive` or `Busy` | E2 | open | -| AC-09 | §12 Names; §4.4 | both | `NameTaken` after removal (Branch and project); create collision (Branch and project; project create was `Exists`); free name after a rebuild collects the row; `ReservedName` for `main` | Exact codes per §4.4; only a rebuild frees an unreferenced name | E2 | open | -| AC-10 | §12 remove_file; §4.3 | both | `Removed` for Commits at or below the frontier, before and after compaction; the next Commit of a mounted older head records the deletion; a file created later at the path is visible | Codes and visibility per §4.3; 2.0 frees no bytes for the removed path | E2 | open | -| AC-11 | §12 remove_file and derived nodes; §4.3 | both | Non-FF `branch.merge` of two heads at or below the frontier; a post-frontier re-creation; a FF merge control; `branch.publish` of an at-or-below Commit; a merge that cannot apply the whiteout | Path stays hidden, whiteout materialized; post-frontier file visible; FF merge creates no tree; published Layer hidden, its mount whiteouts, next Commit records the deletion; inapplicable whiteout refused `Removed` | E2 | open | -| AC-12 | §12 Removed path in a merge; §4.3 | both | Non-FF merge of two at-or-below heads; merge of an at-or-below head with a post-frontier re-creation | No conflict for the removed path, it is omitted; the re-created file is kept and only an ordinary edit conflict on it is reported | E2 | open | -| AC-13 | §12 Unmerged Branches; §4.3 | both | `remove_file` then a Commit on an unmerged Branch created before the frontier; another Commit after the frontier | The earlier Commit's path is hidden, the later one does not inherit it | E2 | open | -| AC-14 | §12 Layer-headed mount; §4.3 | both | Branch whose head is empty and whose base Layer is at or below the frontier; mount it | The path is whiteouted at mount | E2 | open | -| AC-15 | §12 Visibility only; §6.2 | both | Object reachable only through the removed path; marker hiding reads/listings; mount, merge and publish whiteouts | No bytes freed; the object is retained; reads/listings hide the path; derived trees whiteout it | E2 | open | -| AC-16 | §12 Published Branch; §4.4 | both | Branch published by a retained Layer; Branch named by a retained `commit.branch_id`; `branch.remove` then rebuild | Name stays taken; only an unreferenced Branch's name is freed by a rebuild (not by a `none` pass) | E2 | open | -| AC-17 | §12 Late base; §7 step 4 | both | Delta base in an already-passed pack; a depth-2 chain | Worklist copies both; `pack_id` preserved | E2 | open | -| AC-18 | §12 Digest roots; §6.4 | both | Uncommitted saved file's `file_digest` row; another retained root reaching its content | Row collected unless the retained root reaches the content | E2 | open | -| AC-19 | §12 Path grammar; §4.3 | both | `/a` and `a`; empty, `/`, `//a`, `a/`, `a//b`, `./a`, `a/./b`, dot and dot-dot components, backslash, NUL, >4096 bytes, >256 components; directory subtree; child of a removed non-directory; symlink leaf and intermediate component; hardlink | Accepted paths hide correctly; bad forms are `InvalidInput`; directory hides its subtree; child of removed non-directory `NotFound`; intermediate symlink `InvalidInput`, leaf hidden; hardlink survives | E2 | open | -| AC-20 | §12 Repeated removal; §4.3 | both | Second `remove_file` after a re-creation; repeat with no newer project node; a commit in another project; never-existing path | Re-creation hidden; repeat `UpToDate`; other project neither rewrites nor advances the marker; never-existing path accepted, hides nothing | E2 | open | -| AC-21 | §12 Marker epoch; §5 | both | One-shot write racing `branch.remove` or `project.delete`; merge racing `remove_file` | `Conflict`, no Commit left; retry applies the whiteout | E2 | open | -| AC-22 | §12 Concurrent merge and removal (property); §4.3 | both | Property test: a merge and a `remove_file` started together | The path is never left visible in a derived tree; one side wins, the other sees the marker or gets `Conflict` and retries with the whiteout | E3 | open | -| AC-23 | §12 Marker cleanup; §6.4 | both | Rebuild of a Store with a deleted project and a live project with markers; a `none` pass | No `path_removed` row for the deleted project; live markers kept; `none` writes nothing and retains them | E2 | open | -| AC-24 | §12 Position state; §4.1 | both | Converted Store; `store.compact`; a marker write | Serials backfilled; `next_history_serial` never falls; `project_epoch` copied and bumped only on a marker write | E2 | open | -| AC-25 | §12 Marker survival; §4.4 | both | Retained `branch_retired`, `layer_stack_retired` and `path_removed`; a collected base row | Retained markers survive a rebuild; a collected row takes its marker with it | E2 | open | -| AC-26 | §12 project.delete retry; §4 | both | Marker commits and home removal fails; same-home retry; own-Init identity save failure | Hidden, entry present, `home_entry_removed: false`; retry finishes and returns the stored `deleted_ms`, no second marker; the identity save precedes the marker, a save failure removes nothing | E2 | open | -| AC-27 | §12 Home listing; §4.2 | both | Second home's stale entry; `project.list` and `project.get`; Store calls | Stale entry absent from `project.list`, `project.get` answers `NoSuchProject`; Store calls agree | E2 | open | -| AC-28 | §12 PG cleanup; §8 Phase C | PG | Commit/save during Phase C; crash mid-cleanup; failpoint at the marker/base-row batch boundary | The late write survives the high-water bound; cleanup resumes from `gc_op`; marker and base row deleted in one batch, no resurrection or orphan | E2 | open | -| AC-29 | §12 Admission; §5.1 | both | Mount from another home or machine; `machine_stopped` evidence; a second mount of one project; an uncertain dead op | The mount is visible in the Store-side intent; `machine_stopped` ignored; the second mount proceeds; `Unresolved` observed and settled by the owning call | E2 | open | -| AC-30 | §12 NameTaken; §4.4 | both | `NameTaken` detail for live, retired and deleted holders; forced two-reference fixture (Layer FK and `commit.branch_id`); one-reference and no-reference controls; error matrix | Detail carries holder kind, name and state; two references keep the name, the controls do not; matrix covers `MountsActive`, `Busy`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NameTaken`, `ReservedName`, `Removed`, `InvalidInput` | E2 | open | -| AC-31 | §12 Trace salt; §4.3 | both | Call log with path arguments; `Removed` refusal; rebuild | Log stores ``, trace key is the keyed hash; `Removed` echoes the path; the salt is copied by the rebuild | E2 | open | -| AC-32 | §12 Admission (liveness); §5.1 | both | Live-but-unreachable Sandbox; `machine_stopped`; `gc_intent` from another home; non-owning call | Live-unreachable is `Busy`; `machine_stopped` ignored; the other home's intent visible; `Unresolved` observed by the non-owner and settled by the owner | E2 | open | -| AC-33 | §12 Checkpoint/install; §7 steps 1, 7 | SQLite | Live reader during `wal_checkpoint(TRUNCATE)`; a sidecar or `-mj*` present at install | `busy>0` makes the checkpoint `Busy` with no artifact and no op; a sidecar at install is `Conflict` with no writes | E2 | open | -| AC-34 | §12 Bind failure; §5.1 | both | Committed intent whose bind fails; failed intent delete | Intent deleted in the error path; the failed delete is reported; no `MountsActive` remains | E2 | open | -| AC-35 | §12 PG refcount; §5.2 | PG | Two mounts, one unmount; failed all-or-none acquire | Store and project advisory keys stay after one unmount; the failed acquire unlocks the taken subset | E2 | open | -| AC-36 | §12 Worker/lost reply; §4, §9 | both | Unreachable worker op; lost reply; retry | Op is `unknown`, never clean; the sidecar is kept; the retry resolves retired, live or absent | E2 | open | -| AC-37 | §12 Trace; §4.3 | both | Six path names; refusal details; upgraded Store | The six path names are redacted; refusal details are not traced; the salt is backfilled | E2 | open | -| AC-38 | §12 Artifacts (cross-backend); §9 | both | SQLite artifact set `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; PG `lfs_gc_` schema; a stray schema | SQLite leaves and removes the full set; PG records the name before CREATE, a remote machine can remove it, a stray schema is `Exists` | E2 | open | -| AC-39 | §12 Position state (2.0 backfill); §4.1 | both | Every upgraded Store (2.0 included); `gc_op` settle; serial allocation | One `project_epoch` row per stack; `gc_op` cleared on settle and not copied; `next_history_serial` never lowered | E2 | open | -| AC-40 | §12 Digest agreement; §6.4 | both | Same fixture on SQLite and PostgreSQL; retired head | Same `live_heads_digest`; a retired head changes it | E2 | open | -| AC-41 | §12 PostgreSQL mark; §8 Phase M | PG | Mark transfer; table copy; a base object held only by a dead pack | Only `markBytes` crosses the wire; the copy moves no pack bytes; the base is kept | E2 | open | -| AC-42 | §12 Reserved pack; §8 Phase B | PG | Save reserves a pack/ordinal before the mark and publishes during Phase A (or C); cleanup failpoint | `store_totals` moves; the swap `Conflict`s; the pack stays in the old Store; the failpoint keeps it; no dangling location | E2 | open | -| AC-43 | §12 PG cleanup (resume); §8 Phase C | PG | After the swap at `G+1`; crash mid-cleanup; totals, ACL, trigger, hints | Store usable at `G+1`; cleanup resumes; totals/ACL/trigger/hints asserted; no `inode` on PG | E2 | open | -| AC-44 | §12 PG privileges and reply; §8 | PG | Missing `CREATE`, `TEMP` or `pg_read_all_stats`; lost `COMMIT` reply | `Unsupported`; the lost reply is resolved by proving the backend ended, then reading the incarnation | E2 | open | -| AC-45 | §12 SQLite layouts; §7 step 2 | SQLite | Schema 36 (`Monolithic`), 37 (`GroupRows`), 38 (`GroupRowsIndexed`) and their 42/43/44 acquisition variants rebuild | `Monolithic` copies `pack` body and no `pack_unit`; `GroupRows` copies both; `GroupRowsIndexed` copies `pack_unit` and the schema creates its index; `sqlite_sequence` kept when present; `init_*`/acquisition tables empty | E1 | open | -| AC-46 | §12 Install fence; §10 | SQLite | Connection that bypassed the writer lane opening a transaction after install; in-flight transaction; sealed-state or sidecar change at install | `StaleStore`; the in-flight transaction is excluded by the lane (documented limit); a sealed-state or sidecar change is `Conflict` with no writes | E2 | open | -| AC-47 | §12 Artifacts (SQLite exact); §9 | SQLite | Crash mid-compact; next compact; `none` pass; stray name with no op | Crash leaves the full `.compact` set; the next compact removes all of it; `none` leaves no `.compact`; a stray name is `Exists` | E2 | open | -| AC-48 | §12 Threshold and failures; §3 | both | 9%, 10% and 30% wholly dead; empty Store (0/0); started rebuild whose refined share falls below 10%; `Capacity`; verify failure | 9% is `none`, 10% and 30% are `rebuild`; empty Store is `none` with `zeroReason: empty-store`; the started rebuild completes; `Capacity` and verify failure abort, delete the artifact set and leave the old Store unchanged apart from step 1's checkpoint | E2 | open | -| AC-49 | §12 Mount/compact race; §5, §8 | both | Mount during compact; mount wins the race; Commit during the rebuild | `Busy` with the compact op; compaction returns `MountsActive` when the mount wins; the Commit is not lost (PG Phase A write makes the swap `Conflict`; Phase C survives the high-water bound) | E2 | open | -| AC-50 | §12 Profile and scope; §10 | both | All four calls under a folder scope; MCP admin profile | Refused `out of scope`; allowed only in the admin profile | E2 | open | -| AC-51 | §12 Progress; §11 | both | `store.compact` running; `store.status` | `store.status` shows the op phase: SQLite `mark`, `build`, `verify`, `install`; PG `mark`, `build`, `swap`, `cleanup` | E2 | open | -| AC-52 | §12 Report; §11 | both | `none` and `rebuild` paths; every §11 field | `kind: Compacted`, snake_case keys, every §11 field with its unit and sampling point (`freelist_count * page_size`, old size before rename, history `logical` delta, peak temp, `physicalShrink`, `keptWaste`, `residue`, `estimate`, capped `cap`, `zeroReason`); `markBytes` is a #52 metric, not a result field; bound at least refined; PG `backendReusable` = 0 and PG/none `deadBytesRefined` equals `deadBytesBound` | E2 | open | -| AC-53 | §12 Read surface; §4.3 | both | `file.read`/`stat`/`list`/`glob`/`diff`/`revision`/`export`/cp-from before and after compaction; `edit`/`grep`/`blame`/`rm`/`mv`/`mkdir`/`cp`/`apply`; `branch.diff`/`commit.diff` | §4.3 shapes; visible/hidden/absent diff table; `removed_count` 0 and >0; one whiteout case per one-shot tree creator | E2 | open | -| AC-54 | §12 Restore and listing; §4.2 | both | `backup.restore` before and after a removal; second home's stale entry | Restore succeeds; the stale entry stays out; the §4.2 guard test and allow-list hold | E2 | open | -| AC-55 | §12 Error matrix; §10 | both | Exact code and message per case, including `StaleStore`, `Removed`, `InvalidInput` for `project.remove`, `Exists`, `Unsupported`, `Capacity` | Exact code and message; snake_case details | E2 | open | -| AC-56 | §12 Digest golden vector; §6.4 | both | Fixed-id fixture (out-of-order ids, a retired Branch and stack, differing timestamps); absent head | One literal vector on both backends; absent head is 33 zero bytes, not 32; widths fixed at 17/33 | E3 | open | -| AC-57 | §12 Speed and space; §13 | both | At least 3 runs per arm, median and range, declared caches, generated fixtures; §13 arms: `none` at 9%, `rebuild` at 10% and 30%, 50 GB total/35 GB live, 50 GB live, each removal type, removal-then-compact, `remove_file` 0-freed | Downtime per GB live and per GB freed, bytes written, peak disk, PG `ACCESS EXCLUSIVE` duration; absolute bytes when nothing is freed; build phase vs plain `VACUUM INTO`; mark reported separately; a 50% time/5% space option rejected | E3 | open | -| AC-58 | §7 step 1.5 (refined decision) | SQLite | After the seal: `ATTACH` the old file `immutable=1`, create `.compact-visited`, run the complete §6.1+§6.2 closure, choose `none`/`rebuild` from the refined live set | `none` deletes the scratch and stops with no `.compact` and no residue; `rebuild` keeps the scratch, reuses `visited` in step 4; step 2 creates the new file only on `rebuild` | E2 | open | -| AC-59 | §7 step 1 (seal) | SQLite | `PRAGMA wal_checkpoint(TRUNCATE)`; checked-close the last session; sidecar check under the writer guard; `seal_holding()` | `busy == 0` required; WAL never unlinked; sealed `(size, mtime, db-header change counter)` recorded before release; the guard passes into the seal | E2 | open | -| AC-60 | §8 Phase B step 10 (standby) | PG | Effective `synchronous_commit` at least `remote_write` with `synchronous_standby_names` set; standby missing before Phase M; standby missing inside the swap; standby dies after the check | Wait only when required; `FIRST`/`ANY` standbys must be `state=streaming` with matching `sync_state`; a missing standby abandons before COMMIT with no writes (`Busy`, standby list); a standby dying after the check blocks COMMIT | E2 | open | +| Case ID | Contract (spec §) | Backend | Fixture / command | Expected result | Tier | Status | Evidence | +|---|---|---|---|---|---|---|---| +| AC-01 | §12 Crash injection; §9 | both | Store with one wholly dead pack; `kill -9` the daemon at mark, build, verify, before and after install; next `store.compact` | Store is wholly old or wholly new; the leftover is judged by §9; generation and digest asserted | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_before_seal`..`#crash_after_install` (in-process failpoints); T-G28/T-G60 `sqlite_compact_crash.rs::real_process` `#sigkill_before_seal`, `#sigkill_after_seal` (mark), `#sigkill_during_copy` (copy; the gate is immediately before `VACUUM INTO`, because SQLite's copy cannot be interrupted mid-call), `#sigkill_during_verify` (verify), `#sigkill_during_rename`, `#sigkill_before_install` and `#sigkill_after_install` (install), `#sigkill_after_build` (build) and `#sigkill_during_settle` (settle) - 9 re-executed-test-child SIGKILL arms; T-G65 `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` `#sigkill_before_seal`, `#sigkill_after_seal` (mark), `#sigkill_before_vacuum`, `#sigkill_after_build` (build), `#sigkill_during_verify`, `#sigkill_before_install` (verify), `#sigkill_after_install`, `#sigkill_during_settle` (install) - 8 SIGKILL arms that start the built `layerfs store compact` command, wait through the engine's socket gate and kill that process (`status.signal() == 9`) at a Store with one wholly dead pack, then reopen through the CLI and assert `G`/`G + 1`, the artifact set and killed-session sidecars, the `store.status` phase/`uncertain`/`unresolved`, the next compact settling with empty residue, the retained layer readable and `doctor` clean; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover` (in-process, no `kill -9`) - `covered` for SQLite because the killed process is the production process that performs the compaction (no daemon serves Store calls, #352 F-020); the residual is that the long-lived `layerfs worker` serving a machine's Store is still not exercised, and the PostgreSQL half remains T12.3b's | +| AC-02 | §12 Backend matrix; §4.1 | both | Run every removal, admission, report and name case on SQLite and PostgreSQL; open a converted 1.0 Store on both | Each case's §12 result holds on both backends; the additive step adds the §4.1 tables on both | E2 | partial | T12.3a `sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away`; T12.3b `postgresql_position_columns` 4/4 (converted 1.0 Store arm is SQLite-only) | +| AC-03 | §12 Crash matrix; §9 | both | Named failpoints: SQLite mark, build, verify, install, WAL switch, artifact removal, op settle; PG mark, build, swap, cleanup, scratch drop; plus lost COMMIT, power loss (fsync/rename) and unresolved op/lease takeover | §9 state, generation, digest and closure hold after resume; only a live owner blocks; an unreachable machine reports unknown, never clean | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_after_wal_switch` + mark/build/verify/install failpoints; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover`; lost COMMIT/power loss deferred | +| AC-04 | §12 No live object lost; §6.1-§6.2 | both | Property test over random fork, commit, merge, publish, `branch.remove`, `project.delete` and `remove_file` histories; compare closure sets before and after | No live object reaches a collected row; the retained closure is unchanged | E3 | partial | T12.1 `sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript`; T12.3a `sqlite_compact_matrix.rs#arm_random_store_shapes` (seeded; full random generator deferred) | +| AC-05 | §12 Independent walk; §9 | both | After compaction, walk the new Store with `diagnostic/scan.rs` and run the full doctor on the fixtures (#155 D-23) | Walk and doctor pass on the compacted Store; no dangling reference | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql`; `postgresql_diagnostic` 5/5 | +| AC-06 | §12 Regression; §6.4 | both | After compaction, Save bytes whose `file_digest` row named a collected root | The Save succeeds; no stale digest row blocks it | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_digest_row_after_collection` - a Save whose `file_digest` row named a root the rebuild collected; the row goes with its root, recording the same content again succeeds and points at the new root; PostgreSQL half T12.3b | +| AC-07 | §12 Fences; §10 | both | A connection that bypassed admission; a second compaction; a remote PG daemon holding a mount; a one-shot SDK transaction open at swap time | Bypass gets `StaleStore`; second compaction gets `Busy`; the mount blocks compaction and the matching removal, naming itself; the open transaction makes the swap abandon | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_mount_and_compaction_exclude_each_other`; `sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction` - bypass gives `StaleStore`; a live mount makes compaction `MountsActive` naming it; a live compaction refuses a mount and a second compaction `BusyOp`, leaving no op; the open one-shot SDK transaction arm is PostgreSQL (T12.3b) | +| AC-08 | §12 Admission races; §5.1 | both | Start a mount and a removal of the same Branch together | Never both succeed; the loser gets `MountsActive` or `Busy` | E2 | covered | T12.3a `admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner` (SQLite half; PG arm not recorded) | +| AC-09 | §12 Names; §4.4 | both | `NameTaken` after removal (Branch and project); create collision (Branch and project; project create was `Exists`); free name after a rebuild collects the row; `ReservedName` for `main` | Exact codes per §4.4; only a rebuild frees an unreferenced name | E2 | covered | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; `gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches` (SQLite half) | +| AC-10 | §12 remove_file; §4.3 | both | `Removed` for Commits at or below the frontier, before and after compaction; the next Commit of a mounted older head records the deletion; a file created later at the path is visible | Codes and visibility per §4.3; 2.0 frees no bytes for the removed path | E2 | covered | T12.3a `path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position` (SQLite half) | +| AC-11 | §12 remove_file and derived nodes; §4.3 | both | Non-FF `branch.merge` of two heads at or below the frontier; a post-frontier re-creation; a FF merge control; `branch.publish` of an at-or-below Commit; a merge that cannot apply the whiteout | Path stays hidden, whiteout materialized; post-frontier file visible; FF merge creates no tree; published Layer hidden, its mount whiteouts, next Commit records the deletion; inapplicable whiteout refused `Removed` | E2 | partial | T12.3c `sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent`; `sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation`; `sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden`; `sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount` - non-FF whiteouts, the FF control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute; partial because a merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed | +| AC-12 | §12 Removed path in a merge; §4.3 | both | Non-FF merge of two at-or-below heads; merge of an at-or-below head with a post-frontier re-creation | No conflict for the removed path, it is omitted; the re-created file is kept and only an ordinary edit conflict on it is reported | E2 | covered | T12.3c `sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent`; `sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation`; `sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit` - a removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept and only an ordinary edit conflict on it is reported | +| AC-13 | §12 Unmerged Branches; §4.3 | both | `remove_file` then a Commit on an unmerged Branch created before the frontier; another Commit after the frontier | The earlier Commit's path is hidden, the later one does not inherit it | E2 | covered | T12.3c `sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit` - the earlier Commit on an unmerged Branch created before the frontier is hidden by the marker; a Commit after the frontier does not inherit the path | +| AC-14 | §12 Layer-headed mount; §4.3 | both | Branch whose head is empty and whose base Layer is at or below the frontier; mount it | The path is whiteouted at mount | E2 | covered | T12.3c `sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount`; `sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes` - a Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount, and its next Commit records the deletion | +| AC-15 | §12 Visibility only; §6.2 | both | Object reachable only through the removed path; marker hiding reads/listings; mount, merge and publish whiteouts | No bytes freed; the object is retained; reads/listings hide the path; derived trees whiteout it | E2 | partial | T12.3a `kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs`; T12.1 fixture (read/listing hiding and derived whiteouts not one test) | +| AC-16 | §12 Published Branch; §4.4 | both | Branch published by a retained Layer; Branch named by a retained `commit.branch_id`; `branch.remove` then rebuild | Name stays taken; only an unreferenced Branch's name is freed by a rebuild (not by a `none` pass) | E2 | covered | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; T12.1 fixture (SQLite half) | +| AC-17 | §12 Late base; §7 step 4 | both | Delta base in an already-passed pack; a depth-2 chain | Worklist copies both; `pack_id` preserved | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` (depth-2 chain, `pack_id` kept) | +| AC-18 | §12 Digest roots; §6.4 | both | Uncommitted saved file's `file_digest` row; another retained root reaching its content | Row collected unless the retained root reaches the content | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-19 | §12 Path grammar; §4.3 | both | `/a` and `a`; empty, `/`, `//a`, `a/`, `a//b`, `./a`, `a/./b`, dot and dot-dot components, backslash, NUL, >4096 bytes, >256 components; directory subtree; child of a removed non-directory; symlink leaf and intermediate component; hardlink | Accepted paths hide correctly; bad forms are `InvalidInput`; directory hides its subtree; child of removed non-directory `NotFound`; intermediate symlink `InvalidInput`, leaf hidden; hardlink survives | E2 | partial | T12.3a `gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers`; `path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position` (full form table missing) | +| AC-20 | §12 Repeated removal; §4.3 | both | Second `remove_file` after a re-creation; repeat with no newer project node; a commit in another project; never-existing path | Re-creation hidden; repeat `UpToDate`; other project neither rewrites nor advances the marker; never-existing path accepted, hides nothing | E2 | partial | T12.3a `path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date`; `#an_unknown_project_is_missing` (re-creation/other-project arms partial) | +| AC-21 | §12 Marker epoch; §5 | both | One-shot write racing `branch.remove` or `project.delete`; merge racing `remove_file` | `Conflict`, no Commit left; retry applies the whiteout | E2 | partial | T12.3a `epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back`; T12.3b `postgresql_compact_fixtures.rs#a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write` (true concurrent PG trigger arm missing) | +| AC-22 | §12 Concurrent merge and removal (property); §4.3 | both | Property test: a merge and a `remove_file` started together | The path is never left visible in a derived tree; one side wins, the other sees the marker or gets `Conflict` and retries with the whiteout | E3 | covered | T12.3c `sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible`; `epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back` - a deterministic barrier starts a merge and a `remove_file` together for three rounds; the loser sees the marker or gets `Conflict` and retries with the whiteout, and a derived tree never shows the path; PostgreSQL half T12.3b | +| AC-23 | §12 Marker cleanup; §6.4 | both | Rebuild of a Store with a deleted project and a live project with markers; a `none` pass | No `path_removed` row for the deleted project; live markers kept; `none` writes nothing and retains them | E2 | partial | T12.3a `project_delete.rs#deleting_a_project_hides_it_and_marks_once`; T12.1 fixture (dedicated marker-cleanup arm missing) | +| AC-24 | §12 Position state; §4.1 | both | Converted Store; `store.compact`; a marker write | Serials backfilled; `next_history_serial` never falls; `project_epoch` copied and bumped only on a marker write | E2 | covered | T12.3a `gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order`; T12.3b `postgresql_position_columns` + fixture | +| AC-25 | §12 Marker survival; §4.4 | both | Retained `branch_retired`, `layer_stack_retired` and `path_removed`; a collected base row | Retained markers survive a rebuild; a collected row takes its marker with it | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-26 | §12 project.delete retry; §4 | both | Marker commits and home removal fails; same-home retry; own-Init identity save failure | Hidden, entry present, `home_entry_removed: false`; retry finishes and returns the stored `deleted_ms`, no second marker; the identity save precedes the marker, a save failure removes nothing | E2 | covered | T12.3a `project_delete.rs#retry_classifies_retired_live_and_absent`; `#deleting_a_project_hides_it_and_marks_once` (SQLite half) | +| AC-27 | §12 Home listing; §4.2 | both | Second home's stale entry; `project.list` and `project.get`; Store calls | Stale entry absent from `project.list`, `project.get` answers `NoSuchProject`; Store calls agree | E2 | covered | T12.3a `project_delete.rs#paging_skips_a_hidden_project`; `#every_layer_stack_lookup_carries_the_marker_filter` (SQLite half) | +| AC-28 | §12 PG cleanup; §8 Phase C | PG | Commit/save during Phase C; crash mid-cleanup; failpoint at the marker/base-row batch boundary | The late write survives the high-water bound; cleanup resumes from `gc_op`; marker and base row deleted in one batch, no resurrection or orphan | E2 | deferred | T12.3b `postgresql_compact_fixtures.rs#a_save_after_the_swap_survives_the_resumed_cleanup`; `#every_postgresql_failpoint_leaves_a_resumable_leftover` (marker/base-row batch failpoint still missing) | +| AC-29 | §12 Admission; §5.1 | both | Mount from another home or machine; `machine_stopped` evidence; a second mount of one project; an uncertain dead op | The mount is visible in the Store-side intent; `machine_stopped` ignored; the second mount proceeds; `Unresolved` observed and settled by the owning call | E2 | covered | T12.3a `admission.rs#a_mount_blocks_all_three_removals_until_released`; T12.3b `postgresql_compact.rs` owner-adoption tests (mount-work admission deferred) | +| AC-30 | §12 NameTaken; §4.4 | both | `NameTaken` detail for live, retired and deleted holders; forced two-reference fixture (Layer FK and `commit.branch_id`); one-reference and no-reference controls; error matrix | Detail carries holder kind, name and state; two references keep the name, the controls do not; matrix covers `MountsActive`, `Busy`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NameTaken`, `ReservedName`, `Removed`, `InvalidInput` | E2 | partial | T12.3a `branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names`; `gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect` (PG detail matrix deferred) | +| AC-31 | §12 Trace salt; §4.3 | both | Call log with path arguments; `Removed` refusal; rebuild | Log stores ``, trace key is the keyed hash; `Removed` echoes the path; the salt is copied by the rebuild | E2 | covered | T12.3a `gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log`; `#the_trace_key_is_salted_by_the_store_policy_salt` (PG salt backfill deferred) | +| AC-32 | §12 Admission (liveness); §5.1 | both | Live-but-unreachable Sandbox; `machine_stopped`; `gc_intent` from another home; non-owning call | Live-unreachable is `Busy`; `machine_stopped` ignored; the other home's intent visible; `Unresolved` observed by the non-owner and settled by the owner | E2 | partial | T12.3a `admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved`; T12.3b owner-adoption tests (other-home intent deferred) | +| AC-33 | §12 Checkpoint/install; §7 steps 1, 7 | SQLite | Live reader during `wal_checkpoint(TRUNCATE)`; a sidecar or `-mj*` present at install | `busy>0` makes the checkpoint `Busy` with no artifact and no op; a sidecar at install is `Conflict` with no writes | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_checkpoint_busy`; `#arm_sidecar_at_install` | +| AC-34 | §12 Bind failure; §5.1 | both | Committed intent whose bind fails; failed intent delete | Intent deleted in the error path; the failed delete is reported; no `MountsActive` remains | E2 | partial | T12.3c `sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active` - a committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains; partial because the failed-delete sub-arm is not inducible on SQLite (the schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register) and the explicit `Cleanup` report is covered by the seam code path, not executed; PostgreSQL half T12.3b | +| AC-35 | §12 PG refcount; §5.2 | PG | Two mounts, one unmount; failed all-or-none acquire | Store and project advisory keys stay after one unmount; the failed acquire unlocks the taken subset | E2 | deferred | T12.3b deferred: PG advisory-key refcount is the mount/admission layer; no compaction fixture | +| AC-36 | §12 Worker/lost reply; §4, §9 | both | Unreachable worker op; lost reply; retry | Op is `unknown`, never clean; the sidecar is kept; the retry resolves retired, live or absent | E2 | covered | T12.3c `project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry`; `project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry`; `project_delete.rs#an_absent_stack_finishes_a_stale_entry` - a lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent; PostgreSQL half T12.3b | +| AC-37 | §12 Trace; §4.3 | both | Six path names; refusal details; upgraded Store | The six path names are redacted; refusal details are not traced; the salt is backfilled | E2 | partial | T12.3a `gc_trace.rs#the_redacted_set_covers_every_path_bearing_call`; `#the_trace_key_is_salted_by_the_store_policy_salt` (PG salt backfill deferred) | +| AC-38 | §12 Artifacts (cross-backend); §9 | both | SQLite artifact set `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; PG `lfs_gc_` schema; a stray schema | SQLite leaves and removes the full set; PG records the name before CREATE, a remote machine can remove it, a stray schema is `Exists` | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact.rs#every_postgresql_failpoint_leaves_a_resumable_leftover` (remote/stray-schema arms deferred) | +| AC-39 | §12 Position state (2.0 backfill); §4.1 | both | Every upgraded Store (2.0 included); `gc_op` settle; serial allocation | One `project_epoch` row per stack; `gc_op` cleared on settle and not copied; `next_history_serial` never lowered | E2 | covered | T12.3a `gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order`; T12.3b `postgresql_position_columns` 4/4 + fixture | +| AC-40 | §12 Digest agreement; §6.4 | both | Same fixture on SQLite and PostgreSQL; retired head | Same `live_heads_digest`; a retired head changes it | E2 | covered | T12.1 fixture; T12.3b `postgresql_compact_fixtures.rs#the_live_heads_digest_vector_is_the_sqlite_literal` + rebuild fixture | +| AC-41 | §12 PostgreSQL mark; §8 Phase M | PG | Mark transfer; table copy; a base object held only by a dead pack | Only `markBytes` crosses the wire; the copy moves no pack bytes; the base is kept | E2 | deferred | T12.3b `postgresql_compact_fixtures.rs#the_rebuild_report_and_markers_hold_on_postgresql` (depth-2 `pack_id` kept; `markBytes` is a #52 metric, not a result field) | +| AC-42 | §12 Reserved pack; §8 Phase B | PG | Save reserves a pack/ordinal before the mark and publishes during Phase A (or C); cleanup failpoint | `store_totals` moves; the swap `Conflict`s; the pack stays in the old Store; the failpoint keeps it; no dangling location | E2 | deferred | T12.3b deferred: the PG round has no pause point between build and swap; the reserved-pack race is not deterministically injectable | +| AC-43 | §12 PG cleanup (resume); §8 Phase C | PG | After the swap at `G+1`; crash mid-cleanup; totals, ACL, trigger, hints | Store usable at `G+1`; cleanup resumes; totals/ACL/trigger/hints asserted; no `inode` on PG | E2 | deferred | T12.3b `#the_rebuild_report_and_markers_hold_on_postgresql`; `#a_save_after_the_swap_survives_the_resumed_cleanup` | +| AC-44 | §12 PG privileges and reply; §8 | PG | Missing `CREATE`, `TEMP` or `pg_read_all_stats`; lost `COMMIT` reply | `Unsupported`; the lost reply is resolved by proving the backend ended, then reading the incarnation | E2 | covered | T-G38 `postgresql_compact.rs#a_killed_backend_before_the_swap_commit_leaves_the_store_at_g` (a real `pg_terminate_backend(pid, backend_start)` parks Phase B's `gc_op` update before COMMIT: the Store is wholly at `G`, the owner is proved ended and the next `store.compact` adopts the `G` leftover); `#a_killed_backend_during_phase_c_resumes_at_g_plus_1` (a real kill in Phase C after the swap COMMIT: wholly at `G + 1`, the next call reads the incarnation and finishes Phase C); `#a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1` (`PostgresAfterSwap` simulated lost reply, owner then really killed, fresh connection resolves). The privilege half is T-G54 `postgresql_compact.rs#a_role_without_create_on_the_database_is_refused_as_unsupported`, `#a_role_without_create_on_the_store_schema_is_refused_as_unsupported`, `#a_role_without_temp_is_refused_as_unsupported` and `#a_role_without_pg_read_all_stats_is_refused_as_unsupported` (each restricted role is `Unsupported` naming its missing privilege, asserted before the intent, the mark and any scratch schema) plus `#a_fully_privileged_role_compacts` (the owner role with all five grants rebuilds at `G + 1`); `access.rs` now also checks both `CREATE`s and `pg_read_all_stats` | +| AC-45 | §12 SQLite layouts; §7 step 2 | SQLite | Schema 36 (`Monolithic`), 37 (`GroupRows`), 38 (`GroupRowsIndexed`) and their 42/43/44 acquisition variants rebuild | `Monolithic` copies `pack` body and no `pack_unit`; `GroupRows` copies both; `GroupRowsIndexed` copies `pack_unit` and the schema creates its index; `sqlite_sequence` kept when present; `init_*`/acquisition tables empty | E1 | covered | T12.3a `sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none`; `#group_rows_indexed_layout_opens_compacts_and_takes_none`; `#acquisition_layout_opens_compacts_and_clears_working_rows` | +| AC-46 | §12 Install fence; §10 | SQLite | Connection that bypassed the writer lane opening a transaction after install; in-flight transaction; sealed-state or sidecar change at install | `StaleStore`; the in-flight transaction is excluded by the lane (documented limit); a sealed-state or sidecar change is `Conflict` with no writes | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_sidecar_at_install`; `sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale`; `#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction` | +| AC-47 | §12 Artifacts (SQLite exact); §9 | SQLite | Crash mid-compact; next compact; `none` pass; stray name with no op | Crash leaves the full `.compact` set; the next compact removes all of it; `none` leaves no `.compact`; a stray name is `Exists` | E2 | covered | T12.3a `sqlite_compact_crash.rs#crash_during_copy`; `#crash_during_settle`; `sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted` | +| AC-48 | §12 Threshold and failures; §3 | both | 9%, 10% and 30% wholly dead; empty Store (0/0); started rebuild whose refined share falls below 10%; `Capacity`; verify failure | 9% is `none`, 10% and 30% are `rebuild`; empty Store is `none` with `zeroReason: empty-store`; the started rebuild completes; `Capacity` and verify failure abort, delete the artifact set and leave the old Store unchanged apart from step 1's checkpoint | E2 | covered | T12.3a `sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path`; `#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold`; `#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold`; `#an_empty_store_takes_the_none_path`; `#a_capacity_failure_leaves_the_store_unchanged` (PG boundary arms not separately run) | +| AC-49 | §12 Mount/compact race; §5, §8 | both | Mount during compact; mount wins the race; Commit during the rebuild | `Busy` with the compact op; compaction returns `MountsActive` when the mount wins; the Commit is not lost (PG Phase A write makes the swap `Conflict`; Phase C survives the high-water bound) | E2 | executed | T12.3c `sqlite_compact_matrix.rs#arm_mount_during_compaction_is_busy`; `sqlite_compact_matrix.rs#arm_commit_during_rebuild`; `admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner` - a mount during compaction is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild is `StaleStore` or lands on `G + 1` and its retry is never lost; PG Phase A/C arms are T12.3b | +| AC-50 | §12 Profile and scope; §10 | both | All four calls under a folder scope; MCP admin profile | Refused `out of scope`; allowed only in the admin profile | E2 | covered | T12.3a `calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover`; `gc_calls.rs#the_four_gc_calls_take_ids_only` (SQLite-side control test; PG not separate) | +| AC-51 | §12 Progress; §11 | both | `store.compact` running; `store.status` | `store.status` shows the op phase: SQLite `mark`, `build`, `verify`, `install`; PG `mark`, `build`, `swap`, `cleanup` | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_status_phases`; T10 `compact_report_status.rs#status_projects_the_sidecar_phase_and_state`; T-G6a `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases`; T10.5 `postgresql_status.rs#every_postgresql_failpoint_projects_its_phase` (PG `store.status` reads `gc_op`) | +| AC-52 | §12 Report; §11 | both | `none` and `rebuild` paths; every §11 field | `kind: Compacted`, snake_case keys, every §11 field with its unit and sampling point (`freelist_count * page_size`, old size before rename, history `logical` delta, peak temp = pre-truncate staged copy + new file + rollback-journal bound + visited, `physicalShrink`, `keptWaste`, `residue`, `estimate`, capped `cap`, `zeroReason`); `markBytes` is a #52 metric, not a result field; bound at least refined; PG `backendReusable` = 0 and PG/none `deadBytesRefined` equals `deadBytesBound` | E2 | covered | T12.1 golden `sqlite_compact_golden.json`; T10 `compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape`; T12.3b `#the_rebuild_report_and_markers_hold_on_postgresql` | +| AC-53 | §12 Read surface; §4.3 | both | `file.read`/`stat`/`list`/`glob`/`diff`/`revision`/`export`/cp-from before and after compaction; `edit`/`grep`/`blame`/`rm`/`mv`/`mkdir`/`cp`/`apply`; `branch.diff`/`commit.diff` | §4.3 shapes; visible/hidden/absent diff table; `removed_count` 0 and >0; one whiteout case per one-shot tree creator | E2 | partial | T12.3a `project_export.rs#export_omits_a_path_hidden_by_remove_file`; `gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers` (full read-surface table not executed) | +| AC-54 | §12 Restore and listing; §4.2 | both | `backup.restore` before and after a removal; second home's stale entry | Restore succeeds; the stale entry stays out; the §4.2 guard test and allow-list hold | E2 | partial | T12.3a `restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install`; `restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it` (restore-window arm `Unsupported` on this Sandbox) | +| AC-55 | §12 Error matrix; §10 | both | Exact code and message per case, including `StaleStore`, `Removed`, `InvalidInput` for `project.remove`, `Exists`, `Unsupported`, `Capacity` | Exact code and message; snake_case details | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_error_codes`; `#arm_checkpoint_busy`; `sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted` (full engine/API message matrix partial) | +| AC-56 | §12 Digest golden vector; §6.4 | both | Fixed-id fixture (out-of-order ids, a retired Branch and stack, differing timestamps); absent head | One literal vector on both backends; absent head is 33 zero bytes, not 32; widths fixed at 17/33 | E3 | partial | T12.1 SQLite literal vector; T12.3b `postgresql_compact_fixtures.rs#the_live_heads_digest_vector_is_the_sqlite_literal` (3/3 identical runs) | +| AC-57 | §12 Speed and space; §13 | both | At least 3 runs per arm, median and range, declared caches, generated fixtures; §13 arms: `none` at 9%, `rebuild` at 10% and 30%, 50 GB total/35 GB live, 50 GB live, each removal type, removal-then-compact, `remove_file` 0-freed | Downtime per GB live and per GB freed, bytes written, peak disk, PG `ACCESS EXCLUSIVE` duration; absolute bytes when nothing is freed; build phase vs plain `VACUUM INTO`; mark reported separately; a 50% time/5% space option rejected | E3 | partial | T-G61 `sqlite_compact_capacity.rs#capacity_measure` + `sqlite_compact_capacity_measure.json`: a real 9.987 GB Store, `none` at 8.88% dead (3 runs, median 149.318 s) and `rebuild` at 10.00% dead (3 runs, median 246.371 s; mark 149.089 s, build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink 1.017 GB, peak temp 9.144 GB) plus one WAL/FULL rebuild (284.078 s); T-G66 `sqlite_compact_capacity.rs#capacity_measure` + `sqlite_compact_capacity_measure_50gb.json`: a real 49.999 GB Store (384,850 packs; 347,490 live objects; 50.235 GB physical at `none`, 50.863 GB at `rebuild`), `none` at 8.876% dead (1 run, 1,633.585 s, 0 B freed) and `rebuild` at 10.000% dead (1 run, 5,131.835 s; mark 784.251 s, build 4,254.371 s, verify 85.557 s, install 7.311 s; dead 5.062 GB, shrink 5.100 GB, receipt peak temp 45.784 GB under the old `new + visited` formula, sampled real peak 143.1 GB = ~2.9x the Store, T-G72 corrected receipt bound 147.609 GB = 2.90x the old file and `sqlite_compact_capacity.rs#the_peak_temp_receipt_covers_the_staged_copy_and_the_rollback_journal` bounds the journal + unlinked rewrite temp on a moderate fixture); T12.3a `sqlite_compact_matrix.rs#arm_measurement` + `sqlite_compact_measure.json` (tiny fixture); T-G6a `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases` (PG phase durations); §13's 30% dead / 50 GB-live arms, each removal type, removal-then-compact, the plain `VACUUM INTO` comparison and PG `ACCESS EXCLUSIVE` remain deferred/[E]; model source: [`gc-capacity-model.md`](gc-capacity-model.md) (10 GB three-run [M], 50 GB one-run [M] for `none`/`rebuild`; receipt `peakTempBytes` bounds the staged copy + journal + unlinked rewrite temp (T-G72)) | +| AC-58 | §7 step 1.5 (refined decision) | SQLite | After the seal: `ATTACH` the old file `immutable=1`, create `.compact-visited`, run the complete §6.1+§6.2 closure, choose `none`/`rebuild` from the refined live set | `none` deletes the scratch and stops with no `.compact` and no residue; `rebuild` keeps the scratch, reuses `visited` in step 4; step 2 creates the new file only on `rebuild` | E2 | covered | T12.1 fixture; T12.3a `sqlite_compact_crash.rs#crash_before_copy`; `#crash_during_copy` | +| AC-59 | §7 step 1 (seal) | SQLite | `PRAGMA wal_checkpoint(TRUNCATE)`; checked-close the last session; sidecar check under the writer guard; `seal_holding()` | `busy == 0` required; WAL never unlinked; sealed `(size, mtime, db-header change counter)` recorded before release; the guard passes into the seal | E2 | executed | T12.3a `sqlite_compact_matrix.rs#arm_checkpoint_busy`; `#arm_seal`; `sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale` | +| AC-60 | §8 Phase B step 10 (standby) | PG | Effective `synchronous_commit` at least `remote_write` with `synchronous_standby_names` set; standby missing before Phase M; standby missing inside the swap; standby dies after the check | Wait only when required; `FIRST`/`ANY` standbys must be `state=streaming` with matching `sync_state`; a missing standby abandons before COMMIT with no writes (`Busy`, standby list); a standby dying after the check blocks COMMIT | E2 | deferred | T12.3b deferred: a real standby needs a second cluster/GUCs; `compact.rs` predicate unit tests pin the E1 decision | +## Evidence index + +| Source | Report | Tests / fixtures | AC ranges | +|---|---|---|---| +| T12.1 golden fixture | [gc-t12-sqlite-compaction-fixtures.md](gc-t12-sqlite-compaction-fixtures.md) | `tests/sqlite_compact_fixtures.rs`, `tests/fixtures/sqlite_compact_golden.json` | AC-04, 05, 17, 18, 25, 38, 40, 47, 48, 52, 56, 58 | +| T12.3c SQLite gap closure | [gc-t12.3a-sqlite-matrix-report.md](gc-t12.3a-sqlite-matrix-report.md) (regenerated by T12.3c; "SQLite gaps: none") | `engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs` (8 executed rows); `engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs`, `sdk_path_merge_removal.rs`, `sdk_mount_bind_failure.rs`; `engine/crates/layerfs-persistence/tests/epoch_fence.rs`; `control/crates/layerfs-control/tests/project_delete_worker.rs` | AC-06, 07, 11, 12, 13, 14, 22, 34, 36, 49 | +| T12.3a SQLite matrix | [gc-t12.3a-sqlite-matrix-report.md](gc-t12.3a-sqlite-matrix-report.md) | `tests/sqlite_compact_matrix.rs` (executed arms), `sqlite_compact.rs`, `sqlite_compact_crash.rs`, `sqlite_compact_fixtures.rs`, `admission.rs`, `path_removal.rs`, `branch_removal.rs`, `project_delete.rs`, `epoch_fence.rs`, `gc_backfill.rs`, `kept_objects.rs`, `gc_trace.rs`, `gc_calls.rs`, `calls_scope.rs`, `project_export.rs`, `restore_install.rs`, `restored_records.rs` | SQLite half of AC-01..AC-60; executed AC-33/46/51/55/59 | +| T12.3b PostgreSQL acceptance | [gc-t12.3b-postgresql-acceptance.md](gc-t12.3b-postgresql-acceptance.md) | `tests/postgresql_compact_fixtures.rs`, `postgresql_compact.rs`, `postgresql_position_columns.rs`, `postgresql_open_check.rs`, `postgresql_diagnostic.rs` | PG half of AC-02, 28-32, 34-44, 56, 60; also AC-01, 03, 05, 17, 18, 21, 24, 25, 49, 51, 52 | +| T9 PostgreSQL rebuild | [gc-t9-postgresql-rebuild.md](gc-t9-postgresql-rebuild.md) | `tests/postgresql_compact.rs` | AC-28, 41, 43, 44, 49, 60; Phases M/A/B/C | +| T10 report and status | [gc-t10-report-status.md](gc-t10-report-status.md) | `control/crates/layerfs-store/tests/compact_report_status.rs`; `sqlite_compact.rs#the_report_projects_every_section_11_field` | AC-51, 52 | +| T-G6a PG metrics | `engine/crates/layerfs-persistence/src/compact_metrics.rs` (`CompactPhase::{PostgresMark,PostgresBuild,PostgresSwap,PostgresCleanup}`) | `postgresql_compact.rs#a_rebuild_records_bounded_metrics_with_pg_phases`, `#a_none_path_records_only_the_pg_mark_phase` | AC-51, 57 | +| T8 SQLite rebuild | [gc-t8-sqlite-rebuild.md](gc-t8-sqlite-rebuild.md) | `tests/sqlite_compact.rs`, `sqlite_compact_crash.rs` | AC-33, 46, 47, 48, 58, 59 | +| T-G28/T-G60 real-process SIGKILL | - (T-G28 on main layer `32a7d0cf`; T-G60 on `task-g60`) | `sqlite_compact_crash.rs::real_process` (9 SIGKILL arms: mark x2, copy, verify, rename, build, before/after install, settle; child re-execution) | AC-01 | +| T-G65 real-process `store.compact` kill | - (T-G65 on `task-g65`) | `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` (8 SIGKILL arms on the built `layerfs store compact` process: mark x2, build x2, verify x2, install x2); gated failpoint in `control/crates/layerfs-store/src/compact.rs` | AC-01 | +| T-G38/T-G54 PG reply loss and privileges | - (main layers `32dc46a2`, `32a2621d`) | `postgresql_compact.rs` (real `pg_terminate_backend` kills, restricted-role fixtures); `backend/postgresql/access.rs` privilege check | AC-44 | +| T-G61/T-G66 SQLite capacity | [gc-capacity-model.md](gc-capacity-model.md) §1 (10 GB three-run [M], 50 GB one-run [M]) | `engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs` (`#capacity_measure`, opt-in/ignored); `tests/fixtures/sqlite_compact_capacity_measure.json`, `tests/fixtures/sqlite_compact_capacity_measure_50gb.json` | AC-57 | +| Gap record | T12.3a/T12.3b/T12.3c `## Gaps` / `SQLite gaps` | none | SQLite gaps 0 after T12.3c (AC-06, 11, 12, 13, 14, 22, 34, 36 closed); AC-44 covered at T-G38/T-G54; AC-35, 42, 60 deferred | + +T12.3c's report is on main as the regenerated +`gc-t12.3a-sqlite-matrix-report.md` linked above; its eight SQLite-gap rows +now stand `executed` / `covered` / `partial` and its SQLite-gap list is empty. +`deferred` rows name the layer that owns them; none is silently waived. + +## Status summary + +At the plan layer all 60 rows were `pending`; the dispositions after T12.3c, +T-G28, T-G38/T-G54 and T-G65 are: + +| Status | Rows | Count | +|---|---|---| +| `executed` | AC-06, 07, 33, 46, 49, 51, 55, 59 | 8 | +| `covered` | AC-01, 03, 05, 08, 09, 10, 12, 13, 14, 16, 17, 18, 22, 24, 25, 26, 27, 29, 31, 36, 38, 39, 40, 44, 45, 47, 48, 50, 52, 58 | 30 | +| `partial` | AC-02, 04, 11, 15, 19, 20, 21, 23, 30, 32, 34, 37, 53, 54, 56, 57 | 16 | +| `deferred` | AC-28, 35, 41, 42, 43, 60 | 6 | +| `pass` | none | 0 | +| `pending` | none | 0 | + +AC-01 moved from `covered` to `partial` at T-G28 because its real-process +SIGKILL arms re-executed the test binary, not the production process. T-G60 +extended those arms to every phase. T-G65 closed the gap: +`api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` starts the built +`layerfs` command with `store compact`, waits through the engine's socket gate +and `SIGKILL`s that process at the mark, build, verify and install points, so +the killed process is the one that performs the compaction (no daemon serves +Store calls, #352 F-020). AC-01 is `covered` again; the residual note is that +the long-lived `layerfs worker` that serves a machine's Store is still not the +killed process. The six `deferred` rows are +PostgreSQL-only: T12.3b executes the PG halves +of AC-28, 41 and 43, while AC-35 (advisory-key refcount), AC-42 +(reserved-pack race) and AC-60 (a real standby) need server control the +harness does not have. AC-44 moved to `covered` at T-G54: T-G38's real +`pg_terminate_backend` kills cover the lost `COMMIT` reply, and T-G54's +restricted-role fixtures prove each missing `CREATE`, `TEMP` or +`pg_read_all_stats` is `Unsupported` before any scratch schema is created. No row is `pending` +and no SQLite evidence gap remains (T12.3c). ## Coverage map (required area -> cases) @@ -103,6 +180,8 @@ ## Notes +- Statuses record evidence, not #52 acceptance; T12.3c, T12.3b, T-G28, T-G60, T-G65 and T-G75 keep acceptance at #52. +- Evidence cells name the test/file in the report, not a new expected result. - `markBytes` is measured in AC-41 and AC-57, never reported as a `store.compact` field (§11). - A case is accepted at E2 only when the named backend's failpoints are exercised; E3 rows additionally need three runs or the property test, per §12 Speed and space. -- Evidence recording (run id, fixture hash, spec layer, artifact paths) goes beside the status once #52 runs each case. +- Evidence recording (run id, fixture hash, spec layer, artifact paths) is the next #52 step; the cells above name tests, not run ids. diff --git a/core/docs/issues/project-41/gc-capacity-model.md b/core/docs/issues/project-41/gc-capacity-model.md new file mode 100644 index 0000000000..f9a9dd2bd2 --- /dev/null +++ b/core/docs/issues/project-41/gc-capacity-model.md @@ -0,0 +1,122 @@ +# GC capacity model: a 50 GB Store (spec section 13) + +> Deferred 50 GB / §13 capacity model for `layerfs-project#41`. Adds no contract; [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) wins, `§N` cites it. +> **[M]** = measured on the cited artifact; **[E]** = extrapolated from the stated assumptions. Nothing marked [E] is a measurement. +> **T-G61** measured a real **10.0 GB** Store (`sqlite_compact_capacity_measure.json`): three `none` runs at 8.88% dead, three `rebuild` runs at 10.00% dead and one WAL/FULL rebuild. **T-G66** then measured the **50 GB** arm on the same host (`sqlite_compact_capacity_measure_50gb.json`): a **49.999 GB** pack-byte Store (384,850 packs, 347,490 live whole-file objects, 33,850 dead = 8.876%, built in 761.2 s), **one** `none` run and **one** `rebuild` run at 10.000% dead. One run per arm is what §13 asks of the 50 GB arm, not a three-run median; the 10 GB three-run medians stay the repeated-run basis. **The 50 GB numbers do not scale linearly from the 10 GB ones**: the 10 GB Store and its rebuild artifacts fitted the host's page cache, the 50 GB one does not, so its `rebuild` is **85.5 min** and its real temp peak on disk is **143.1 GB above the pre-rebuild baseline** (**~2.9×** the Store) — not the [E] ~20 min and 1.9×. **T-G72** replaced the receipt's `new + visited` peak with a documented bound over the run's temp artifacts (pre-truncate staged copy + new file + rollback journal + visited): **147.609 GB** for the same 50 GB `rebuild`, 3.2% above the sampled peak and no longer under-reporting the journal. The 30% dead, 50 GB-live/35 GB-live, each-removal-type and PostgreSQL `ACCESS EXCLUSIVE` arms remain [E]. + +## 1. Measured basis + +| Quantity | Tiny fixture (T12.3a) [M] | Capacity fixture (T-G61), 10 GB [M] | 50 GB arm (T-G66) [M] | +|---|---|---|---| +| Store size | 20 live roots, 20 dead packs | **9.987 GB pack bytes** (76,870 packs; 69,399 live 131,071-byte whole-file objects, 6,770 dead); 10.035 GB physical | **49.999 GB pack bytes** (384,850 packs; 347,490 live 131,071-byte whole-file objects, 33,850 dead); **50.235 GB physical** at the `none` seal, 50.863 GB at the `rebuild` seal | +| Profile | `Disposable` (WAL/OFF) | `Disposable` (WAL/OFF); one `Durable` (WAL/FULL) control | `Disposable` (WAL/OFF); no `Durable` arm (owner profile disabled) | +| `none` wall | n/a | 148.519 / 149.704 / **149.318 s** median; range 148.519–149.704 | **1,633.585 s** (one run) | +| `rebuild` wall | 26.738 / 28.201 / 28.681 ms; median 28.201 ms | 246.371 / 255.548 / 243.306 s; median **246.371 s**; range 243.306–255.548 | **5,131.835 s** (one run) | +| `Durable` rebuild | n/a | **284.078 s** (one run) | not run (owner profile) | +| `deadBytesBound` | 1,100 B | **887,601,160 B** at 8.88% (`none`); **1,011,104,896 B** at 10.00% (`rebuild`) | **4,438,005,800 B** at 8.876% (`none`); **5,062,342,096 B** at 10.000% (`rebuild`) | +| `physicalShrink` | 20,480 B | **1,016.5–1,019.0 MB** (mean 1.017 GB); `none` 0 | **5,099,651,072 B** (5.100 GB); `none` 0 | +| `peakTempBytes` (receipt) | **758,240 B** (T-G72 bound; old formula 258,048 B) | **29,481,103,648 B** (29.481 GB bound; old formula 9.144 GB) | **147,608,626,376 B** (147.609 GB bound; old formula 45.784 GB) | +| Peak on disk, sampled | n/a | not sampled | **143.1 GB above the pre-rebuild baseline** (§3) | +| New file | n/a | 9,139,773,440 B (9.140 GB) | 45,763,203,072 B (45.763 GB) | +| Fixture build | n/a | 150 s | **761.2 s** (65.7 MB/s over 49.999 GB) | +| Host | linux-x86_64, rustc 1.85.1 | 88 vCPU, 135 GB RAM, ext4, 4.0 TB free | 88 vCPU, 125 GB visible RAM, ext4 (`/dev/sda2`), 4.0 TB free; same rustc 1.85.1 | + +Phase durations, 50 GB `rebuild` (one run, WAL/OFF): **mark 784.251 s**, **build 4,254.371 s**, **verify 85.557 s**, **install 7.311 s** (sum 5,131.490 s of the 5,131.835 s wall). `none`: mark 1,633.579 s and nothing else. +Sources: [`sqlite_compact_capacity_measure.json`](../../../../engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json) (10 GB) and [`sqlite_compact_capacity_measure_50gb.json`](../../../../engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json) (50 GB). + +Caches [M]: the 10 GB Store fitted in the host's page cache and no cache drop ran between its runs; the 50 GB arm did not fit. Its `none` mark ran immediately after the fixture build (50 GB of fresh writes still in the page cache) at **35.9 s per live GB**; the `rebuild` mark, after that read pass, ran at **17.2 s per live GB** — the 16.4 s/GB of the 10 GB arm. The mark is still single-threaded payload/identity hashing (~62 MB/s effective); the 2.1× spread is cache state, not the object count. No cache drop ran between the two 50 GB arms either. + +Per-GB rates measured on this host (GB = 10^9 B): + +| Step | 10 GB [M] | 50 GB [M] | Basis | +|---|---|---|---| +| Mark (`none`) | 14.9 s per stored GB / 16.4 per live GB | **35.9 s per live GB** (32.7 per stored GB) | 1,633.579 s over 45.561 GB live; cold page cache, right after the fixture build | +| Mark (`rebuild`) | 16.4 s per live GB | **17.2 s per live GB** | 784.251 s over 45.561 GB live; warm after the `none` read pass | +| Build (`VACUUM INTO` + prune + `VACUUM`) | 10.3 s per new GB | **93.0 s per new GB** | 4,254.371 s over 45.763 GB new; the phase moves ~141 GB (staged copy 50.85 GB, 44.6 GB rollback journal, target rewrite 45.76 GB) at ~33 MB/s | +| Verify | 0.034 s per new GB (0.31 s total) | **1.87 s per new GB** (85.557 s) | at 50 GB it reads the payload back; the 10 GB read was cache-only | +| Install | ~1.9 s fixed | **7.311 s** | rename, fsync, settle | + +The mark dominates at 10 GB; at 50 GB the build phase dominates by 5.4× — the opposite. Do not extrapolate either phase linearly from the 10 GB arm: the 10 GB run was cache-optimistic, and the 50 GB build rate depends on free page cache and device write throughput rather than on CPU. Use the 50 GB [M] rates above for a 50 GB Store and re-measure with §7's harness on the target host. + +## 2. The 50 GB / 9% Store and the rule + +`rebuild iff total > 0 AND dead * 10 >= total` (§3); `dead` = bytes in packs with no live object after the metadata walk (§6.4). GB = 10^9 B. + +| Dead share | Old | Dead | Live | `dead*10 >= total` | Path | Measured? | +|---|---|---|---|---|---|---| +| 9% | 50.0 GB | 4.5 GB | 45.5 GB | 45 >= 50: **no** | `none` | **50 GB [M]**: 49.999 GB pack bytes, 8.876% dead → `none`, 0 B freed | +| 10% | 50.0 GB | 5.0 GB | 45.0 GB | 50 >= 50: **yes** | `rebuild` | **50 GB [M]**: 50.623 GB pack bytes, 10.000% dead → `rebuild`, 5.100 GB freed | +| 30% | 50.0 GB | 15.0 GB | 35.0 GB | 150 >= 50: **yes** | `rebuild` | [E] only; §13's 50 GB total/35 GB live and 50 GB-live arms remain deferred | + +At **9% the measured path is `none`**: mark only, no file, generation unchanged (7 → 7), **0 bytes freed**; the report still gives the dead bound (4.438 GB measured at 50 GB; 0.888 GB at 10 GB). +The measured 50 GB rebuild copied a 45.763 GB new file to free 5.100 GB — **8.97 B copied per B freed** [M], against 9.04 at 10 GB [M] and the frozen 10% floor's ~9.0 [E]. At 30% it is ~2.3 [E]. + +## 3. Temp-space model + +**SQLite (measured).** 50 GB arm: old physical file 50.863 GB; new `.compact` 45.763 GB; `.compact-visited` 20,365,312 B. The rebuild's temp artifacts are the *whole* staged copy, a rollback journal and SQLite's rewrite temp database — not the new file plus visited. `VACUUM INTO` first writes the whole sealed Store to `.compact` (the old page count, 50.863 GB); the prune's `DELETE ... WHERE NOT IN visited` grows `.compact-journal` to **44.6 GB** (44,630,503,928 B observed); the closing `VACUUM` then writes the vacuumed Store to SQLite's unlinked temp database (≈45.763 GB, in the process temp directory) while the staged file is rewritten in place and the rollback journal records the pages it overwrites. A 20 s disk sampler saw the run's real temp peak at **143.1 GB above the pre-rebuild baseline** (sample max 315.9 GB used against 172.8 GB before the rebuild) = staged 50.85 + rewrite temp 45.76 + journal 44.6–45.8 GB; **~2.9×** the 50 GB pack-byte Store [M]. +The old receipt computed `target_bytes + visited_bytes` (`Built::peak_temp_bytes` in `engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs`) *after* the staged file was truncated: 45.784 GB = new + visited, under-reporting the rollback journal and the staged/temp copies (the T-G66 finding). +**T-G72 bounds the whole run instead.** `Built::peak_temp_bytes` is `staged_bytes + journal_bound + target_bytes + visited_bytes`: the staged extent measured right after `VACUUM INTO`, the rollback-journal upper bound (one record per staged page, `page_size + 8` B, plus an 8192 B header allowance), the new file (which also bounds the rewrite's temp database) and the visited scratch. The 50 GB arm is 50,862,854,144 + 50,962,203,848 + 45,763,203,072 + 20,365,312 = **147,608,626,376 B (147.609 GB) = 2.90× the old file**, 4.5 GB (3.2%) above the sampled peak: the bound charges the journal the whole staged page count instead of the pages the transactions actually touch. The total footprint is old + peak = **~198 GB (3.9×)** at 50 GB; the measured sample was old + 143.1 ≈ 194 GB (3.8×) [M]. Upstream issue: [Ephemeral-AI-Lab/layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). +Visited is metadata-scale: 20,365,312 B for 347,490 live objects = **58.6 B per visited object** [M] (10 GB: 57 B). +`physicalShrink` (5.100 GB) exceeds `deadBytesBound` (5.062 GB) by ~0.7%, as at 10 GB (0.6%): shrink is file-level and includes free pages; do not equate the two. + +**PostgreSQL.** Peak = old tables (50 GB, dead included) + scratch schema `lfs_gc_` (LIKE copy of live `pack`/`object_location`) + index/PK + `ANALYZE` + WAL "up to the live bytes" (§8 "Space"); mark temp tables are session-temp and small. +Extrapolated [E; SQLite measured: the T-G72 receipt bound is 2.90× the old file and old + peak ~3.9×, so the SQLite rows below are receipt-bound estimates and every [E] row stays a projection]: + +| Backend | Case | Old | New file / scratch | Visited / index | Journal / WAL | Peak temp | Total footprint | +|---|---|---|---|---|---|---|---| +| SQLite | 9% `none` | 50.0 | 0 | <=0.1 GB | 0 | **<=0.2 GB** | ~50.2 GB | +| SQLite | 10% `rebuild` | 50.0 | ~45.8 new + ~50.9 staged | ~0.02 GB | ~44.6 GB prune / ~45.8 GB rewrite | **~147.6 GB bound** (sampled 143.1) | **~198 GB (3.9× bound; 3.8× sampled)** | +| SQLite | 30% `rebuild` | 50.0 | ~35.2 new + ~50.9 staged | ~0.02–0.1 GB | ~35 GB prune/rewrite [E] | ~137 GB bound [E] (actual ~120 [E]) | **~187 GB [E] (3.7×)** | +| PG | 9% `none` | 50.0 | 0 | 0 | mark only | **~50 GB** | ~50 GB | +| PG | 10% `rebuild` | 50.0 | ~45.0 | ~4–6 GB | <=45 GB | **~99–101 GB, <=146 GB** | same | +| PG | 30% `rebuild` | 50.0 | ~35.0 | ~3–5 GB | <=35 GB | **~88–90 GB, <=125 GB** | same | + +**Headroom [M].** Keep **>=150 GB free** for a 50 GB `rebuild` at ~10% dead (measured 143.1 GB sampled peak, ~2.9× the Store), >=115 GB at <=30% dead [E]; `none` needs a few GB. The T-G61 [E] guidance of 95–110 GB came from the receipt field and is **too low**. +Size `pg_wal` separately: a full `pg_wal` stops the server (§8). + +## 4. Time model + +Measured 50 GB [M] on this host, beside the 10 GB medians. Linearity is **not** assumed: the 50 GB column is measured, not scaled. + +| Dead | Path | Measured old | 10 GB wall (median) [M] | 50 GB wall [M] | s/GB live at 50 GB [M] | Freed at 50 GB | +|---|---|---|---|---|---|---| +| 9% | `none` | 49.999 GB | 149.318 s | **1,633.585 s (~27.2 min)** | 35.9 | 0 | +| 10% | `rebuild` | 50.623 GB | 246.371 s | **5,131.835 s (~85.5 min)** | 112.6 | 5.100 GB | +| 10% | `rebuild`, Durable | - | 284.078 s | not run | - | - | +| 30% | `rebuild` | - | - | **~65–70 min [E]** (mark 17.2 s/GB + build 93 s/GB over ~35.8 GB new) | ~110 [E] | ~15.0 GB | + +The oldest [E] table said 1.8–2.0 min for a 50 GB rebuild; the linear T-G61 [E] said ~20.3 min; the measured 50 GB rebuild is **85.5 min**, 4.2× the linear projection. Two measured causes: (a) the build phase is **9× slower per new GB** than at 10 GB (93.0 against 10.3 s/GB) because the rebuild's ~146 GB working set (old + staged + journal) no longer fits the 125 GB page cache, so the copy is real device I/O; (b) the mark has a 2.1× cold/warm spread (35.9 against 17.2 s/GB live). Plan **~27 min** for a 50 GB `none` and **~85 min** for a 50 GB `rebuild` on a host like this one [M]; re-measure with §7's harness on the target host. +PostgreSQL is not measured by AC-57; its server-side `INSERT ... SELECT` copy, index build and `ANALYZE` extrapolate to **4–10 min**, `ACCESS EXCLUSIVE` only at the swap, **~2 s** [E]. + +## 5. Operation window, mounts and commits + +- **SQLite.** `writer_admission` is held checkpoint→rename, so the exclusive lane is the whole §7 run: **27.2 min for a 50 GB `none`** and **85.5 min for a 50 GB `rebuild`** [M]; the 10 GB values are 2.5 min each [M]. A live mount = `MountsActive`; a mount during compaction = `BusyOp` (AC-07/AC-49); a Commit racing install = `StaleStore` or lands on `G+1`, retry never lost. `none` is mark-only but still holds the lane for its mark. +- **PostgreSQL.** The session advisory key spans M–C; mounts get `BusyOp`, a post-mark commit makes the swap `Conflict` with no writes (§8). `ACCESS EXCLUSIVE` is the swap only (~2 s) [E]; the `G+1` Store is usable during C. Plan 25 minutes quiet for SQLite at 50 GB [M for the phases, E for the quiet-window policy], mostly online for PG [E]. +- **Monitoring.** `store.status` gives `op {id,name,phase,started,uncertain}`, `state` busy/unresolved and phases — SQLite `mark/build/verify/install`, PG `mark/build/swap/cleanup` (§11). The capacity runs' T10.3 phase sums are in the fixtures (`none` = all mark; `rebuild` = mark/build/verify/install). + `compact_metrics` has runs, operations, refusals, settles, retries, status, slo, phase durations and last §11 gauges: process-local, no CLI/SDK renderer, no PG metrics. + Don't loop a direct reader: it makes the checkpoint `Busy` (§7; [`gc-runbook.md`](gc-runbook.md) §1). + +## 6. Risks + +- **Disk full (both).** SQLite `SQLITE_FULL`/`ENOSPC`/`EDQUOT`/`EFBIG` → `Capacity`, artifact deleted, old Store unchanged apart from checkpoint (§7 step 8; AC-55). The measured 50 GB run needed **143.1 GB** of sampled temp peak (~2.9× the 50 GB Store); the T-G72 receipt bound is **147.609 GB** (2.90× the old file), so keep **>=155 GB free** and re-check before a 30% run (>=140 GB there [E]). +- **Peak temp.** `peakTempBytes` now bounds the run's temp artifacts (T-G72): pre-truncate staged copy + new file + rollback-journal bound + visited. The 50 GB receipt is 147.609 GB (2.90× the old file; sampled 143.1 GB); total footprint = old + peak ≈ 198 GB (3.9×). The journal term is conservative: the bound charges one record per staged page while the run journaled 44.6 GB [M]. +- **WAL.** SQLite needs `wal_checkpoint(TRUNCATE) busy == 0` and never unlinks `-wal`/`-shm` (a live reader is `Busy`); PG WAL ≤ live bytes + standby lag. +- **PG standby.** `synchronous_commit ≥ remote_write` with `synchronous_standby_names` needs a streaming sync/quorum standby; missing → `Busy` no writes; one dying after the check blocks COMMIT and `SET LOCAL` cannot bound it (§8; AC-60 deferred). +- **Installed-but-uncertain / races.** Rename/fsync failure or lost COMMIT keeps the new Store at `G+1`, settles `unknown`; never delete the artifact; PG: prove the backend ended (`pid`, `backend_start`, `usename`), then read the incarnation. + `MountsActive`/`BusyOp`/`StaleStore` are expected: quiesce, retry, never kill an owner. The 20,480 B vs 1,100 B ratio and 28.2 ms are tiny-fixture effects; the 10 GB and 50 GB numbers above are the capacity evidence. + +## 7. Runbook thresholds (50 GB Store) + +1. **Eligibility:** only `dead*10 >= total` (>=10% dead, §3); below it `none`, no file, 0 freed — but still a full mark (**27 min at 50 GB** [M]). Record the dead bound and re-check later. +2. **Economics/space [M]:** at 10% dead the measured rebuild copies **8.97 B per B freed** [M] (10 GB: 9.04 [M]) and needs **~148 GB free** (147.609 GB receipt bound, 2.90×) [M]; prefer **>=20–30% dead** (30% ~2.3 B per B [E], ~137 GB bound [E]). +3. **Window/quiesce [M]:** SQLite **~27 min** quiet for a 50 GB `none` and **~85 min** for a 50 GB `rebuild`; the 10 GB values are 2.5 min each. No full doctor; expect `BusyOp`. A `Busy` owner: wait, do not terminate (runbook §2). +4. **Recovery gate:** if `store.status.state` is `unresolved`, rerun `store.compact` to adopt the op; never hand-delete artifacts or `gc_intent`/`gc_dead` (runbook §2–§3). +5. **Backup/alerts/failures:** `backup.create` before the first run ([`gc-rollout.md`](gc-rollout.md) §3; §9); alert on `Capacity`, `unresolved`, `retries`, nonzero `residue`, PG WAL/standby lag. `Capacity` → free space and rerun; uncertain → keep the new Store, settle `unknown`. +6. **Re-measure per host.** `CAPACITY_MEASURE=1 UPDATE_CAPACITY=1 cargo test -p layerfs-persistence --release --test sqlite_compact_capacity -- --ignored --nocapture` with a TMPDIR of **>=3× the target** (the receipt bound is 2.90×; the measured temp peak was 143.1 GB). The 50 GB arm used `CAPACITY_GROUPS=3510 CAPACITY_DEAD_OBJECTS=33850 CAPACITY_RUNS=1 CAPACITY_KEEP=1 CAPACITY_OUT=<...>_50gb.json`; the mark is CPU-bound and the copy I/O-bound, so a host with faster SHA-256 and more page cache can be several times quicker. + +## Sources + +- [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md): §3 threshold, §6.4 dead set, §7 SQLite rebuild/cost/Capacity, §8 PG phases/space/standby, §11 report and `peakTempBytes`, §13 deferred 50 GB arms and estimates. +- [`gc-t12.3a-sqlite-matrix-report.md`](gc-t12.3a-sqlite-matrix-report.md) (AC-57) and `sqlite_compact_measure.json` (the tiny [M] numbers; its `peakTempBytes` is the T-G72 bound, 758,240 B); `sqlite_compact_capacity_measure.json` and `sqlite_compact_capacity.rs#capacity_measure` (the 10 GB [M] numbers); `sqlite_compact_capacity_measure_50gb.json` (the 50 GB [M] numbers, one run per arm). The two capacity JSONs still record the T-G66 old-formula `peakTempBytes` (45.784 GB at 50 GB); the T-G72 bounds in this page are arithmetic over their measured page counts and sizes, not a re-measurement. +- [`gc-t9-postgresql-rebuild.md`](gc-t9-postgresql-rebuild.md) (M/A/B/C, scratch schema, WAL/standby gaps) and [`gc-implementation-report.md`](gc-implementation-report.md) (status/metrics projection). +- [`gc-runbook.md`](gc-runbook.md) and [`gc-rollout.md`](gc-rollout.md) (safe cleanup, rehearsal). diff --git a/core/docs/issues/project-41/gc-implementation-report.md b/core/docs/issues/project-41/gc-implementation-report.md new file mode 100644 index 0000000000..82456cefb3 --- /dev/null +++ b/core/docs/issues/project-41/gc-implementation-report.md @@ -0,0 +1,485 @@ +# GC implementation report: soft removal and Store compaction + +> Final report for `layerfs-project#41`. The frozen +> [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) (2.0.0) wins; `§N` and `AC-nn` +> cite it. This page certifies main layer `32ae51ca` (audited by T-G75 on +> 2026-11-15) in §8; it adds no contract and leaves acceptance sign-off to +> #52. + +## 1. Scope and what shipped +Four ids-only, admin-profile calls on SQLite and PostgreSQL: +`branch.remove`, `project.delete`, `project.remove_file` and `store.compact`. +The first three change visibility and retention only; `store.compact` +reclaims bytes. +**Schema and removals.** The additive open step adds `branch_retired`, +`layer_stack_retired`, `path_removed`, `project_epoch`, `gc_intent`, `gc_op`, +`gc_dead` and `kept_node`, plus `history_meta.next_history_serial` (never +lowered), `commit.history_serial`, `layer.history_serial` and +`store_policy.trace_salt`; it backfills serials deterministically, creates one +zero epoch per stack and raises the next serial above the maximum. `branch` and +`layer_stack` stay append-only; 1.0 Stores convert offline; there is no +down-migration (§4.1). `branch.remove` writes `branch_retired`; +`project.delete` writes `layer_stack_retired` plus a retry sidecar and returns +`home_entry_removed: false` on a home-removal failure; `project.remove_file` +writes `path_removed` at frontier `next_history_serial - 1`. Visibility is +filtered statements plus an allow-list guard, not views, and the three removal +markers are never rewritten (§4.2-§4.4). + +**Admission and mark.** `gc_intent` is one Store-scoped row per operation +(kind, scope, state, owner pid/start/boot, phase, generation, artifact, high +water). SQLite uses two `BEGIN IMMEDIATE` transactions and reports +`MountsActive`, `Busy` or `Unresolved`; PostgreSQL holds `(kind, id)` advisory +keys at Store, project and Branch scope, records the owner in `gc_op`, and ends +a dead idle owner with `pg_terminate_backend(pid, backend_start)`. The epoch +fence re-checks `project_epoch.epoch` before commit (same transaction on +SQLite, a final `FOR SHARE` on PostgreSQL); a changed or missing epoch is +`Conflict` with no Commit (§5). `HistoryProvider::mark_gc` reads the tuple +(`catalog_incarnation`, `next_history_serial`, `next_pack_id`, `next_ordinal`, +`store_totals`, `MAX(stamp)`, history rows), runs the §6.1 history closure and +the §6.2 object closure (`FinalizedObject::admit` plus +`ChainBases::base_of`, one visit level, delta bases), and writes `kept_node`, +the object/pack set, `deadBytesBound`/`deadBytesRefined`, `gc_dead` and the +§6.4 `live_heads_digest` (absent head is 33 zero bytes). `keptWaste` covers +retained-pack waste, delta bases and all `inode_value_block` rows. + +**SQLite rebuild.** `SqliteHandles::compact` holds `writer_admission` from +checkpoint through rename, runs `wal_checkpoint(TRUNCATE)` (busy is `Busy`; the +WAL is never unlinked), seals `(size, mtime, db-header change counter)`, +attaches the old file `immutable=1`, builds `.compact-visited`, runs +`VACUUM INTO .compact`, and in one transaction keeps retained rows, deletes the +complement, sets `G + 1`, recomputes `store_totals` and clears +`binding_hint_block`. Verification checks schema, foreign keys, counts, both +digests, allocator and totals; install rechecks sidecars and `-mj*`, applies +`chmod 0600` and `sync_all`, renames and fsyncs; settle removes the artifact +set and op record. Failpoints `AfterSeal`, `AfterBuild`, `AfterWalSwitch`, +`AfterVerify`, `BeforeInstall`, `AfterInstall` and `CapacityAtBuild` cover the +crash states; the artifact set is +`.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; a stray name is +`Exists`, a `(dev, ino)` change is `StaleStore`, and a sealed-tuple change at +install is `Conflict`. The path is `none` below 10 percent, `rebuild` at or +above it; an empty Store is `none` with `zeroReason: empty-store`; a started +rebuild completes (§7). + +**PostgreSQL rebuild.** `PostgresHandles::compact` runs four phases. M writes +`kept_node`, `gc_op` (phase `mark`, scratch `lfs_gc_`), `gc_dead` and the +dead totals. A creates the scratch schema with LIKE copies of `pack` and +`object_location`, copies retained rows in idempotent batches of 512 packs, +then adds the prefix index, foreign key and `ANALYZE`. B is one short +transaction: `ACCESS EXCLUSIVE NOWAIT` (`55P03` is `Busy`), re-read the mark +tuple and digest (`Conflict` on change), copy owner and ACL, swap the tables, +recreate `pack_immutable`, settle the totals, set `G + 1`, re-check +`synchronous_commit` and the standby predicate, drop the scratch, `COMMIT`. C +deletes only `gc_dead` members, clears `binding_hint_block` and the acquisition +rows, reduces `store_totals` once against the row sums, clears `gc_dead` and +`gc_op`, and releases the intent. The next call adopts a dead idle owner, is +`Busy` on a live owner, and resumes Phase C at `G + 1`; a lost `COMMIT` reply +is resolved by proving the backend ended, then reading the incarnation. +PostgreSQL reports `backendReusable = 0` and `postgresql-in-place`, and has no +`inode` table (§8, §9). + +**Report, status, metrics.** `CompactReport`/`CompactStatus` in +`engine/crates/layerfs-persistence/src/report.rs` and the single serializer in +`control/crates/layerfs-store/src/report.rs` render every §11 field in +snake_case (`dead_bytes_bound`, `kept_waste`, `residue`, `estimate`, and one +`zero_reason` per zero field from +`empty-store|none-path|no-history|no-waste|postgresql-in-place`). `store.status` +adds `op {id, name, phase, started, uncertain}`, `state` (`busy` or +`unresolved`; `uncertain` is true for every remaining record) and the backend +`phases` (SQLite `mark/build/verify/install`; PostgreSQL +`mark/build/swap/cleanup`). `compact_metrics.rs` publishes counters (`runs`, +`operations{outcome}`, `refusals{code}`, `settles{kind}`, `retries{reason}`, +`status{state}`, `slo{objective,result}`), phase durations and last-run §11 +gauges, with labels only from closed enums. + +**API, MCP, CLI, SDK.** The call table is 76 entries (38 work / 38 admin); the +four calls are admin-only and refused under a folder scope with `out of +scope: `. `project.remove` is retired: old clients get `InvalidInput`, +MCP reports an unknown tool, and the CLI exits 2; `project.init` on a held name +is `NameTaken`. MCP names are `branch_remove`, `project_delete`, +`project_remove_file` and `store_compact`; the CLI has `layerfs branch remove`, +`project delete`, `project remove-file` and `store compact`; JS and Python +expose the same four methods. The call log redacts path-bearing arguments +(``) and hashes keys with `trace_salt`. + +| Expected-base CAS upstream | `CommitRequest` carries `expected_head`, `expected_base` and `expected_root`; a moved Branch is `HeadMoved` and nothing is written, so a stale daemon commit cannot clobber a removal | +| Whiteouts | `remove_file` hides at the frontier; merge, publish and mount materialize the whiteout; a merge that cannot is `Removed` with full path elements | +| `gc_intent` / `gc_op` / `gc_dead` | intent is admission and liveness; `gc_op` is the one compaction row with phase, scratch and owner; `gc_dead` is persisted membership, not a predicate | +| Generation | `catalog_incarnation` is `G`; a rebuild installs `G + 1`; leftovers, fences and resume judge by the recorded generation | +| Reclaim unit | the pack; a rebuild copies retained rows into a fresh Store; no in-place pack delete; `rebuild iff total > 0 AND dead * 10 >= total`; a started rebuild completes | +| Visibility-only `remove_file` | 2.0 hides the path and keeps objects reached only through it; a path-aware walk is a follow-up | +| Thresholds | `deadBytesBound` from the metadata walk, `deadBytesRefined` from the payload sweep; PostgreSQL always reports them equal | + +## 3. Test evidence + +T-G75 re-ran the suites from main `32ae51ca` on 2026-11-15 in a Sandbox +Workspace (Branch `task-g75`); the counts below are those runs unless the row +names an earlier run. Earlier lanes were recorded on main `32df79da` or the +Branch the row names. The final lanes are: + +| Suite | Result | Recorded by | +|---|---|---| +| `layerfs-persistence` (full lane) | 271 passed, 0 failed, 3 ignored | T-G75 (T-G24: 264/0/2) | +| `layerfs-store-service` | 214 passed, 0 failed, 15 ignored | T-G75 (T-G24: same) | +| `layerfs-store` | 59 passed, 0 failed, 0 ignored | T-G75 (T-G24: same) | +| `postgresql_compact` | 20 passed (real `pg_terminate_backend` reply-loss arms and the restricted-role privilege arms) | T-G38/T-G54 | +| `postgresql_status` | 3 passed (persistence) + 1 passed (control) | T10.5/T-G53 | +| `postgresql_compact_fixtures` | 5 passed | T12.3b | +| `sqlite_compact` | 20 passed | T-G52; re-run T-G75 | +| `sqlite_compact_crash` | 23 passed (13 in-process + 9 real-process SIGKILL + the re-executed child placeholder) | T-G28/T-G60, re-run T-G52 and T-G75 | +| `sqlite_compact_fixtures` | 1 passed | T-G52; re-run T-G75 | +| `sqlite_compact_matrix` | 1 runner, 8 executed arms | T-G52; re-run T-G75 | +| `sqlite_compact_capacity` | 1 passed, 1 ignored (the opt-in 50 GB arm is not run by default) | re-run T-G75 | +| PostgreSQL doctor (physical) | 16 passed | T-G55 | + +**SQLite lane (T12.3-era baseline; the final counts are the table above).** + +| Binary | Tests | Pins | +|---|---|---| +| `sqlite_compact` | 20 | `none`/`rebuild`, 9/10/30 percent, empty Store, capacity abort, layouts, stray name, report fields, leftover op | +| `sqlite_compact_crash` | 23 | in-process failpoints at seal/build/WAL switch/verify/install/settle and the artifact set, plus 9 real-process SIGKILL arms (mark: BeforeSeal/AfterSeal; copy: BeforeVacuum; verify: DuringVerify; rename: DuringRename; build: AfterBuild; install: BeforeInstall/AfterInstall; settle: DuringSettle) and the re-executed child placeholder | +| `sqlite_compact_fixtures` | 1 | golden receipt with every §11 field, digest vector, three outcomes | +| `sqlite_compact_matrix` | 1 runner, 8 arms | checkpoint busy, sidecar at install, status phases, error codes, seal, random shapes, measurement, digest row after collection | +| `compact_metrics`, `admission`, `path_removal`, `branch_removal`, `project_delete` | 4, 5, 3, 3, 5 | metrics, admission races, frontier, markers | +| `epoch_fence`, `gc_backfill`, `kept_objects`, `kept_node`, layouts, upgrade | 7, 4, 5, 7, 7+9, 2+2 | epoch rollback, backfill, retention, layouts | +| `gc_trace`, `gc_calls`, `calls_scope` (API) | 5, 8, 4 | redaction, four-call shapes, scope refusal | +| `compact_report_status`, `project_delete` (control) | 3, 15+6+2 | wire shape, status, sidecar and frames | +| `branch_remove`, `admission`, `epoch_fence`, `project_delete`, `kernel_guard` (store-service) | 3, 1, 1, 1, 7 | service removals and guard | + +**PostgreSQL lane (T12.3b baseline; the final counts are the table above).** +PostgreSQL 17.12, `cargo test -p layerfs-persistence +--features postgresql`, `--test-threads=1`: 15 test binaries, 73 tests passed +at T12.3b. The run covers `postgresql_compact` (8 at that layer; 20 on +`32df79da`), `postgresql_compact_fixtures` (5, +three consecutive runs), `postgresql_position_columns` (4), +`postgresql_open_check` (3), `postgresql_diagnostic` (5), +`postgresql_acquisition` (4), `postgresql_acquisition_compaction` (4), +`postgresql_identical_ids` (3), `layerstack_removal` (2), +`postgresql_commit_round_trips` (1) and the SQLite controls. The new fixture +proves the §6.4 digest is byte-identical to the SQLite golden and the rebuild +report and markers hold on PG. + +**Harness and CI.** `tools/ci/test_store_ext4.sh` requires +`LAYERFS_TEST_TMPDIR` on ext4, xfs or btrfs (apfs or hfs on macOS), refuses +tmpfs, overlay, FUSE and network mounts before cargo, and points `TMPDIR`, +`CARGO_TARGET_DIR` and `CARGO_HOME` there. `--quick` runs a cross-suite +subset; `--full` runs every Store suite and adds `--features postgresql` when +`LAYERFS_TEST_POSTGRES_URL` is set. `.github/workflows/store-ext4.yml` is the +CI lane: `quick` on push and pull request, `full-postgresql` (a `postgres:17` +service) on schedule and manual dispatch; it is report-only. + +## 4. Acceptance matrix status + +The matrix has 60 cases (47 both, 6 SQLite-only, 7 PostgreSQL-only). Only +#52 records acceptance; these notes record evidence and tier. + +| Runner | Executed | Covered/pass | Partial | Deferred | Gaps | +|---|---|---|---|---|---| +| T12.3a SQLite half (pre-T12.3c) | 5 | 24 | 16 | 7 (PG-only) | 8 | +| T12.3b PostgreSQL half | 0 | 11 | 11 | 8 | 0 | +| T12.3c SQLite gap closure | 8 | 29 | 16 | 7 (PG-only) | 0 | +| T-G28 real-process SIGKILL (AC-01) | 8 | 28 | 17 | 7 (PG-only) | 0 | +| T-G38/T-G54 PG reply loss and privileges (AC-44) | 8 | 29 | 17 | 6 (PG-only) | 0 | +| T-G60 remaining SIGKILL points (AC-01) | 8 | 29 | 17 | 6 (PG-only) | 0 | +| T-G65 real-process `store.compact` kill (AC-01) | 8 | 30 | 16 | 6 (PG-only) | 0 | + +T12.3c closed the eight SQLite gaps: AC-06 (post-compaction Save of a +collected digest), AC-11 (non-FF merge whiteouts and publish at the +frontier), AC-12 (removed path in a merge), AC-13 (unmerged Branch marker +inheritance), AC-14 (layer-headed mount whiteout), AC-22 (concurrent +merge/removal property), AC-34 (bind failure and failed intent delete) and +AC-36 (worker lost reply); its report records "SQLite gaps: none". The +PostgreSQL half's partial and deferred rows are AC-02, AC-29, AC-30, AC-31, +AC-32, AC-34, AC-35, AC-36, AC-37, AC-38, AC-41, AC-42 and AC-60. T-G28 +added the real-process SIGKILL arms of `sqlite_compact_crash.rs`; AC-01 moved +from `covered` to `partial` because the kill targeted a test child, not the +production process. T-G60 extended the real arms to the mark (`BeforeSeal`/ +`AfterSeal`), copy (`BeforeVacuum`), verify and rename points. T-G65 then +started the built `layerfs store compact` command, killed that process at the +mark, build, verify and install points, and asserted `G`/`G + 1`, the +artifact set, the `store.status` projection and the settling run, so AC-01 is +`covered` again (the long-lived `layerfs worker` is still not the killed +process). T-G38's real `pg_terminate_backend` kills and T-G54's +restricted-role fixtures moved AC-44 from `deferred` to `covered`, so the +final dispositions are 8 executed, 30 covered, 16 partial and 6 deferred. +T-G75 re-checked all 60 matrix rows against `32ae51ca` and kept those +dispositions (AC-01 and AC-44 `covered`, AC-57 `partial`). + +## 5. Known gaps and limitations + +These are the gaps that remain on `32ae51ca`; tasks that closed an earlier gap +are recorded in §7 instead. + +- **Clippy clean (fixed by G58/G63/G71).** The gap is closed: G58/G63 fixed + the former 11 pre-existing warnings and G71 fixed the last two; T-G75 + re-ran + `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` + and it exits 0 with zero warnings on the lib and test targets. +- **Branch-delete CLI deployed (fixed 2026-11-15).** The gap is closed: the + 2.0.0 build `4e65ebbae` is live with host CLI `1dc15838…`, bundle `layerfs` + `c4737445…`, `layerfs-daemon` `b8dd0766…` and + `layerfs-sandbox-helper` `3d033ec0…`; `layerfs help branch` ends with + `branch remove `, and a probe against a missing name + returns `NoSuchBranch`. Operators use `layerfs branch remove` directly; MCP + `branch_remove` and the SDK method stay equivalent. The cutover opened the + Store with the one-way GC schema migration: the sqlite file grew + 1,111,203,840 -> 1,111,310,336 B (live hash `d5d65e3b…`) on first open. +- **PG `physicalShrink` is a keyword proxy.** PostgreSQL computes + `physical_shrink` from the `pg_total_relation_size` delta of `pack` and + `object_location` around the swap (an in-place rewrite), not from file-level + bytes; `backendReusable = 0` with `postgresql-in-place` names that absence. + SQLite keeps the file-level measure. +- **Real `kill -9` of the production compaction process (T-G28/T-G60/ + T-G65; AC-01 `covered`).** `sqlite_compact_crash.rs::real_process` spawns a + separate process on the fixture Store and the parent sends `SIGKILL` while + it is stopped at the failpoint; T-G60 extended those arms to the mark + (`BeforeSeal`/`AfterSeal`; the `BeforeSeal` kill keeps the writer + session's WAL sidecars, which the next open replays), copy + (`BeforeVacuum`, immediately before `VACUUM INTO` because SQLite runs that + copy as one uninterruptible call), verify (`DuringVerify`) and rename + (`DuringRename`). Those arms kill a re-executed test child, so T-G65 adds + the production process: + `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs` starts the + built `layerfs store compact` command, the engine's gate stops it at the + named failpoint, and the test `SIGKILL`s it there, then reopens through the + CLI and asserts `G`/`G + 1`, the artifact set and sidecars, the + `store.status` phase/`uncertain`/`unresolved`, the settling next run, the + retained layer and `doctor`. No daemon serves Store calls (#352 F-020), so + this is the process that performs the native compaction; the long-lived + `layerfs worker` that serves a machine's Store is still not the killed + process. +- **Capacity model: 10 GB and 50 GB measured (T-G61, T-G66).** + [`gc-capacity-model.md`](gc-capacity-model.md) rests on a real 9.987 GB + Store measurement (`sqlite_compact_capacity.rs#capacity_measure`): `none` at + 8.88% dead (3 runs, median 149.318 s) and `rebuild` at 10.00% dead (3 runs, + median 246.371 s; mark 149.089 s, build 94.317 s, verify 0.312 s, install + 1.916 s; dead 1.011 GB, shrink 1.017 GB, peak temp 9.144 GB, 1.90x total + footprint), plus one WAL/FULL rebuild (284.078 s). T-G66 then measured the + **50 GB** arm (one run per arm, `sqlite_compact_capacity_measure_50gb.json`): + 49.999 GB pack bytes / 50.863 GB physical, `none` at 8.876% dead + (1,633.585 s) and `rebuild` at 10.000% dead (5,131.835 s; mark 784.251 s, + build 4,254.371 s, verify 85.557 s, install 7.311 s; dead 5.062 GB, shrink + 5.100 GB, old-formula receipt peak temp 45.784 GB). The 50 GB numbers are + **not** linear from 10 GB: the build phase is 93.0 s/GB new instead of 10.3, + so the rebuild is 85.5 min instead of the [E] ~20 min, and the sampled real + peak is 143.1 GB (~2.9x the Store). T-G72 replaced the old `new + visited` + receipt with a bound over the staged copy, the new file, the rollback + journal and the visited scratch: **147.609 GB (2.90x the old file)** for the + same 50 GB `rebuild`, 3.2% above the sampled peak, tracked as + [layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + The 30% dead / 50 GB-live arms and the `ACCESS EXCLUSIVE` duration remain + [E]. +- **PostgreSQL `--test-threads`.** T-G14 gave each `postgresql_compact` test + its own catalog binding (`postgresql-compact-`), so its 20 tests pass + with the default test threads (three consecutive runs); the Store-scoped + advisory key is no longer shared. Other PG suites that still share one + schema or session state keep `--test-threads=1` until they get the same + per-test binding. +- **`.git/index` merge rule.** `/.lww` lists `/.git/index` after + `%include .gitignore`, so merging a Branch forked from an older layer keeps + the newer index whole instead of conflicting on the binary file. Only this + generated file is added; no source file is listed. +- **Branch cleanup is soft and bounded (updated 2026-11-15).** `branch.remove` + is live, but removal is a marker write: the name stays taken (a re-fork is + `NameTaken` with holder `state: "retired"`, reads are `NoSuchBranch`) until + `store.compact` collects the rows, and `main` is `ReservedName`. A Branch + with a live mount is `MountsActive`, and a Branch pinned by a retained Layer + or `commit.branch_id` also keeps `NameTaken`; a rebuild needs an idle Store. + The 2026-11-15 cleanup retired 145 Branches (20 empty + 125 superseded) and + deliberately left 162 (unknown or non-empty); T-G75 re-probed the live CLI + on `32ae51ca` (a missing name is `NoSuchBranch`) and counted 174 live + Branches, the 162 that cleanup left plus later gap-fix task Branches. The + remaining work is data judgment, not a missing capability. +- **`project.remove` retired.** Old clients get `InvalidInput`, MCP reports an + unknown tool, and the CLI exits 2; there is no compatibility alias. +- `remove_file` reclaims no bytes in 2.0; a path-aware walk is a follow-up. + PostgreSQL ACL replay is best-effort, and `markBytes` is a #52 metric, not a + report field. Compaction metrics are process-local with no SDK or CLI + renderer, and PostgreSQL compactions have no metrics yet. +- AC-60's real standby arms and the reserved-pack race of AC-42 are deferred; + AC-28/41/43 have T12.3b evidence and AC-35's advisory-key refcount does not. + +## 6. How to run + +``` +# SQLite compaction lane, on a filesystem that carries SQLite locks +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --full +cargo test -p layerfs-persistence --test sqlite_compact +cargo test -p layerfs-persistence --test sqlite_compact_crash +cargo test -p layerfs-persistence --test sqlite_compact_fixtures +cargo test -p layerfs-persistence --test sqlite_compact_matrix +UPDATE_GOLDEN=1 cargo test -p layerfs-persistence --test sqlite_compact_fixtures + +# PostgreSQL, one thread +LAYERFS_TEST_POSTGRES_URL="host=127.0.0.1 port=55432 user=postgres dbname=postgres sslmode=disable" \ + cargo test -p layerfs-persistence --features postgresql --test postgresql_compact -- --test-threads=1 +``` + +The ext4/xfs/btrfs requirement is the kernel guard +(`engine/crates/layerfs-store-service/src/kernel_guard.rs`): tmpfs, overlay, +FUSE and network mounts are refused `Unsupported` because they cannot carry +the WAL locks and the directory flock. Inside a Docker Sandbox the run +directory (`$HOME`, `/layerfs/data/machine/native/run/`) is ext4; +`/tmp` is tmpfs there. The PG value is a libpq keyword string; +`postgresql_diagnostic` needs the URI form. + +## 7. Commit and branch index + +Main layers are publications; heads are the saved Branch tips. `T12.3b` was in +flight at the baseline and is now published on main (`32223b46`, tidy `32927884`). + +| Epic | Branch | Head commit | Main layer | Result | +|---|---|---|---|---| +| T1 schema and position | task-t1.2d-merged, task-t1.7-merged3 | 12eee0da5b, 12fb3019bd | 32b602714d, 32860cbe46 | markers, intents, serials, epoch, trace salt, backfill | +| T2 history contract | task-t2.4-ctors-test-merged3 | 12ebeb2dfb | 32c5500017 | typed `NameTaken`/`Removed`, catalog stubs | +| T3 `branch.remove` | task-t3-final5 | 12dabb4673 | 3256c829a6 | `branch_retired`, `ReservedName`, name lifetime | +| T4 `project.delete` | task-t4.6-merged | 12e8a214b4 | 32aca049c3 | `layer_stack_retired`, sidecar, retry | +| T5 `project.remove_file` | task-t5.8 | 12f58625e0 | 3299d92d08 | `path_removed`, frontier, whiteouts, grammar | +| T6 admission and epoch | task-t6.5-merged4 | 12ebc5d982 | 32e895c4eb | `gc_intent`, codes, epoch fence, PG advisory keys | +| T7 mark and closure | task-t7.2 | 12a41bf33f | 3240699d11 | mark tuple, closures, `kept_node`, digest | +| T8 SQLite rebuild | task-t8.1-merged | 122151d179 | 323465c8b3 | seal, visited, verify, rename install, failpoints | +| T9 PostgreSQL rebuild | task-t9e | 1225694fd7 | 32c9181e3d | phases M/A/B/C, `gc_op` resume, standby | +| T10 report, status, metrics | task-t10.1b, task-t10.3-final | 121262d292, 1236755412 | 32515e1b2a, 3271a46892 | §11 report, status op, `compact_metrics` | +| T11 API, MCP, CLI, SDK | task-t11.3-final | 1271019370 | 3218fea5bc | 76 calls, admin profile, `project.remove` retired | +| T12.1 SQLite fixtures | task-t12.1-merged2 | 12db7c4ea4 | 32444eb25c | golden receipt, §11 fields, digest vector | +| T12.2 SQLite crash matrix | task-t12.2-merged2 | 12433023b7 | 32502588ab | failpoints, artifact set, `StaleStore` | +| T12.3a SQLite matrix | merged3-t12.3a | 1277362ac5 | 327d6a42bd | 5 executed / 24 covered / 16 partial / 8 gaps | +| T12.3b PostgreSQL acceptance | task-t12.3b-final | 1210405949 | 32223b46 | PG-half matrix, 15 binaries / 73 passed (published) | +| T13 spec gaps and runbook | task-t13b, task-g17-publish | 12c0efe9dd, 12dcce8e15 | 32ab564277, 3284fb82 | G1-G6 decisions, operator runbook, testing docs | +| T-G13 default-feature build | task-g13-pub2 | 1299885ee9 | 32971532c5 | PG imports gated behind the feature | +| T-G28 real-process SIGKILL | task-g28-merged | 12106c4dd7 | 32a7d0cf07 | 4 SIGKILL arms; AC-01 `partial` | +| T-G52 SQLite crash re-run | task-g52 | 12c002e409 | 32bf908cf5 | 20/18/1/1, matrix 8 arms | +| T10.5 PG `store.status` | merged-t10.5c | 12f80d29df | 328a16a4c9 | persistence `postgresql_status` 3, control 1 | +| T-G38 PG reply loss | task-g38b-merged | 128162224e | 32dc46a25a | real backend kills and lost `COMMIT` | +| T-G53 PG status control | g53-merged | 12876c4f1f | 322a2c59a1 | control `postgresql_status` +1 | +| T-G54 PG privileges | g54-merged | 12f5197908 | 32a2621dbb | AC-44 `covered`; `postgresql_compact` 20 | +| T-G55 doctor physical | g55-integ | 122880b8b6 | 3285f0d588 | 16 checks | +| T-G57 closeout base | g57-merged | 12babcbaea | 32df79da | T-G57 publication | + +The docs-only review rounds A-M are on `gc-phase-a` ... `gc-phase-m` and the +`gc-schema`/`gc-contract` branches; they changed no product contract. The +post-closeout gap fixes are published as `task-g58-merged2` (G58/G63 clippy), +`task-g63`, `task-g64`, `g61-integ` (capacity harness), `task-g65-merged` and +`task-g65-merged2` (real-process kill), `task-g67` (physical doctor proxy), +`task-g69` (runbook and cleanup), `g70-merged` (report), `task-g71` (final +clippy), `task-g66b` (50 GB measurement), `task-g72` (peak-temp bound) and +`task-g74` (report generator); T-G75 audits those layers. + +## 8. Closeout + +- **Final main layer.** `32ae51ca` (root `d0b744cb`), the layer this audit + certifies; it is the T-G74 publication (commit `128899cd74`). The chain from + the previous closeout is T-G57 `32df79da` -> T12.8 `3255cfb1fa` -> T-G60 + `32284e2d4c` -> T-G59 `327b250e45` -> T-G58 `329abfa4b4` -> T-G63 + `32e51aa657` -> T-G64 `32feb7df87` -> G61 `32977da52f` -> T-G67 + `327892e625` -> T-G69 `32cfd1e67a` -> T-G65 `3299d19429` -> `32957696ef` + -> G70 `32bb6d7557` -> G71 `32f89ff7b8` -> T-G66b `32c363ad8a` -> T-G72 + `327aa16ab0` -> T-G74 `32ae51ca`. This closeout publishes as main's next + layer. +- **Shipped.** Four admin-only, ids-only calls on SQLite and PostgreSQL: + `branch.remove`, `project.delete`, `project.remove_file` and + `store.compact`; the §4.1 additive schema and deterministic backfill; the + `gc_intent`/`gc_op` admission intent, the epoch fence and the §6.1/§6.2 + mark with the §6.4 digest; the SQLite seal/visited/verify/rename rebuild + with the 10% `none`/`rebuild` threshold and named failpoints; the + PostgreSQL Phase M/A/B/C rebuild with `gc_op` resume, ACL and the standby + predicate; the §11 `CompactReport`/`CompactStatus`, `store.status` and + `compact_metrics` (SQLite phases plus `pg_mark`/`pg_build`/`pg_swap`/ + `pg_cleanup`); the 76-call API/MCP/CLI/SDK table with `project.remove` + retired; and the ext4 harness plus report-only CI lane. +- **Final test evidence on `32ae51ca` (T-G75 re-run).** 271 passed / 0 failed + / 3 ignored (`layerfs-persistence`; T-G24 recorded 264/0/2), 214/0/15 + (`layerfs-store-service`) and 59/0/0 (`layerfs-store`), 544 passed / 0 + failed / 18 ignored in all. The five SQLite compaction targets re-ran to + `sqlite_compact` 20, `sqlite_compact_crash` 23, `sqlite_compact_fixtures` + 1, `sqlite_compact_matrix` 1 runner (8 arms) and `sqlite_compact_capacity` + 1 passed + 1 ignored (the opt-in 50 GB arm). The PostgreSQL lanes were not + re-run here (this Sandbox has no server): 20 `postgresql_compact` (T-G54), + 3+1 `postgresql_status` (T10.5/T-G53), 5 `postgresql_compact_fixtures` + (T12.3b) and 16 doctor-physical checks (T-G55) stay as recorded. The full + table is in §3. +- **Clippy clean (G58/G63/G71).** G58/G63 fixed the former 11 pre-existing + warnings and G71 fixed the last two; T-G75 re-ran + `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` + to exit 0 with zero warnings on the lib and test targets. +- **Matrix after T12.3c, T-G28/T-G60, T-G38/T-G54 and T-G65.** 60 cases: 8 + `executed`, 30 `covered`, 16 `partial`, 6 `deferred` (all + PostgreSQL-only), 0 SQLite evidence gaps, 0 `pending`; T-G75 re-checked all + 60 rows on `32ae51ca` (AC-01 and AC-44 `covered`, AC-57 `partial`). +- **Remaining partial (16).** AC-02, 04, 11, 15, 19, 20, 21, 23, 30, 32, + 34, 37, 53, 54, 56, 57. Why: the SQLite half is proven but the PostgreSQL + half is T12.3b's or not named (AC-02/30/32/34), the full form/read-surface + table is not one test (AC-19/53), the named arm is not inducible or does not + exist (AC-11's inapplicable-whiteout branch, AC-34's failed intent delete), + or the E3 scale/three-run arms remain (AC-04, 56, 57). +- **Remaining deferred (6, PostgreSQL-only).** AC-28, 35, 41, 42, 43, 60. + T12.3b executes the PG halves of AC-28/41/43; AC-35 (advisory-key refcount) + and AC-42 (reserved-pack race) have no deterministic pause point, while + AC-60 (a real standby) needs server control the harness does not have. +- **Capacity model.** [`gc-capacity-model.md`](gc-capacity-model.md) records a + real 9.987 GB measurement ([M]: `none` 149.318 s, `rebuild` 246.371 s median, + phase sums and the T-G72 peak bound in + `sqlite_compact_capacity_measure.json`) and the real 49.999 GB measurement + ([M], one run per arm, phase sums and 5.100 GB shrink in + `sqlite_compact_capacity_measure_50gb.json`; 85.5 min rebuild, sampled peak + 143.1 GB). T-G72 replaced the old `new + visited` receipt with a documented + bound over the staged copy, the new file, the rollback journal and the + visited scratch: **147.609 GB (2.90x the old file)** for the 50 GB + `rebuild`, 3.2% above the sampled peak, tracked as + [layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + AC-57 stays `partial` for the section 13 30% dead / 50 GB-live arms and the + PostgreSQL arms. +- **Branch-removal cutover and cleanup (2026-11-15, main `327892e6`).** + `branch.remove` is live end to end. Build identity: version `2.0.0`, + commit `4e65ebbae`; host CLI `1dc15838…`, bundle `layerfs` `c4737445…`, + `layerfs-daemon` `b8dd0766…`, `layerfs-sandbox-helper` `3d033ec0…`. The + Store was opened once by the new build (one-way GC schema migration, + 1,111,203,840 -> 1,111,310,336 B, live hash `d5d65e3b…`). Probe: + `layerfs branch remove … __deploy_probe_missing__` is `NoSuchBranch`; + `help branch` lists `remove`. Semantics are unchanged (§4.2): the marker + hides the Branch, `branch remove … main` is `ReservedName`, and the name + stays taken until `store.compact` rebuilds. Retirement: 307 -> 287 (20 + empty) -> 162 (125 superseded), 145 total, 0 refusals; 162 remained by + choice (unknown or non-empty, deliberately untouched). T-G75 re-probed the + live CLI from `32ae51ca`: `help branch` lists `remove`, a missing name is + `NoSuchBranch`, and the Store holds 174 live Branches (the 162 that cleanup + left plus later gap-fix task Branches). Backup: + `/home/dell/lfs-work/backup-20261115-180517` (earlier + `/home/dell/lfs-work/backup-20261115-180149`); rollback was not needed. + `layerfs doctor` is `{"ok":true,"findings":[]}`, and no Sandbox/Workspace of + the cutover or cleanup remains. +- **Validation checklist (T12.8, re-verified T-G75).** Every artifact below + exists on `32ae51ca`; rows 12-13 record the 2026-11-15 cutover and are + re-verified on main `32ae51ca`: + + | # | Artifact | Path | Check | + |---|---|---|---| + | 1 | implementation report | `core/docs/issues/project-41/gc-implementation-report.md` | present; updated by T-G75 | + | 2 | acceptance matrix | `core/docs/issues/project-41/gc-acceptance-matrix.md` | present; 60 rows, 8/30/16/6 | + | 3 | capacity model | `core/docs/issues/project-41/gc-capacity-model.md` | present; 10 GB (3 runs) and 50 GB (1 run) measured [M]; T-G72 peak bound 147.609 GB (2.90x); 30% dead / 50 GB-live and PG arms [E] | + | 4 | 76-call coverage | `docs/architecture/api/calls/coverage.md` | generated block: 76 calls, 10 families, no front-end problems | + | 5 | ext4 harness | `tools/ci/test_store_ext4.sh` | present, executable | + | 6 | CI lane | `.github/workflows/store-ext4.yml` | present; `quick` and `full-postgresql` | + | 7 | PG phase metric | `engine/crates/layerfs-persistence/src/compact_metrics.rs` | `PostgresMark => "pg_mark"` | + | 8 | real SIGKILL arms | `engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs` | `sigkill_before_seal`, `sigkill_after_seal`, `sigkill_during_copy`, `sigkill_during_verify`, `sigkill_during_rename`, `sigkill_after_build`, `sigkill_before_install`, `sigkill_after_install`, `sigkill_during_settle` | + | 9 | privilege check | `engine/crates/layerfs-persistence/src/backend/postgresql/access.rs` | `compaction_missing` checks both `CREATE`s and `pg_read_all_stats` | + | 10 | PG conninfo helper | `engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs` | present | + | 11 | PG status control | `control/crates/layerfs-store/tests/postgresql_status.rs` | present; 1 test | + | 12 | `branch.remove` live | host CLI `layerfs branch remove` | live on 2.0.0/`4e65ebbae`, re-probed T-G75; `help branch` lists `remove`; probe on a missing name returns `NoSuchBranch` | + | 13 | doctor clean after cutover | `layerfs doctor` | `{"ok":true,"findings":[]}`; 145 Branches retired (20 empty + 125 superseded), 162 remained then and 174 are live on `32ae51ca`; removal is soft, so a name returns only after `store compact` | + | 14 | clippy clean | `cargo clippy -p layerfs-persistence -p layerfs-store-service --all-targets --features postgresql --no-deps -- -D warnings` | T-G75: exit 0, zero warnings | + | 15 | final suite re-run | `cargo test -p layerfs-persistence -p layerfs-store-service -p layerfs-store --no-fail-fast` | T-G75: 544 passed / 0 failed / 18 ignored; the five SQLite compaction targets keep 20/23/1(+1 ignored)/1/1 | + + `git diff --check` is clean on the closeout Branch. +- **How to run.** + `LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick` + (or `--full`, adding `--features postgresql` when `LAYERFS_TEST_POSTGRES_URL` + is set) needs ext4/xfs/btrfs. `cargo test -p layerfs-persistence --test + sqlite_compact_matrix` runs the T12.3c arms; `UPDATE_REPORT=1 cargo test -p + layerfs-persistence --test sqlite_compact_matrix` regenerates the report. + The opt-in capacity arm is + `CAPACITY_MEASURE=1 UPDATE_CAPACITY=1 cargo test -p layerfs-persistence + --release --test sqlite_compact_capacity -- --ignored --nocapture` with a + TMPDIR of >=2.2x the target; it writes + `tests/fixtures/sqlite_compact_capacity_measure.json`. + `.github/workflows/store-ext4.yml` is the report-only CI lane. diff --git a/core/docs/issues/project-41/gc-runbook.md b/core/docs/issues/project-41/gc-runbook.md index b976397931..2a238425c6 100644 --- a/core/docs/issues/project-41/gc-runbook.md +++ b/core/docs/issues/project-41/gc-runbook.md @@ -88,7 +88,10 @@ installed → it records the outcome. A dead owner's op is adopted; only a live `commit.branch_id` keeps `NameTaken` (§4.4); `remove_file` reversal does not unhide at-or-below versions; `rebuild` install is irreversible (§9). Content recovery is `backup.create`/`backup.restore` from before compaction (SQLite; the server's backup for PG); - no undo copy is kept (§9, #155 D-24). + no undo copy is kept (§9, #155 D-24). The 2026-11-15 branch cutover keeps its + pre-change home and Store in `/home/dell/lfs-work/backup-20261115-180517` + (earlier `backup-20261115-180149`); restore that backup rather than editing + markers (see §7). - `project.remove` is gone; an old client gets `InvalidInput` (§4, §10). ## 5. Known gaps at this baseline @@ -112,3 +115,67 @@ installed → it records the outcome. A dead owner's op is adopted; only a live 4. Compare artifact/schema with the op; leave anything a record names to the next compact (`G` removes, `G+1` records). 5. Never hand-edit `gc_intent`, `gc_dead`, markers or `project_epoch`; never unlink `-wal`/`-shm`; never drop a schema `gc_op` names. 6. To re-use a name after a removal, re-create after the frontier (or after a rebuild frees it); don't delete markers. Escalate if old content must be restored. + +## 7. Branch retirement (`branch.remove`, live since 2026-11-15) + +`branch.remove` is live on the 2.0.0 build `4e65ebbae` (host CLI `1dc15838…`, +bundle `layerfs` `c4737445…`, daemon `b8dd0766…`, helper `3d033ec0…`); +`layerfs help branch` lists it, and a probe on a missing name returns +`NoSuchBranch` (exit 125). The call is a marker write, not a delete: it hides +the Branch, keeps its rows and its name until a later `store.compact` rebuild +collects the marker, and never touches `main` (`ReservedName`). + +**Preflight.** + +1. `layerfs sandbox list ` is empty, or every listed Sandbox belongs + to a task you have agreed to exclude; a mounted Branch is `MountsActive`. +2. `layerfs doctor` is `{"ok":true,"findings":[]}`; `machine status docker` is + `running`. +3. Take a backup; the cutover used + `/home/dell/lfs-work/backup-20261115-180517` (bundle, host, daemon, helper, + `bundle.json`, `worker.json` and the Store volume; the earlier one is + `backup-20261115-180149`). Keep it until the retired names are no longer + needed. +4. Page the inventory: `layerfs branch list --page-size 100 + --page-pos `; the last page has `has_more: false`. Record the + names and the count before and after. +5. Build the retire list: an empty Branch (`head: null`), or an explicitly + reviewed superseded Branch (typically a `-merged`/`-integ`/`-final`/ + `-publish` variant) whose content is already on `main` or no longer needed; + `layerfs branch diff main --paths-only` shows the paths. + Never include `main`, a running task's Branch, or a Branch with a live + mount. + +**Procedure.** + +- For each name: `layerfs branch remove `. One call per + Branch; there is no batch, force or dry-run form. `Busy` is another + operation holding the Store scope (wait, call once more); `MountsActive` is + a live mount; `NoSuchBranch` means already gone; `ReservedName` is `main`. + Record every refusal. +- Verify: the retired name reads `NoSuchBranch`, a re-fork is `NameTaken` with + the holder's `state: "retired"`, `branch list` no longer shows it, and the + `main` head is unchanged. +- **Soft name retention.** The name is not reusable until a rebuild collects + the `branch_retired` rows; plan around that and never hand-delete markers + (§4). `store.compact` below is the only supported reclaim. + +**Reclaim (optional, later).** `layerfs store compact ` only when the +Store is idle (no mounts, daemon or other op). A rebuild copies retained rows +into a fresh Store and installs `G + 1`; only then are retired names and their +rows collected. A `none` outcome (<10 % wholly dead) keeps everything, and the +retired names stay taken. Follow the compaction procedure in §6 and the +rollback limits in §4. + +**Rollback.** Restore the whole home and Store from the pre-cutover backup +above (or `backup.restore` from a `backup.create` taken during the window); +markers cannot be undone by dropping tables (§4). The 2026-11-15 cutover was +not rolled back: no call failed, and `main` and `Cargo.toml` were byte-identical +before and after. + +**2026-11-15 result.** 145 Branches retired (20 empty at cutover + 125 +superseded), 0 refusals; 162 remained by choice (unknown or non-empty, +deliberately untouched). `main` stayed `32977da5…` (root `56dbe645…`), +`layerfs doctor` stayed clean, and no Sandbox/Workspace of the cleanup was +left running. The new build's one-way GC schema migration grew the sqlite +Store 1,111,203,840 -> 1,111,310,336 B on first open. diff --git a/core/docs/issues/project-41/gc-t10-report-status.md b/core/docs/issues/project-41/gc-t10-report-status.md index 48df05d38d..e8646ff64d 100644 --- a/core/docs/issues/project-41/gc-t10-report-status.md +++ b/core/docs/issues/project-41/gc-t10-report-status.md @@ -74,8 +74,71 @@ backend-only field, so PostgreSQL reports its zeros through the same keys wire change. `store.status` reads the Store-side `gc_op` at the same point in `status.rs`; nothing is read from a file there. +## Metrics and SLO signals (T10.3, PostgreSQL T10.4) + +`engine/crates/layerfs-persistence/src/compact_metrics.rs` publishes the +compaction observations from the real `store.compact` and `store.status` +paths: + +- counters: `compact.runs`, `compact.operations{outcome}`, + `compact.refusals{code}`, `compact.settles{kind}`, + `compact.retries{reason}`, `compact.status{state}` and + `compact.slo{objective,result}` (`residue_free`, `generation`); +- duration aggregates: `compact.phase.duration_ns{phase,statistic}` for + `mark`/`build`/`verify`/`install` on SQLite and + `pg_mark`/`pg_build`/`pg_swap`/`pg_cleanup` on PostgreSQL, and + `compact.duration_ns{statistic}`, each with `count`, `sum` and `max`; +- the last successful run's §11 gauges: `compact.dead_bytes{kind}`, + `compact.kept_waste{part}`, `compact.kept_waste_bytes`, + `compact.residue_bytes` and `compact.generation{which}`. + +Labels come only from closed enums: outcome, phase, refusal code, settle kind, +retry reason, status state, SLO objective/result, statistic and report key. A +Store name, path, operation id or artifact name is never a label, so metric +cardinality is fixed. A SQLite `none` run times only `mark` and a PostgreSQL +`none` run only `pg_mark`; a rebuild times all four phases its backend names; +a failure or unknown run records its outcome and total but publishes no report +gauge; a run that settles an earlier leftover adds `leftover` to +`compact.settles` and `compact.retries`. A `Busy`/`Unresolved` refusal that +stops a SQLite call before a run is counted once by the control path, and a +refusal a measured run itself produces (either backend) is counted by that +run. + +PostgreSQL §8 emits the same observation from +`backend/postgresql/compact.rs`: its `RunMetrics` times Phase M/A/B/C as +`pg_mark`/`pg_build`/`pg_swap`/`pg_cleanup`. Those are distinct closed values, +not aliases of the SQLite phases, so a process total never blends a SQLite +phase with a PostgreSQL phase of another meaning. It records the same §11 +gauges from the PostgreSQL receipt (where `dead_bytes_refined == +dead_bytes_bound`, `backend_reusable = 0` and `residue` is empty) and maps +`Busy`/`BusyOp` to `compact.refusals{code="busy"}` and `Unresolved` to +`code="unresolved"`. A resumed `G + 1` leftover records Phase C, the receipt's +original `generation {before, after}` and the `leftover` settle/retry; a `G` +leftover is settled before the new round. + +The registry follows the telemetry crate's rules (fixed low-cardinality +labels, recording never changes a result). `store.compact` has no new public +metrics call: `snapshot()` reads the process totals and +`CompactMetricsCapture` captures one thread, which the tests use. + ## Tests +- `engine/crates/layerfs-persistence/tests/compact_metrics.rs`: a successful + SQLite rebuild (all four phase durations, outcome, + dead/kept-waste/residue/generation gauges and both SLO checks), the `none` + path (only `mark` timed, generation kept), a stopped `AfterBuild` run + followed by the leftover settle (`uncertain` outcome, then `rebuild` + + `retries{leftover}` + `settles{leftover}`), and a definite failure with no + report gauge. +- `engine/crates/layerfs-persistence/tests/postgresql_compact.rs`: the same + capture on a real PostgreSQL Store: a successful rebuild (the four `pg_*` + phases, outcome and the same report gauges and SLO checks), the `none` path + (only `pg_mark`), a Busy swap refusal (`failed` + + `refusals{code="busy"}`, phases M/A/B, no report gauge) and an uncertain + leftover intent refusal (`failed` + `refusals{code="unresolved"}`, no phase, + no report gauge). +- `engine/crates/layerfs-persistence/tests/support/metrics.rs`: the one + closed label-key/value assertion both capture tests use. - `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: `the_report_projects_every_section_11_field`, `the_empty_store_report_explains_its_zeros`, @@ -86,11 +149,32 @@ wire change. `store.status` reads the Store-side `gc_op` at the same point in - `control/crates/layerfs-store/tests/compact_report_status.rs`: the report wire is the snake_case contract shape (including `zero_reasons` and the `delta_bases.references` hex), and `store.status` projects an idle Store, - a live owner's record and a leftover one (`op`, `state`, `phases`). + a live owner's record and a leftover one (`op`, `state`, `phases`), and the + `compact.status{state}` metric of each read. +- `engine/crates/layerfs-persistence/tests/postgresql_status.rs` (T10.5): on a + real PostgreSQL Store, an idle Store projects null; a live non-idle owner + projects `busy` with `uncertain`; a gone owner projects `unresolved`; each + `mark`/`build`/`swap` failpoint projects its recorded phase from the scratch + name after `gc_intent` is released; and a completed run leaves no row. +- `control/crates/layerfs-store/tests/postgresql_status.rs` (T10.5): the same + projection through the control `store.status` call on a real PostgreSQL + Store: no row projects `idle` with `op`/`phases` null; a live + `pg_stat_activity` owner projects `busy`; the gone owner projects + `unresolved`; each of `mark`/`build`/`swap`/`cleanup` is projected from the + row; the bounded `compact.status{state}` samples are emitted once each; and + a Store whose row was removed is idle again. ## Gaps -- T9's PostgreSQL rebuild and its `gc_op` status read are not implemented; - the trait and the status comment are the only PostgreSQL pieces. +- The compaction metrics are emitted into the process registry only; no SDK + call or CLI command renders `snapshot()` yet. +- PostgreSQL compaction metrics cover the run and its admission/refusal paths. + T10.5 supplies the PostgreSQL `store.status` source: `status.rs` reads the + Store-side `gc_op` with its `gc_intent` on one status query (no Store open) + and projects `busy`/`unresolved` and the `mark`/`build`/`swap`/`cleanup` + phases, so `compact.status{state}` now has both sources. +- No control-level test joins `store.compact` and `store.status` in one run: + the control case writes the rows a compaction leaves and checks the + projection, while the persistence suites run the compactions themselves. - The `{value, cap}` capped-field shape is not produced: the SQLite receipt has no cap today. diff --git a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md new file mode 100644 index 0000000000..f3b4ce8c9a --- /dev/null +++ b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md @@ -0,0 +1,203 @@ +# T12.3a: SQLite acceptance-matrix runner report + +> Generated by `cargo test -p layerfs-persistence --test sqlite_compact_matrix`. +> The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) section 12 and +> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) AC-01..AC-60 win; this +> page records evidence and adds no contract. Regenerate with +> `UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix`. + +## Scope and counts + +- Rows: 60 (AC-01..AC-60); backend split: both 47, SQLite-only 6, PG-only 7. +- Executed here: 8; covered by a named existing test/fixture: 29; partial: 16; deferred to T12.3b: 7; SQLite gaps: 0. +- **T-G28/T-G60 follow-up.** T-G28 later added the real-process SIGKILL arms + and T-G60 extended them to mark, copy, verify and rename, so + `sqlite_compact_crash` now has 23 tests (9 real-process SIGKILL arms and the + child placeholder), not the 13 recorded below. Those arms still kill a + re-executed test child, so T-G65 added + `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs`: 8 SIGKILL + arms on the built `layerfs store compact` process at mark, build, verify and + install, so AC-01 is `covered` (the matrix now reads covered 30 / partial + 16). This page keeps the T12.3c run's original counts. +- A `covered`/`partial` row names a file and test function that this runner + asserts exists; an `executed` row names an arm this binary really runs. +- A cross-backend row records its SQLite half only; every PostgreSQL half + and the PostgreSQL-only rows are deferred to T12.3b by name. +- **T-G61 capacity follow-up.** A separate opt-in test, + `sqlite_compact_capacity.rs#capacity_measure` (ignored by default), measured a + real **9.987 GB** Store: `none` at 8.88% dead (three runs, median 149.318 s) + and `rebuild` at 10.00% dead (three runs, median 246.371 s; mark 149.089 s, + build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink + 1.017 GB, peak temp 9.144 GB), plus one WAL/FULL rebuild (284.078 s). The + numbers are in `tests/fixtures/sqlite_compact_capacity_measure.json`; the + 50 GB arms and the 30% arm remain [E]. + +## Matrix (SQLite half) + +| Case | Backend | Matrix tier | Disposition | Evidence | Note | +|---|---|---|---|---|---| +| AC-01 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_after_verify; sqlite_compact_crash.rs#crash_after_install; sqlite_compact_crash.rs#crash_during_settle | The SQLite section 9 crash matrix through named failpoints; a real kill -9 of a separate daemon process is T12.2's in-process stop points, not run here. | +| AC-02 | both | E2 | partial | sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | The SQLite arm runs through the suite; the converted 1.0 Store arm and the PostgreSQL backend matrix are deferred to T12.3b. | +| AC-03 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_after_wal_switch; sqlite_compact_crash.rs#crash_before_install; sqlite_compact_crash.rs#crash_after_install | SQLite mark/build/verify/install and WAL-switch stop points; lost COMMIT and power-loss arms belong to the PostgreSQL/fsync halves. | +| AC-04 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Deterministic closure fixture plus a seeded six-history reachability property arm; the full random fork/commit/merge/publish/removal generator is not built here. | +| AC-05 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Full doctor (including the diagnostic scan) on the compacted Store, also in the property arm. | +| AC-06 | both | E2 | executed | this runner #arm_digest_row_after_collection | A Save's `file_digest` row named a root the rebuild collected; the row goes with its root and recording the same content again succeeds. The PostgreSQL half is T12.3b. | +| AC-07 | both | E2 | executed | this runner #arm_mount_and_compaction_exclude_each_other; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | SQLite half executed: bypass gives `StaleStore`, a mount blocks compaction (`MountsActive`, naming the holder) and a live compaction refuses a mount and a second compaction (`BusyOp`). The open one-shot SDK transaction arm is PostgreSQL, T12.3b. | +| AC-08 | both | E2 | covered | admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | One winner; the loser sees MountsActive, and a removed Branch is Missing. | +| AC-09 | both | E2 | covered | branch_removal.rs#a_removed_branch_is_hidden_and_the_epoch_bumps_once; branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches | Reserved main, absent Branch and holder details are asserted; project-side name collisions are in AC-30. | +| AC-10 | both | E2 | covered | path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Removed/UpToDate codes and the frontier; 2.0 frees no bytes for the removed path. | +| AC-11 | both | E2 | partial | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden; sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount | Non-FF whiteouts, the fast-forward control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute. A merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed. | +| AC-12 | both | E2 | covered | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit | A removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept, and only an ordinary edit conflict on it is reported. | +| AC-13 | both | E2 | covered | sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit | The earlier Commit on an unmerged Branch is hidden by the marker; a Commit after the frontier does not inherit the path. | +| AC-14 | both | E2 | covered | sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount; sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes | A Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount; its next Commit records the deletion. | +| AC-15 | both | E2 | partial | kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retention of the only-reaching-path object is proven; marker hiding in reads/listings and derived whiteouts are not in one named test. | +| AC-16 | both | E2 | covered | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | T12.1 pins a retained retired Branch name and a collected unreferenced one; the branch.remove-then-rebuild arm is exercised through those markers. | +| AC-17 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#a_delta_base_is_enqueued_and_reported | Depth-2 chain copied, pack ids preserved. | +| AC-18 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Collected digest row gone, retained one kept. | +| AC-19 | both | E2 | partial | gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers; path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Path-grammar refusals and the marker frontier are named; the full form table (symlink/hardlink, >4096 bytes, >256 components) is not one test. | +| AC-20 | both | E2 | partial | path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date; path_removal.rs#an_unknown_project_is_missing | UpToDate and never-existing paths are named; the re-creation and other-project arms are partial. | +| AC-21 | both | E2 | partial | epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back; epoch_fence.rs#a_concurrent_commit_and_removal_have_one_winner | One-shot writes racing removals roll back as Conflict; the merge-racing-remove_file arm is not named. | +| AC-22 | both | E3 | covered | sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible; epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back | A deterministic barrier starts a merge and a `remove_file` together for three rounds: the loser sees the marker or gets `Conflict` and retries with the whiteout, and a tree derived afterwards never shows the path. The PostgreSQL half is T12.3b. | +| AC-23 | both | E2 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Deleted-project rows and markers are collected by the rebuild; a dedicated path_removed cleanup arm is not named. | +| AC-24 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order; gc_backfill.rs#existing_serials_are_kept_and_new_ones_start_above_the_maximum | Serials backfilled, next_history_serial never falls. | +| AC-25 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retained branch_retired/layer_stack_retired markers survive; collected rows take their markers. | +| AC-26 | both | E2 | covered | project_delete.rs#retry_classifies_retired_live_and_absent; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Hidden, entry present, retry returns the stored deleted_ms. | +| AC-27 | both | E2 | covered | project_delete.rs#paging_skips_a_hidden_project; project_delete.rs#every_layer_stack_lookup_carries_the_marker_filter | Stale home entries stay out of list/lookup. | +| AC-28 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup and its failpoints: T12.3b. | +| AC-29 | both | E2 | covered | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved; admission.rs#a_mount_blocks_all_three_removals_until_released | Owner liveness and Unresolved settlement; machine_stopped and the other-home intent are partial. | +| AC-30 | both | E2 | partial | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect | Holder detail and refusal codes are named; the full exact-message matrix (AC-55) is partial. | +| AC-31 | both | E2 | covered | gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Byte-count path arguments and the keyed trace hash are asserted. | +| AC-32 | both | E2 | partial | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved | Live/uncertain ownership is named; live-but-unreachable, machine_stopped and another home's intent are partial. | +| AC-33 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_sidecar_at_install | A live reader makes wal_checkpoint(TRUNCATE) Busy with no artifact and no op; a -mj* super-journal at install is Conflict with no writes. | +| AC-34 | both | E2 | partial | sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active | A committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains. The failed-delete sub-arm is not inducible on SQLite: the storage schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register too; the explicit `Cleanup` report is covered by the seam's code path, not executed here. The PostgreSQL half is T12.3b. | +| AC-35 | PG | E2 | deferred (T12.3b) | - | PostgreSQL advisory-key refcount: T12.3b. | +| AC-36 | both | E2 | covered | project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry; project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry; project_delete.rs#an_absent_stack_finishes_a_stale_entry | A lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent. The PostgreSQL half is T12.3b. | +| AC-37 | both | E2 | partial | gc_trace.rs#the_redacted_set_covers_every_path_bearing_call; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Redaction and the keyed hash are named; salt backfill on an upgraded Store is partial. | +| AC-38 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_during_settle | The SQLite artifact set is removed exactly; the PostgreSQL schema/name arms are T12.3b. | +| AC-39 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order | One project_epoch row per stack, gc_op not copied, next_history_serial never lowered. | +| AC-40 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#the_digest_is_stable_and_moves_with_a_head | SQLite live_heads_digest is identical before/after and moves with a head; PostgreSQL agreement is deferred. | +| AC-41 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase M mark transfer: T12.3b. | +| AC-42 | PG | E2 | deferred (T12.3b) | - | PostgreSQL reserved pack/Phase B: T12.3b. | +| AC-43 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup resume: T12.3b. | +| AC-44 | PG | E2 | deferred (T12.3b) | - | PostgreSQL privileges and lost COMMIT reply: T12.3b. | +| AC-45 | SQLite | E1 | covered | sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none; sqlite_compact.rs#group_rows_indexed_layout_opens_compacts_and_takes_none; sqlite_compact.rs#acquisition_layout_opens_compacts_and_clears_working_rows; sqlite_group_rows.rs#explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order | Monolithic/GroupRows/GroupRowsIndexed and the acquisition variants compact; working rows are cleared. | +| AC-46 | SQLite | E2 | executed | this runner #arm_sidecar_at_install; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | Sealed-state/sidecar change at install gives Conflict with no writes; a replaced Store gives StaleStore. The in-flight-transaction exclusion is a documented lane limit, not probed. | +| AC-47 | SQLite | E2 | covered | sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_during_settle; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Crash leaves the full set, the next compact removes it, none leaves no artifact, a stray name is Exists. | +| AC-48 | both | E2 | covered | sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path; sqlite_compact.rs#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold; sqlite_compact.rs#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold; sqlite_compact.rs#an_empty_store_takes_the_none_path; sqlite_compact.rs#a_capacity_failure_leaves_the_store_unchanged | 9/10/30 percent, empty-store zeroReason, capacity abort; verify-failure and re-refine arms are partial. | +| AC-49 | both | E2 | executed | this runner #arm_mount_during_compaction_is_busy; this runner #arm_commit_during_rebuild; admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | SQLite half executed: a mount during compact is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild gets `StaleStore` or lands, and its retry is never lost. The PG Phase A/Phase C arms are T12.3b. | +| AC-50 | both | E2 | covered | calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover; gc_calls.rs#the_four_gc_calls_take_ids_only | Folder scope refuses the four calls; the admin profile is the only allowance. | +| AC-51 | both | E2 | executed | this runner #arm_status_phases; compact_report_status.rs#status_projects_the_sidecar_phase_and_state; sqlite_compact_crash.rs#crash_before_seal | The op sidecar records mark/build/verify/install (runner); T10 projects it through store.status, and the named control test asserts the phase, state and phases list. | +| AC-52 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json; compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape | Every section 11 field pinned with units and sampling point (golden) and rendered by the T10 wire-shape test; markBytes is a metric, not a result field. | +| AC-53 | both | E2 | partial | project_export.rs#export_omits_a_path_hidden_by_remove_file; gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers | One hidden-path export case is named; the full read-surface table (read/stat/list/glob/diff/revision/edit/grep/blame/cp/apply) is not executed. | +| AC-54 | both | E2 | partial | restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install; restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it | Restore install guards and restored records are named; restore before/after a removal and the stale-entry listing arm are partial. The restore-window arm needs a lock-capable filesystem (ext4/xfs/btrfs), so on this Sandbox it reports Unsupported while the other two cases pass. | +| AC-55 | both | E2 | executed | this runner #arm_error_codes; this runner #arm_checkpoint_busy; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted | Exact SQLite texts asserted: Busy, Conflict("Store sidecar at install"), Exists("unrecorded compaction artifact"), Capacity("disk space"); the full engine/API message matrix stays partial. | +| AC-56 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json | The literal SQLite vector is pinned (absent head 33 zero bytes); the shared PostgreSQL literal and the three-run E3 shape are deferred. | +| AC-57 | both | E3 | partial | this runner #arm_measurement; sqlite_compact_measure.json; T-G61 sqlite_compact_capacity.rs#capacity_measure; sqlite_compact_capacity_measure.json | The tiny three-run arm is median 28.2 ms. T-G61 measured a real 9.987 GB Store with 3 runs per arm: `none` at 8.88% dead (median 149.318 s) and `rebuild` at 10.00% dead (median 246.371 s, mark 149.089 s / build 94.317 s / verify 0.312 s / install 1.916 s), plus one WAL/FULL rebuild at 284.078 s; profile, host and toolchain are declared in the fixture, and the warm page cache is declared below. The section 13 50 GB arms, the 30% arm, each removal type, the plain `VACUUM INTO` comparison and PostgreSQL timings remain deferred/[E]. | +| AC-58 | SQLite | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_before_copy; sqlite_compact_crash.rs#crash_during_copy | none deletes the scratch and leaves no .compact; rebuild keeps the visited scratch through the build; step 2 writes only on rebuild. | +| AC-59 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_seal; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale | busy != 0 refuses Busy and the WAL is never unlinked; a successful seal records (size, mtime, counter) as fenced by the StaleStore arm. | +| AC-60 | PG | E2 | deferred (T12.3b) | - | PostgreSQL synchronous standby handling: T12.3b. | + +## Executed in this runner + +- AC-33 / AC-59: `arm_checkpoint_busy` - a live reader makes + `wal_checkpoint(TRUNCATE)` report busy, the compaction refuses `Busy`, + leaves no `.compact`, `.compact-visited` or `.compact-op`, and the WAL is + never unlinked; the next run rebuilds at `G -> G+1`. +- AC-33 / AC-46: `arm_sidecar_at_install` - a `-mj*` super-journal placed + at install is `Conflict("Store sidecar at install")` with no artifact, + no op and an unchanged generation. +- AC-51: `arm_status_phases` - stop points at mark, build, verify and install + record the same phase in `.compact-op`; every phase is observed. +- AC-55: `arm_error_codes` - exact `Exists("unrecorded compaction artifact")` and + `Capacity("disk space")` texts, with no artifact after the capacity abort. +- AC-59: `arm_seal` - a successful seal leaves no sidecar, reports the main-file + bytes and a SQLite version, and the file reopens. +- AC-04 / AC-05: `arm_random_store_shapes` - a seeded property arm over six + reachability shapes: every live object stays readable, every wholly dead one + is collected, the retained-closure digest is unchanged and the doctor is clean. +- AC-06: `arm_digest_row_after_collection` - a `file_digest` row named a + root the rebuild collected, so the row goes; recording the same content + again succeeds and points at the new root. +- AC-07 / AC-49: `arm_mount_and_compaction_exclude_each_other` - a live mount + makes compaction `MountsActive` naming it, and a live compaction makes a + mount and a second compaction `BusyOp`; the refused runs leave no op. +- AC-49: `arm_mount_during_compaction_is_busy` - the stopped run's live + compaction intent refuses a mount with `BusyOp`, and the next run adopts + the leftover and rebuilds. +- AC-49: `arm_commit_during_rebuild` - a Commit started inside the install + fence is `StaleStore` or lands on `G + 1`; a retry is never lost. +- AC-57: `arm_measurement` - three runs of the same rebuild with median and + range in `engine/crates/layerfs-persistence/tests/fixtures/ + sqlite_compact_measure.json` (refresh with `UPDATE_MEASURE=1`). The separate + T-G61 `sqlite_compact_capacity.rs#capacity_measure` measures a real 9.987 GB + Store (three `none` + three `rebuild` runs, one WAL/FULL control) into + `sqlite_compact_capacity_measure.json`; it is opt-in because the fixture is + ~10 GB and the ignored test is not part of a normal suite. + +## Deferred to T12.3b (PostgreSQL) + +- AC-28 - PostgreSQL Phase C cleanup and its failpoints: T12.3b. +- AC-35 - PostgreSQL advisory-key refcount: T12.3b. +- AC-41 - PostgreSQL Phase M mark transfer: T12.3b. +- AC-42 - PostgreSQL reserved pack/Phase B: T12.3b. +- AC-43 - PostgreSQL Phase C cleanup resume: T12.3b. +- AC-44 - PostgreSQL privileges and lost COMMIT reply: T12.3b. +- AC-60 - PostgreSQL synchronous standby handling: T12.3b. +- The PostgreSQL half of every `both` row (digest agreement, artifact + names, Phase M/B/C, standby, lost reply, 50 GB arms). + +## SQLite gaps (not claimed by this runner) + +- none; every SQLite gap this runner can execute is executed or covered. + +## E3 arms + +- AC-04: the T12.1 closure fixture (deterministic) plus the seeded + reachability property arm; the full random history generator is deferred. +- AC-22: a deterministic barrier starts a merge and a `remove_file` together + for three rounds; the loser retries and a derived tree never shows the path. +- AC-56: the SQLite literal vector is pinned by the T12.1 golden; the shared + PostgreSQL literal is deferred. +- AC-57: three-run measurement arm executed on this Sandbox (small fixture), + plus T-G61's real 9.987 GB capacity fixture with three runs per arm + (`none` 149.318 s median, `rebuild` 246.371 s median, phase sums recorded). + Its 50 GB projection, the 30% arm, PG `ACCESS EXCLUSIVE` duration and the + plain `VACUUM INTO` comparison are deferred/[E]. + +## Status and report projection + +T10 (merged from main after this branch forked) projects `.compact-op` +through `store.status`: `op {id, name, phase, started, uncertain}`, the +busy/unresolved `state` and the backend `phases`. The named control tests +in AC-51 and AC-52 assert that projection and the snake_case section 11 +render; the runner still asserts the recorded SQLite phases directly, so +AC-51 does not depend on the projection alone. + +## Run environment + +- T-G61 capacity run (separate opt-in test): Docker Sandbox `g61sbx` + (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 overlay, 4.0 TB free), rustc + 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF) plus one + `Durable` control, `CARGO_TARGET_DIR=/var/tmp/target-g61`, fixture under + `/var/tmp/g61big`; the test ran 1678 s. Caches: the ~10 GB Store fits in the + 135 GB page cache and no cache drop ran between the three runs of an arm; + the mark is CPU-bound at ~62 MB/s, not cache-bound. +- Docker Sandbox `t123csbx` (Linux x86_64, tmpfs `/tmp`, overlayfs + `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable` + (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`. +- Verified here: `sqlite_compact_matrix` (1 test); the persistence suites + `sqlite_compact` (20), `sqlite_compact_crash` (13), + `sqlite_compact_fixtures` (1), `admission` (5), `path_removal` (3), + `branch_removal` (3), `project_delete` (5), `epoch_fence` (7), + `gc_backfill` (4), `kept_objects` (5); the Store-service path suites + `sdk_path_merge_removal` (5), `sdk_mount_bind_failure` (1), + `sdk_path_whiteouts` (7), `sdk_path_hidden_semantics` (2), + `sdk_path_removal` (2); and `layerfs-control --features postgresql + project_delete_worker` (5), `layerfs-store --features postgresql + project_delete` (15) and `compact_report_status` (3). + All of those pass on the merged branch. +- The control `restore_install` arm that opens a Store needs ext4/xfs/btrfs; + this Sandbox has no such direct filesystem, so that one case reports + `Unsupported` while its other two cases pass. It is an environment limit, + not a matrix result; the cited API tests are validated by source name only + in this run. diff --git a/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md b/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md new file mode 100644 index 0000000000..ddefe61a52 --- /dev/null +++ b/core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md @@ -0,0 +1,150 @@ +# T12.3b: the PostgreSQL half of the GC acceptance matrix + +> Acceptance note for the frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) +> §8/§9/§11/§12, the companion of +> [`gc-t12-sqlite-compaction-fixtures.md`](gc-t12-sqlite-compaction-fixtures.md) +> (SQLite half) and [`gc-t9-postgresql-rebuild.md`](gc-t9-postgresql-rebuild.md) +> (the PostgreSQL rebuild). It adds no contract: the spec and the +> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) win; `§N` and `AC-nn` +> cite them. The matrix statuses stay `open` (only #52 records acceptance); +> this note records the PG-half evidence and its tier. +> +> **T-G28/T-G60 follow-up.** `sqlite_compact_crash` now has 23 tests (9 +> real-process SIGKILL arms at mark, copy, verify, rename, build, install and +> settle, plus the child placeholder), not the 13 passed recorded below, and +> the matrix statuses are no longer `open`: AC-01 is `partial` and the other +> rows are `executed`/`covered`/`partial`/`deferred`. This page keeps the +> T12.3b baseline counts. + +## Environment + +- PostgreSQL **17.12** (`postgresql-17_17.12-1.pgdg12+1`, Debian 12), server on + `127.0.0.1:55432`, one schema per test dropped at test end, `postgres` + superuser, `sslmode=disable`. +- `LAYERFS_TEST_POSTGRES_URL` was set for every run below. The suites that pass + the URL straight to the driver take the keyword form + (`host=127.0.0.1 port=55432 user=postgres dbname=postgres sslmode=disable`). + The suites that append startup options (`postgresql_diagnostic`) need the URI + form (`postgresql://postgres@127.0.0.1:55432/postgres?sslmode=disable`); + with the keyword form those three tests fail at connect, which is a harness + URL-shape issue, not a Store result. +- Toolchain: Rust 1.85.1, `cargo test -p layerfs-persistence --features + postgresql`, `-j 8`, `CARGO_TARGET_DIR=/tmp/target-merged`. +- Branch `task-t12.3b` forked main layer `32c9181e`; it was merged with main + before publishing, and every result below was re-run on the merged tree + (main `3284fb82` at the merge), so the commands and counts are the merged + branch's, not the fork base's. + +## The fixture + +`engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs` is +the T12.1 fixture rebuilt on PostgreSQL through the public APIs only. It is a +new test target (5 tests); it changes no production code. + +| Test | What it pins | Matrix rows | +|---|---|---| +| `the_live_heads_digest_vector_is_the_sqlite_literal` | The §6.4 vector on PG is **byte-identical to the T12.1 SQLite golden**: the Commit id, `absentHeadDigest` (33 zero bytes) and `retiredBranchDigest` are the literals embedded in `fixtures/sqlite_compact_golden.json`. | AC-40, AC-56 (PG half) | +| `the_rebuild_report_and_markers_hold_on_postgresql` | The full T12.1 rebuild fixture on PG: `live_heads_digest` equals the SQLite golden **rebuild** literal; `deadBytesBound == deadBytesRefined`; `backendReusable = 0` with `postgresql-in-place`; `physicalShrink`, `peakTempBytes`, `keptWaste.{deadInRetainedPacksBytes, inodeBlocksBytes, deltaBases}`; `logicalDiscarded == 160`; empty residue; `G -> G+1`; depth-2 chain kept with `pack_id` and pack length unchanged; retained pack keeps its unreached group; markers/digest rows follow the retained set; full doctor; epoch copied, serial never lowered. | AC-02, AC-05, AC-17, AC-18, AC-25, AC-40, AC-41, AC-43, AC-52 | +| `a_save_after_the_swap_survives_the_resumed_cleanup` | `PostgresAfterSwap` aborts the round at `G+1`; a Save lands after the swap; the resumed Phase C deletes only the persisted dead set, the late write (object, pack, totals) survives the high-water bound, and the Store is usable at `G+1`. | AC-28, AC-43, AC-49 (Phase C arm) | +| `every_postgresql_failpoint_leaves_a_resumable_leftover` | All four named §8 failpoints (`mark`, `build`, `swap`, `cleanup`) leave the documented state (generation, dead pack, scratch schema, op phase, owner pid) and the next call adopts/resumes it; the heads digest is unchanged and the doctor is clean. | AC-01 (in-process arm), AC-03 (PG failpoints), AC-28, AC-43 | +| `a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write` | Commit/Layer leave `project_epoch` alone; a Branch removal and a `remove_path` marker bump it once each; a repeated marker is `UpToDate` and does not bump; compact copies the epoch and never lowers `next_history_serial`; a removal-first stale Commit and a missing epoch row are both `Conflict("project epoch")` and write nothing. | AC-21, AC-24, AC-39 | + +The digest equality is not a value comparison on "both backends ran" alone: the +fixture ids, roots and Commit records are content-addressed, so the PostgreSQL +mark hashing the SQLite literal proves the §6.4 vector agrees byte for byte. +Run three times, the five tests passed 3/3 with the same literals (the matrix +lists AC-56 at E3; the deterministic two-backend literal is E2, three +identical runs are recorded here for the E3 arm). + +## Matrix: requested PostgreSQL-half rows + +Counts: **19 rows — 5 passed, 6 partial, 8 deferred** (AC-33 is a SQLite-only +row and is not part of the PG half). + +| Case | Result | Tier | Evidence / leftover | +|---|---|---|---| +| AC-02 | partial | E1 | The additive step on PG (position columns + serial backfill) passes `postgresql_position_columns` 4/4; the report/removal parity is covered below. A converted **1.0** Store is SQLite-only: PostgreSQL is a 2.0 format with no 1.0 reader, so that arm is not runnable on this backend. | +| AC-28 | **pass** | E1 | `a_save_after_the_swap_survives_the_resumed_cleanup`: late Save survives the resumed Phase C; op/generation asserted. Deferred sub-arm: the marker/base-row batch boundary failpoint (no injected point inside `cleanup`; Postgres only has the four round failpoints). | +| AC-29 | partial | E1 | `postgresql_compact` 8/8: a live idle owner is terminated by `(pid, backend_start)` and adopted; the caller's own uncertain round is adopted without self-termination. Mount-work admission belongs to the Store/Workspace layer and is not in this harness (deferred). | +| AC-30 | deferred | E0 | The `NameTaken` detail/error matrix is backend-neutral history work; no PG-specific fixture was added here. | +| AC-31 | deferred | E0 | Call-log/trace-salt redaction is the control-layer path, not the persistence harness. | +| AC-32 | partial | E1 | Same T9 owner-adoption evidence as AC-29 (uncertain op settled by the owning call). "Another home's intent visible" is control-layer, deferred. | +| AC-34 | deferred | E0 | A failed intent bind/delete needs the admission registry around a mount/init call, not this harness. | +| AC-35 | deferred | E0 | The project/Store advisory-key refcount exercise is the mount layer (`pg_advisory` keys), outside the persistence fixture. | +| AC-36 | partial | E1 | Lost swap reply (`postgresql_compact`), lost cleanup reply and lost mark/build replies (`every_postgresql_failpoint_leaves_a_resumable_leftover`) all resolve as documented. An unreachable worker/remote Store is deferred (needs the worker transport). | +| AC-37 | deferred | E0 | Trace redaction is the control-layer call log. | +| AC-38 | partial | E1 | The scratch name `lfs_gc_` is recorded before CREATE and removed at the swap/settle: the four-failpoint test asserts `scratch_exists` per phase. Deferred: a remote machine removing the scratch schema and a stray schema refusing with `Exists`. | +| AC-39 | **pass** | E1 | `postgresql_position_columns` 4/4 (one `project_epoch` row per stack, serials backfilled once, never lowered) plus the new fixture: the rebuild copies the epoch, `gc_op`/`gc_dead` are cleared on settle and not copied. | +| AC-40 | **pass** | E2 | `the_live_heads_digest_vector_is_the_sqlite_literal` + `the_rebuild_report_and_markers_hold_on_postgresql` vs the T12.1 SQLite golden; the retired head moves the digest on both. | +| AC-41 | partial | E1 | The depth-2 chain is kept only as delta bases, both reported in identity order; every retained pack keeps its `pack_id` and its `length` after the swap. Deferred: `markBytes` as a wire metric — it is a #52 metric and is not a `store.compact` result field (§11 matrix note), so there is nothing to assert in the receipt. | +| AC-42 | deferred | E0 | A Save reserved before the mark and published during Phase A must make the swap `Conflict`. The PG round has no pause point between build and swap (`stop()` returns immediately), so a publication cannot be interleaved deterministically; a wall-clock race would not be a fixed fixture. | +| AC-43 | **pass** | E2 | After the swap and the resumed cleanup: the four `store_totals` equal their row sums, `pack`/`object_location` keep owner and the `pack_immutable` trigger bound to the schema function (and the trigger still refuses an update), `binding_hint_block` is cleared, there is no `inode` table on PG, the Store opens at `G+1` and passes the full doctor. | +| AC-44 | deferred | E0 | Missing `CREATE`/`TEMP`/`pg_read_all_stats` needs roles the suite may create and a server it may reconfigure; a truly lost `COMMIT` reply needs a killed backend. The `PostgresAfterSwap` failpoint is the deterministic substitute and proves resume, not reply loss. | +| AC-56 | **pass** | E2 (3/3 runs) | The PG digests are the T12.1 SQLite literals: absent head (`6864c4…`), retired branch (`dafcdd…`), and the rebuild vector (`d3a3ea…`); the frozen Commit id `121c28…` is identical on both backends. | +| AC-60 | deferred | E0 (predicate E1) | A real synchronous standby needs a second cluster and `synchronous_standby_names`/`synchronous_commit` changes on a dedicated server; a shared suite URL cannot own those GUCs. `compact.rs` unit tests pin the predicate (no names / below `remote_write` → no wait; names → a streaming sync/quorum standby with a matching `sync_state`). | + +## Matrix: additional PostgreSQL rows recorded by this note + +Counts: **11 rows — 6 passed, 5 partial, 0 deferred**. + +| Case | Result | Tier | Evidence / leftover | +|---|---|---|---| +| AC-01 | partial | E1 | The four in-process PG aborts leave a wholly old (`mark`, `build`) or wholly new (`swap`, `cleanup`) Store and the next call settles. A `kill -9` of a daemon at the same points is not injectable in this harness. | +| AC-03 | partial | E1 | Named PG failpoints `mark`, `build`, `swap`, `cleanup` are all exercised with state, generation, op phase, scratch and digest asserted. Lost `COMMIT`, power loss and lease takeover are deferred (AC-44 note). | +| AC-05 | **pass** | E2 | Full doctor on the compacted PG Store in both rebuild tests; `postgresql_diagnostic` 5/5 (transport, cancellation, denied signature). | +| AC-17 | **pass** | E1 | Depth-2 PREFIX chain copied; `pack_id` preserved for both bases and the target. | +| AC-18 | **pass** | E1 | The collected `file_digest` root is gone, the retained one survives. | +| AC-21 | partial | E1 | Removal-first refusal (`Missing`/`Conflict`, no Commit) and the missing-epoch-row `Conflict` are deterministic on PG; the true concurrent one-shot-write arm (a trigger injecting an epoch change mid-transaction, as the SQLite `epoch_fence` tests do) is not implemented for PG here. | +| AC-24 | **pass** | E1 | Marker write bumps `project_epoch` once; `UpToDate` does not; compact copies it; serial never falls. | +| AC-25 | **pass** | E1 | A retained `branch_retired` row survives a rebuild; a collected Branch takes its marker with it. | +| AC-49 | partial | E1 | Phase C survives the high-water bound (late Save); the mount-wins-the-race arm needs the mount/lease layer, deferred. | +| AC-51 | partial | E1 | The op row records the §8 phase (`swap` asserted after the failpoint, `mark`/`build` in the matrix test). Projecting it through `store.status` needs a Store-home fixture; the T10 serializer test is SQLite-side today. | +| AC-52 | **pass** | E2 | The PG report has every §11 component: `deadBytesBound == deadBytesRefined`, `backendReusable = 0` + `postgresql-in-place`, `physicalShrink > 0`, `peakTempBytes > 0`, `keptWaste` three parts, empty residue, `logicalDiscarded` = history delta, `estimate = true`, zero reasons named. | + +## Commands and results + +Server: PostgreSQL 17.12 on `127.0.0.1:55432`; `LAYERFS_TEST_POSTGRES_URL` set as +above; `-j 8`; `--test-threads=1`. + +| Command (`cargo test -p layerfs-persistence --features postgresql --test …`) | Result | +|---|---| +| `postgresql_compact_fixtures` (new) | **5 passed**; 3 consecutive runs, 5 passed each | +| `postgresql_compact` | 8 passed | +| `postgresql_position_columns` | 4 passed | +| `postgresql_open_check` | 3 passed | +| `postgresql_commit_round_trips` | On the fork base: 1 failed, `13 != 11`; current main already fixes the expectation (see below), the merged tree passes 1/1 | +| `postgresql_diagnostic` (URI URL) | 5 passed | +| `postgresql_acquisition_compaction` | 4 passed | +| `postgresql_acquisition` | 4 passed | +| `postgresql_identical_ids` | 3 passed | +| `layerstack_removal` | 2 passed (one is the PostgreSQL project-removal arm) | +| `sqlite_compact_fixtures` | 1 passed (the golden still matches) | +| `sqlite_compact_crash` | 13 passed | +| `sqlite_compact` | 20 passed | + +## Baseline note: the stale round-trip count is already fixed on main + +`postgresql_commit_round_trips.rs` expected 11 round trips for `lane + head + +commit_branch` on the fork base; the path takes 13. The two extra statements are +T6.5's writer fence (`gc_epoch_read` at the start of the write transaction and +`gc_epoch_share` … `FOR SHARE` as its last statement, `src/history/epoch.rs`). +Current main already carries the updated expectation (13), so this branch does +not change that file; it only records the finding. + +## Gaps (not contract changes) + +- **AC-42** (a Save reserved before the mark racing the swap) and **AC-60** + (real standby, missing standby before/inside the swap, dying after the check) + need a hook the deterministic round does not have or a dedicated server whose + GUCs the suite owns. +- **AC-44** privileges and a killed backend's lost `COMMIT` need server roles + and processes the suite cannot own; the swap failpoint is the recorded + substitute. +- **AC-28**'s marker/base-row batch boundary needs a failpoint inside Phase C; + only the four round failpoints exist. +- The mount/lease admission rows (**AC-29/32/35/49**) are Store-layer, not + compaction-layer; only their compaction-op liveness arm is here. +- **AC-30/31/34/37** (error detail, trace, intent bind, redaction) are + control-layer rows and are untouched. +- `markBytes` (AC-41) is not a `store.compact` field, so the PG half cannot + assert it from the receipt. diff --git a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md index cc8461befe..ca544b11a7 100644 --- a/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md +++ b/core/docs/issues/project-41/gc-t8-sqlite-rebuild.md @@ -88,6 +88,16 @@ without killing the process. the collected history rows' `logical` bytes; `parallel` note: `estimate` is always true until #52 measures the arms. +`peakTempBytes` is the §11 peak temporary space as a documented upper bound +(T-G72): `staged_bytes + journal_bound + target_bytes + visited_bytes`. The +staged copy holds the sealed Store's whole extent until the rewrite truncates +it (`VACUUM INTO`); the rewrite holds SQLite's unlinked temp database (the new +file) beside it and journals the pages it overwrites; the journal bound is one +rollback-journal record per staged page (`page_size + 8` B) plus 8192 B of +header. The 50 GB arm's bound is 147.609 GB against a 143.1 GB sampled temp +peak; the old `new + visited` receipt (45.784 GB) dropped the 44.6 GB +rollback journal (T-G66 finding, T-G72 fix). Upstream issue: [Ephemeral-AI-Lab/layerfs#695](https://github.com/Ephemeral-AI-Lab/layerfs/issues/695). + ## Tests `engine/crates/layerfs-persistence/tests/sqlite_compact.rs`: @@ -106,6 +116,12 @@ always true until #52 measures the arms. next call judges it and settles; - generation increases once per installed rebuild. +T-G72 (`tests/sqlite_compact_capacity.rs`): a moderate fixture samples the +staged copy, the rollback journal and SQLite's unlinked rewrite temp database +from a second thread and asserts the receipt bounds the new file plus the +journal and the whole sampled footprint; `compact.rs`'s unit test pins the +50 GB arithmetic. + Runs: `cargo test -p layerfs-persistence --offline` (all binaries pass), `cargo check --workspace --offline` passes. `kernel_guard` in `layerfs-store-service` fails in this container because `/tmp` is tmpfs and @@ -132,7 +148,10 @@ the guard refuses direct Store access; it is unrelated to this change. and a live intent that outlived its op record refused every later GC call `BusyOp`. The op record is kept only when the release itself fails, as the next run's recovery breadcrumb (§9). -- Peak temp is the new file plus the visited scratch; journal/WAL sizes - after the checkpoint are zero and are not added. +- ~~Peak temp is the new file plus the visited scratch; journal/WAL sizes + after the checkpoint are zero and are not added.~~ T-G72: `peakTempBytes` + bounds the pre-truncate staged copy, the new file (the rewrite's temp + database), the prune/rewrite rollback journal and the visited scratch. It is + an estimate and over-reports the sampled peak by a few percent. - The mark runs before the seal under the same lane (the spec's step 1.5 runs it after the seal); the sealed tuple re-read makes the window explicit. diff --git a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md index 1d9265c448..041dd5db97 100644 --- a/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md +++ b/core/docs/issues/project-41/gc-t9-postgresql-rebuild.md @@ -94,6 +94,12 @@ postgresql_compact` — 8 passed on PostgreSQL 17.12 (Debian, server on `127.0.0.1:55432`); `--lib` 4 passed; the SQLite suite (`--test sqlite_compact`) passed (17 when T9 branched, 20 on the rebased main with the T12 fixtures); `sqlite_compact_crash` passed 13. +> **T-G28/T-G60 follow-up.** `sqlite_compact_crash` grew to 23 tests with the +> real-process SIGKILL arms (9 arms after T-G60), so the 13 passed here is the +> T9 baseline. The +> original `Gaps` bullet below still says those rows remain `open`; the current +> matrix has them `partial`/`deferred`. This page keeps the T9 counts. + ## Gaps (not contract changes) - **AC-41/AC-42/AC-43/AC-44/AC-60 remain `open`** in the acceptance matrix: diff --git a/docs/architecture/api/README.md b/docs/architecture/api/README.md index 467bbcc365..c85436b1fc 100644 --- a/docs/architecture/api/README.md +++ b/docs/architecture/api/README.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The public surface: the `layerfs-api` call table with its one dispatcher, and the front ends (the `layerfs` command, Node, Python and a local MCP server) that expose its **73 calls in 10 families** with the same names and arguments. +The public surface: the `layerfs-api` call table with its one dispatcher, and the front ends (the `layerfs` command, Node, Python and a local MCP server) that expose its **76 calls in 10 families** with the same names and arguments. | | | | --- | --- | @@ -85,7 +85,7 @@ give each call's own arguments, answer and errors. | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | -| [`calls_table.rs`](../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 73 calls in ten families; one source file per call; refusals before any effect | +| [`calls_table.rs`](../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 76 calls in ten families; one source file per call; refusals before any effect | | [`coverage.mjs`](../../../api/node/test/coverage.mjs) | `node --test api/node/test/coverage.mjs` | macOS or Linux, with the built addon, Python module and CLI | every front end matches the table | | [`file-history.mjs`](../../../api/node/examples/file-history.mjs) | `node api/node/examples/file-history.mjs` | macOS or Linux, with the built addon | a Store, Init, file writes and history reads | diff --git a/docs/architecture/api/calls/README.md b/docs/architecture/api/calls/README.md index d17fa5cc97..2744390222 100644 --- a/docs/architecture/api/calls/README.md +++ b/docs/architecture/api/calls/README.md @@ -41,7 +41,7 @@ The call table and the one dispatcher every front end goes through: what a call | Behaviour | Detail | Source | | --- | --- | --- | -| Table | ten `Family { name, list }`; a `Call` is `{ family, call, args, options, help, run }`, an `Arg` `{ name, kind, optional }`; 73 calls, help order | [`table.rs`](../../../../api/crates/layerfs-api/src/calls/table.rs), [`lib.rs`](../../../../api/crates/layerfs-api/src/lib.rs) `families` | +| Table | ten `Family { name, list }`; a `Call` is `{ family, call, args, options, help, run }`, an `Arg` `{ name, kind, optional }`; 76 calls, help order | [`table.rs`](../../../../api/crates/layerfs-api/src/calls/table.rs), [`lib.rs`](../../../../api/crates/layerfs-api/src/lib.rs) `families` | | JSON form | `table_json()`: `[{ family, call, args: [{ name, kind, optional }], options, help }]`; Node's `table`, Python's `TABLE`; the CLI parser and help read the same `Call` values | `table.rs` `table_json` | | Shared options | `PAGE` (`page_size`, `page_pos`), `DIFF` (`paths_only`, `page_size`, `page_pos`, `cursor`), `MESSAGE` (`message`), `PROGRESS` (`on_progress`) | `table.rs` | | Kinds | below | `table.rs` `Kind` | @@ -160,7 +160,7 @@ changes it. Reading and clearing it are the [`logs`](../sdk/logs.md) calls. | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | -| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host, temporary folders | ten families and 73 calls; unknown calls and arguments refused before any effect; each family reached; JSON round trips; sizes and keys; a relative home refused; one source file per call | +| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host, temporary folders | ten families and 76 calls; unknown calls and arguments refused before any effect; each family reached; JSON round trips; sizes and keys; a relative home refused; one source file per call | | [`calls_scope.rs`](../../../../api/crates/layerfs-api/tests/calls_scope.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_scope` | any Unix host | a scope names its project, refuses every call it does not cover, needs a Sandbox of its project, and a scoped `branch.list` reads every page | | [`scope_references.rs`](../../../../api/crates/layerfs-api/tests/scope_references.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test scope_references` | any Unix host | fork and publish references outside the namespace refused before any effect; the JSON form refuses the same | | [`logs.rs`](../../../../api/crates/layerfs-api/tests/logs.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test logs` | any Unix host | one line per call, rendering and redaction, `logs.list` and `logs.reset` | diff --git a/docs/architecture/api/calls/coverage.md b/docs/architecture/api/calls/coverage.md index a6cfa5dd53..53f4a26fcb 100644 --- a/docs/architecture/api/calls/coverage.md +++ b/docs/architecture/api/calls/coverage.md @@ -42,7 +42,7 @@ check and proves a missing, renamed or extra call is reported. -61 calls in 9 families, counted from the Node table. Calls present: Node 61, Python 61, Python `aio` 61, CLI 61, MCP 61 (34 in `work`, 27 admin only). +76 calls in 10 families, counted from the Node table. Calls present: Node 76, Python 76, Python `aio` 76, CLI 76, MCP 76 (38 in `work`, 38 admin only). Argument names that are Python keywords: none. @@ -53,24 +53,30 @@ Front-end problems: none. | `project.init` | project, folder | timeout, on_progress() | yes | yes | yes | yes (--progress) | admin | | `project.list` | ?store | page_size, page_pos | yes | yes | yes | yes | yes | | `project.get` | project | - | yes | yes | yes | yes | yes | +| `project.stats` | project, ?branch, ?commit | - | yes | yes | yes | yes | yes | | `project.export` | project, destination | branch, on_progress() | yes | yes | yes | yes (--progress) | yes | -| `project.remove` | project | - | yes | yes | yes | yes | admin | -| `branch.list` | project | page_size, page_pos | yes | yes | yes | yes | yes | +| `project.delete` | project | - | yes | yes | yes | yes | admin | +| `project.remove_file` | project, path | - | yes | yes | yes | yes | admin | +| `branch.list` | project | page_size, page_pos, page | yes | yes | yes | yes | yes | | `branch.get` | project, branch | - | yes | yes | yes | yes | yes | +| `branch.commits` | project, branch | page, page_size, at_commit, include_storage | yes | yes | yes | yes | yes | +| `branch.get_commit` | project, branch, commit | - | yes | yes | yes | yes | yes | | `branch.fork` | project, name, from_branch, ?from_commit | - | yes | yes | yes | yes | yes | | `branch.merge` | project, name, a, b | message, units | yes | yes | yes | yes | yes | | `branch.publish` | project, branch, ?commit | - | yes | yes | yes | yes | yes | -| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | +| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor, page, from_commit, to_commit, path_prefix, include_attribution, continuation | yes | yes | yes | yes | yes | +| `branch.remove` | project, branch | - | yes | yes | yes | yes | admin | | `commit.list` | project, branch | page_size, page_pos | yes | yes | yes | yes | yes | | `commit.get` | project, branch, commit | - | yes | yes | yes | yes | yes | | `commit.diff` | project, from_commit, to_commit | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | | `file.read` | project, branch, path, ?commit | start_line, end_line | yes | yes | yes | yes | yes | | `file.stat` | project, branch, path, ?commit | - | yes | yes | yes | yes | yes | -| `file.list` | project, branch, path, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.glob` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.grep` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.blame` | project, branch, path, ?commit | from_line | yes | yes | yes | yes | yes | -| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit | yes | yes | yes | yes | yes | +| `file.list` | project, branch, path, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.glob` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.grep` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.revision` | project, branch, path | at_commit, traversal, page, page_size, continuation | yes | yes | yes | yes | yes | +| `file.blame` | project, branch, path, ?commit | from_line, line_count | yes | yes | yes | yes | yes | +| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit, include_attribution, allow_missing, page, page_size | yes | yes | yes | yes | yes | | `file.write` | project, branch, path, content | message, mode | yes | yes | yes | yes (stdin/--from) | yes | | `file.edit` | project, branch, path | old, new, message | yes | yes | yes | yes | yes | | `file.rm` | project, branch, path | message | yes | yes | yes | yes | yes | @@ -78,7 +84,7 @@ Front-end problems: none. | `file.mv` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.cp` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.apply` | project, branch, changes | message | yes | yes | yes | yes (stdin/--from) | yes | -| `sandbox.create` | project, name, ?machine | image | yes | yes | yes | yes | admin | +| `sandbox.create` | project, name, ?machine | image, on_progress() | yes | yes | yes | yes (--progress) | admin | | `sandbox.list` | ?project | page_size, page_pos | yes | yes | yes | yes | yes | | `sandbox.get` | name | page_size, page_pos | yes | yes | yes | yes | yes | | `sandbox.close` | name | force | yes | yes | yes | yes | admin | @@ -91,15 +97,19 @@ Front-end problems: none. | `machine.list` | - | - | yes | yes | yes | yes | admin | | `machine.platforms` | - | - | yes | yes | yes | yes | admin | | `machine.get` | name | - | yes | yes | yes | yes | admin | +| `machine.images` | name | - | yes | yes | yes | yes | admin | +| `machine.load` | name, image | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.prune` | name, ?image | unused | yes | yes | yes | yes | admin | | `machine.status` | name | - | yes | yes | yes | yes | admin | -| `machine.start` | name | - | yes | yes | yes | yes | admin | -| `machine.stop` | name | - | yes | yes | yes | yes | admin | -| `machine.create` | name, platform | cpus, memory, disk, image | yes | yes | yes | yes | admin | +| `machine.start` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.stop` | name | force | yes | yes | yes | yes | admin | +| `machine.create` | name, platform | cpus, memory, disk, image, os_image, on_progress() | yes | yes | yes | yes (--progress) | admin | | `machine.remove` | name | - | yes | yes | yes | yes | admin | | `machine.update` | name | cpus, memory, image, docker, wsl, daemon | yes | yes | yes | yes | admin | | `store.list` | - | - | yes | yes | yes | yes | admin | | `store.get` | name | - | yes | yes | yes | yes | admin | | `store.status` | name | - | yes | yes | yes | yes | admin | +| `store.stats` | name | - | yes | yes | yes | yes | admin | | `store.create` | name | - | yes | yes | yes | yes | admin | | `store.connect` | name, url | password, synchronous_commit | yes | yes | yes | yes | admin | | `store.attach` | name, machine | url, password | yes | yes | yes | yes | admin | @@ -107,12 +117,15 @@ Front-end problems: none. | `store.push` | name, machine | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.pull` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.remove` | name | - | yes | yes | yes | yes | admin | +| `store.compact` | name | - | yes | yes | yes | yes | admin | | `store.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.connect` | name, url | key, region, part_size | yes | yes | yes | yes | admin | | `backup.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.list` | ?bucket | - | yes | yes | yes | yes | admin | | `backup.create` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | | `backup.restore` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `logs.list` | - | op, project, client, tag, outcome, since, cursor, limit | yes | yes | yes | yes | admin | +| `logs.reset` | - | - | yes | yes | yes | yes | admin | diff --git a/docs/architecture/api/front-ends/mcp.md b/docs/architecture/api/front-ends/mcp.md index 4f4fd02c3e..1a91f60383 100644 --- a/docs/architecture/api/front-ends/mcp.md +++ b/docs/architecture/api/front-ends/mcp.md @@ -98,12 +98,12 @@ Each call belongs to exactly one bucket, `work` (listed by both profiles) or | Bucket | Calls | | --- | --- | | `work` | the `branch`, `commit`, `file` and `workspace` families; `project.list`, `project.get`, `project.export`, `sandbox.list`, `sandbox.get` | -| `admin` | `project.init`, `project.remove`, `sandbox.create`, `sandbox.close`; the `machine`, `store`, `backup` and `logs` families | +| `admin` | `project.init`, `project.delete`, `project.remove_file`, `branch.remove`, `store.compact`, `sandbox.create`, `sandbox.close`; the `machine`, `store`, `backup` and `logs` families | `work` lets an agent read a project, work on Branches and files (each write is one explicit Commit), run in a mounted Workspace and export a snapshot. `admin` adds the calls that start workers, move whole Stores, hold -credentials, delete records or change the shape of the home. A new call with +credentials, retire or compact records, or change the shape of the home. A new call with no bucket fails `tools()`, generation and the test rather than appearing silently. Regenerate the table below with `node api/mcp/scripts/gen-profiles.mjs` (`--check` compares; both need the @@ -111,16 +111,17 @@ built addon). -73 calls: 38 in `work`, 35 in `admin` only; `--profile admin` lists all 73. +76 calls: 38 in `work`, 38 in `admin` only; `--profile admin` lists all 76. | Call | Bucket | Profiles that list it | Why | | --- | --- | --- | --- | | `project.init` | admin | admin | import a host folder into a Store; administration | -| `project.list` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | -| `project.get` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | -| `project.stats` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | +| `project.list` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | +| `project.get` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | +| `project.stats` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | | `project.export` | work | work, admin | export a Branch snapshot to a local folder under a launch-time --export-root; a folder scope refuses it | -| `project.remove` | admin | admin | remove a project from its Store; administration | +| `project.delete` | admin | admin | retire a project in its Store and remove its home entry; administration; a folder scope refuses it | +| `project.remove_file` | admin | admin | hide one absolute path from every node at or below its marker frontier; administration; a folder scope refuses it | | `branch.list` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.get` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.commits` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | @@ -129,6 +130,7 @@ built addon). | `branch.merge` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.publish` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.diff` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | +| `branch.remove` | admin | admin | retire one Branch in its Store; administration; a folder scope refuses it | | `commit.list` | work | work, admin | read a Branch's Commit history | | `commit.get` | work | work, admin | read a Branch's Commit history | | `commit.diff` | work | work, admin | read a Branch's Commit history | @@ -180,6 +182,7 @@ built addon). | `store.push` | admin | admin | create, connect, transfer and remove Stores; administration | | `store.pull` | admin | admin | create, connect, transfer and remove Stores; administration | | `store.remove` | admin | admin | create, connect, transfer and remove Stores; administration | +| `store.compact` | admin | admin | reclaim one sqlite Store's dead bytes; administration; a folder scope refuses it | | `store.disconnect` | admin | admin | create, connect, transfer and remove Stores; administration | | `backup.connect` | admin | admin | upload and restore a Store on a bucket; administration | | `backup.disconnect` | admin | admin | upload and restore a Store on a bucket; administration | diff --git a/docs/architecture/api/sdk/README.md b/docs/architecture/api/sdk/README.md index a1b2ff608b..c994cdfa8f 100644 --- a/docs/architecture/api/sdk/README.md +++ b/docs/architecture/api/sdk/README.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -Every call a program can make: **73 calls in 10 families**, the same names and arguments in JavaScript, Python and the `layerfs` command, one page per family. +Every call a program can make: **76 calls in 10 families**, the same names and arguments in JavaScript, Python and the `layerfs` command, one page per family. | | | | --- | --- | @@ -49,14 +49,14 @@ built table. | Family | Calls, in help order | Runs by default | Source | | --- | --- | --- | --- | -| [`project`](project.md) | [`init`](project.md#projectinit), [`list`](project.md#projectlist), [`get`](project.md#projectget), [`stats`](project.md#projectstats), [`export`](project.md#projectexport), [`remove`](project.md#projectremove) | `list`, `get`: home records; the rest where the Store's calls run | [`project/`](../../../../api/crates/layerfs-api/src/project) | -| [`branch`](branch.md) | [`list`](branch.md#branchlist), [`get`](branch.md#branchget), [`commits`](branch.md#branchcommits), [`get_commit`](branch.md#branchget_commit), [`fork`](branch.md#branchfork), [`merge`](branch.md#branchmerge), [`publish`](branch.md#branchpublish), [`diff`](branch.md#branchdiff) | where the Store's calls run | [`branch/`](../../../../api/crates/layerfs-api/src/branch) | +| [`project`](project.md) | [`init`](project.md#projectinit), [`list`](project.md#projectlist), [`get`](project.md#projectget), [`stats`](project.md#projectstats), [`export`](project.md#projectexport), [`delete`](project.md#projectdelete), [`remove_file`](project.md#projectremove_file) | `list`, `get`: home records; the rest where the Store's calls run | [`project/`](../../../../api/crates/layerfs-api/src/project) | +| [`branch`](branch.md) | [`list`](branch.md#branchlist), [`get`](branch.md#branchget), [`commits`](branch.md#branchcommits), [`get_commit`](branch.md#branchget_commit), [`fork`](branch.md#branchfork), [`merge`](branch.md#branchmerge), [`publish`](branch.md#branchpublish), [`diff`](branch.md#branchdiff), [`remove`](branch.md#branchremove) | where the Store's calls run | [`branch/`](../../../../api/crates/layerfs-api/src/branch) | | [`commit`](commit.md) | [`list`](commit.md#commitlist), [`get`](commit.md#commitget), [`diff`](commit.md#commitdiff) | where the Store's calls run | [`commit/`](../../../../api/crates/layerfs-api/src/commit) | | [`file`](file.md) | [`read`](file.md#fileread), [`stat`](file.md#filestat), [`list`](file.md#filelist), [`glob`](file.md#fileglob), [`grep`](file.md#filegrep), [`revision`](file.md#filerevision), [`blame`](file.md#fileblame), [`diff`](file.md#filediff), [`write`](file.md#filewrite), [`edit`](file.md#fileedit), [`rm`](file.md#filerm), [`mkdir`](file.md#filemkdir), [`mv`](file.md#filemv), [`cp`](file.md#filecp), [`apply`](file.md#fileapply) | where the Store's calls run | [`file/`](../../../../api/crates/layerfs-api/src/file) | | [`sandbox`](sandbox.md) | [`create`](sandbox.md#sandboxcreate), [`list`](sandbox.md#sandboxlist), [`get`](sandbox.md#sandboxget), [`close`](sandbox.md#sandboxclose) | records here; start and stop on the Sandbox's machine | [`sandbox/`](../../../../api/crates/layerfs-api/src/sandbox) | | [`workspace`](workspace.md) | [`mount`](workspace.md#workspacemount), [`exec`](workspace.md#workspaceexec), [`exec_cancel`](workspace.md#workspaceexec_cancel), [`commit`](workspace.md#workspacecommit), [`unmount`](workspace.md#workspaceunmount), [`status`](workspace.md#workspacestatus) | the Sandbox's daemon; `exec` and `exec_cancel` in the Sandbox's runtime | [`workspace/`](../../../../api/crates/layerfs-api/src/workspace) | | [`machine`](machine.md) | [`list`](machine.md#machinelist), [`platforms`](machine.md#machineplatforms), [`get`](machine.md#machineget), [`images`](machine.md#machineimages), [`load`](machine.md#machineload), [`prune`](machine.md#machineprune), [`status`](machine.md#machinestatus), [`start`](machine.md#machinestart), [`stop`](machine.md#machinestop), [`create`](machine.md#machinecreate), [`remove`](machine.md#machineremove), [`update`](machine.md#machineupdate) | the caller's process, through the platform adapter | [`machine/`](../../../../api/crates/layerfs-api/src/machine) | -| [`store`](store.md) | [`list`](store.md#storelist), [`get`](store.md#storeget), [`status`](store.md#storestatus), [`stats`](store.md#storestats), [`create`](store.md#storecreate), [`connect`](store.md#storeconnect), [`attach`](store.md#storeattach), [`detach`](store.md#storedetach), [`push`](store.md#storepush), [`pull`](store.md#storepull), [`remove`](store.md#storeremove), [`disconnect`](store.md#storedisconnect) | the caller's process; sqlite file steps in the Store machine's worker | [`store/`](../../../../api/crates/layerfs-api/src/store) | +| [`store`](store.md) | [`list`](store.md#storelist), [`get`](store.md#storeget), [`status`](store.md#storestatus), [`stats`](store.md#storestats), [`create`](store.md#storecreate), [`connect`](store.md#storeconnect), [`attach`](store.md#storeattach), [`detach`](store.md#storedetach), [`push`](store.md#storepush), [`pull`](store.md#storepull), [`remove`](store.md#storeremove), [`compact`](store.md#storecompact), [`disconnect`](store.md#storedisconnect) | the caller's process; sqlite file steps in the Store machine's worker | [`store/`](../../../../api/crates/layerfs-api/src/store) | | [`backup`](backup.md) | [`connect`](backup.md#backupconnect), [`disconnect`](backup.md#backupdisconnect), [`list`](backup.md#backuplist), [`create`](backup.md#backupcreate), [`restore`](backup.md#backuprestore) | the caller's process; `create` stages on the Store's machine | [`backup/`](../../../../api/crates/layerfs-api/src/backup) | | [`logs`](logs.md) | [`list`](logs.md#logslist), [`reset`](logs.md#logsreset) | the caller's process, home files only | [`logs/`](../../../../api/crates/layerfs-api/src/logs) | @@ -103,20 +103,20 @@ in the [API layer page](../README.md#calling-conventions). | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | | [`inventory.mjs`](../../../../api/node/test/branch-file/inventory.mjs) | `perl -e 'alarm 120; exec @ARGV' node --test api/node/test/branch-file/inventory.mjs` | macOS or Linux, with the built addon (`npm --prefix api run build:native`) and `target/debug/layerfs` | the count line and one heading per call match the built table; branch and file CLI help exposes every option | -| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 73 calls in ten families; one source file per call; unknown calls and arguments refused before any effect | +| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 76 calls in ten families; one source file per call; unknown calls and arguments refused before any effect | | [`file-history.mjs`](../../../../api/node/examples/file-history.mjs) | `node api/node/examples/file-history.mjs` | macOS or Linux, with the built addon | a Store, Init, file writes and history reads | ## Pages | Page | What it covers | | --- | --- | -| [project](project.md) | Init, project records, stats, export, remove | -| [branch](branch.md) | Branches, saved history, fork, merge, publish, diff; the saved-query rules | +| [project](project.md) | Init, project records, stats, export, delete, path removal | +| [branch](branch.md) | Branches, saved history, fork, merge, publish, diff, retire; the saved-query rules | | [commit](commit.md) | a Branch's Commits and saved comparisons | | [file](file.md) | reads at a saved version; one-Commit writes | | [sandbox](sandbox.md) | Sandboxes: create, list, get, close | | [workspace](workspace.md) | mount, exec, cancel, commit, status, unmount | | [machine](machine.md) | built-in and VM machines | -| [store](store.md) | sqlite and PostgreSQL Stores | +| [store](store.md) | sqlite and PostgreSQL Stores; compaction | | [backup](backup.md) | buckets and Store backups | | [logs](logs.md) | the home's call log | diff --git a/docs/architecture/api/sdk/branch.md b/docs/architecture/api/sdk/branch.md index 480c6faae1..85933cdfd9 100644 --- a/docs/architecture/api/sdk/branch.md +++ b/docs/architecture/api/sdk/branch.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The `branch` calls: list and read a project's Branches and their saved history, fork, merge, publish to `main`, and compare two Branches, within the [SDK](README.md). +The `branch` calls: list and read a project's Branches and their saved history, fork, merge, publish to `main`, compare two Branches and retire one, within the [SDK](README.md). | | | | --- | --- | @@ -35,6 +35,7 @@ The `branch` calls: list and read a project's Branches and their saved history, | [`branch.merge`](#branchmerge) | `branch.merge` | write | | [`branch.publish`](#branchpublish) | `branch.publish` | write | | [`branch.diff`](#branchdiff) | `branch.diff_branches` | saved read | +| [`branch.remove`](#branchremove) | `branch.remove` | write | Each entry forwards a typed request to `layerfs_store::calls::branch`, which renames the table's snake_case fields to the Store method's @@ -287,6 +288,31 @@ comparison and attribution are each bounded to 64 MiB per request ([`read/branch_compare.rs`](../../../../engine/crates/layerfs-store-service/src/read/branch_compare.rs), [`read/compare.rs`](../../../../engine/crates/layerfs-store-service/src/read/compare.rs)). +### branch.remove + +Retires one Branch in its Store (spec §4). Its row is hidden from listings and +name resolution; fork, merge, publish and mount of it are refused, its Commits +and objects stay until a later [`store.compact`](store.md#storecompact), and +`main` is `ReservedName`. + +```text +JS await layerfs.branch.remove(project, branch) +Python layerfs.branch.remove(project, branch) +CLI layerfs branch remove +``` + +Returns `{ kind: "Removed", project, branch, removed_ms }`. A repeat is +`NoSuchBranch` (only `project.delete` is retry-idempotent); a concurrent +remove is `Busy` with phase `"retire"`. + +| Error | When | Source | +| --- | --- | --- | +| `ReservedName` | the Branch is `main` | [`methods/branch/remove.rs`](../../../../engine/crates/layerfs-store-service/src/methods/branch/remove.rs) | +| `NoSuchStore`, `NoSuchBranch` | the Store or Branch is missing (or already retired) | same | +| `MountsActive` | a mount of the Branch is live | [`history/admission.rs`](../../../../engine/crates/layerfs-store-service/src/history/admission.rs) | +| `Busy`, `Unresolved` | another user or operation holds the Branch, or a dead owner left an uncertain one | spec §5.1 | +| `Unknown` | worker route: the answer was lost | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | + ## Saved queries Shared by `branch.commits`, `branch.get_commit`, `branch.diff` and the diff --git a/docs/architecture/api/sdk/project.md b/docs/architecture/api/sdk/project.md index e300021835..597349806f 100644 --- a/docs/architecture/api/sdk/project.md +++ b/docs/architecture/api/sdk/project.md @@ -2,12 +2,12 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The `project` calls: import a host folder into a Store as `/`, read the home's project records, count or export a saved tree, and remove a project, within the [SDK](README.md). +The `project` calls: import a host folder into a Store as `/`, read the home's project records, count or export a saved tree, retire a project, and hide one path from every node at or below a frontier, within the [SDK](README.md). | | | | --- | --- | | Source | [`api/crates/layerfs-api/src/project`](../../../../api/crates/layerfs-api/src/project), [`layerfs-store/src/project`](../../../../control/crates/layerfs-store/src/project), [`layerfs-control/src/project.rs`](../../../../control/crates/layerfs-control/src/project.rs) | -| Runs by default | `list`, `get`: the caller's process, home records only. `init`, `stats`, `export`, `remove`: where the Store's calls run (the caller's process for PostgreSQL and a sqlite Store on a locally executing machine, else that machine's worker) | +| Runs by default | `list`, `get`: the caller's process, home records only. `init`, `stats`, `export`, `delete`, `remove_file`: where the Store's calls run (the caller's process for PostgreSQL and a sqlite Store on a locally executing machine, else that machine's worker) | | Design | [Stores and projects](../../control/stores-and-projects/README.md), [projects](../../control/stores-and-projects/projects.md), [Project Init](../../engine/project-init/README.md) | ## Calls @@ -22,7 +22,7 @@ The `project` calls: import a host folder into a Store as `/`, r +--> stats export (one pinned saved tree) +--> branch.*, commit.*, file.*, sandbox.create ... v - remove / (no Sandbox of it may be recorded) + delete / remove_file / ``` | Call | Returns | @@ -32,7 +32,8 @@ The `project` calls: import a host folder into a Store as `/`, r | [`project.get`](#projectget) | the project summary | | [`project.stats`](#projectstats) | entry counts of one pinned tree | | [`project.export`](#projectexport) | pinned identity and local output counts | -| [`project.remove`](#projectremove) | `{ stack, packs_kept: true, project }` | +| [`project.delete`](#projectdelete) | `{ kind: "Removed", project, deleted_ms, home_entry_removed }` | +| [`project.remove_file`](#projectremove_file) | `{ kind: "Removed" \| "UpToDate", project, path, frontier_serial, removed_ms }` | The **project summary** is `ProjectRecord::to_sdk` ([`project/record.rs`](../../../../control/crates/layerfs-store/src/project/record.rs)): @@ -231,36 +232,63 @@ per 200 ms besides the first and last Export only reads history: a lost reply creates no source mutation and no uncertain intent. -### project.remove +### project.delete -Drops one project's LayerStack (Layers, Branches, Commits) from its Store in -one write, then the project's home entry; other projects stay. +Retires one project in its Store and removes its home entry (spec §4). The +project's Layers, Branches, Commits, packs and objects stay until a later +[`store.compact`](store.md#storecompact) collects them: the project is hidden +from listings and name resolution, and the Store stays addressable. No mount +or operation of the project may be live. ```text -JS await layerfs.project.remove(project) -Python layerfs.project.remove(project) -CLI layerfs project remove +JS await layerfs.project.delete(project) +Python layerfs.project.delete(project) +CLI layerfs project delete ``` -Returns `{ stack, packs_kept: true, project }`: `stack` is the removed -LayerStack's id; `packs_kept` is always true because the project's packs and -objects stay in the Store -([`project/remove.rs`](../../../../control/crates/layerfs-store/src/project/remove.rs)). -It runs through `layerfs_control::project::remove` -([`project.rs`](../../../../control/crates/layerfs-control/src/project.rs)), -under the Store's lease, locally or as the worker operation `project.remove`. +Returns `{ kind: "Removed", project, deleted_ms, home_entry_removed }`. The +Store marker is the commit point: if the home entry cannot be removed the +call still succeeds with `home_entry_removed: false` and keeps its durable +intent `store//delete/.json`, so a repeat from the same home +reuses `deleted_ms` and finishes the entry. The call is idempotent only +through that intent; it never resolves the name again. A lost worker reply +is `Unknown` and keeps the intent. | Error | When | Source | | --- | --- | --- | -| `NoSuchProject` | also when the Store holds no LayerStack of that name | [`identity.rs`](../../../../control/crates/layerfs-store/src/identity.rs) | -| `InUse` | a Sandbox of the project is recorded; `detail.related` lists `{ kind: "sandbox", name }` | [`inventory.rs`](../../../../control/crates/layerfs-control/src/inventory.rs) | -| `Unsupported` | a PostgreSQL Store ("removing a project from a PostgreSQL Store is not built"); the kernel guard refuses the Store's directory when opened (non-private mode/owner/type — another user's directory or group/other access instead of `0700` — or unsupported filesystem) | [`project/family.rs`](../../../../control/crates/layerfs-store/src/project/family.rs) | -| `Failed` | the Store write failed, its outcome included | `identity.rs` `remove_project` | -| `Unknown` | worker route: the answer was lost | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | +| `NoSuchStore`, `NoSuchProject` | the Store or project is missing (or already retired) | [`project/delete.rs`](../../../../control/crates/layerfs-store/src/project/delete.rs) | +| `MountsActive` | a mount of the project is live; its holders are named | [`history/admission.rs`](../../../../engine/crates/layerfs-store-service/src/history/admission.rs) | +| `Busy`, `Unresolved` | another user or operation holds the project, or a dead owner left an uncertain one | spec §5.1 | +| `Unsupported` | the Store is on another machine and the project record names no LayerStack id | [`project/delete.rs`](../../../../control/crates/layerfs-store/src/project/delete.rs) | +| `Unknown` | worker route: the answer was lost; the intent stays | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | + +### project.remove_file + +Hides one absolute project path from every node at or below its marker +frontier (spec §4.3). It writes one `path_removed` marker and changes no +pack, object, Commit or Layer; reads and stats of the path afterwards are +refused `Removed`. + +```text +JS await layerfs.project.remove_file(project, path) +Python layerfs.project.remove_file(project, path) +CLI layerfs project remove-file +``` + +The path is normalized (one leading `/` stripped) and must be an absolute +POSIX path of at most 4096 bytes and 256 components, with no empty, `.` or +`..` component and no `\` or NUL +([`path.rs`](../../../../core/crates/layerfs-history/src/contract/path.rs)). +Returns `{ kind: "Removed" | "UpToDate", project, path, frontier_serial, +removed_ms }`: the marker's stored values, `UpToDate` when no project node is +above this path's marker frontier. -After `Failed` or `Unknown` the home entry is kept (it is removed only after -success). `branch.list` answering `NoSuchProject` from the Store means the -stack is gone. +| Error | When | Source | +| --- | --- | --- | +| `InvalidInput` | the path is not a normalized absolute POSIX path | [`path.rs`](../../../../core/crates/layerfs-history/src/contract/path.rs) | +| `NoSuchStore`, `NoSuchProject` | the Store or project is missing (or retired) | [`methods/project.rs`](../../../../engine/crates/layerfs-store-service/src/methods/project.rs) | +| `MountsActive`, `Busy`, `Unresolved` | project-scope admission, as `project.delete` | spec §5.1 | +| `Unknown` | worker route: the answer was lost | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | ## Family errors diff --git a/docs/architecture/api/sdk/store.md b/docs/architecture/api/sdk/store.md index 9ecdbe155b..f29787e1cd 100644 --- a/docs/architecture/api/sdk/store.md +++ b/docs/architecture/api/sdk/store.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The `store` calls: create, connect, inspect, move and remove the databases that hold every project's history, within the [SDK](README.md). +The `store` calls: create, connect, inspect, move, remove and compact the databases that hold every project's history, within the [SDK](README.md). | | | | --- | --- | @@ -24,6 +24,7 @@ The `store` calls: create, connect, inspect, move and remove the databases that | (back to native) detach v v remove (file + projects) disconnect (data stays) + compact (reclaim dead bytes) ``` | Call | Kind | Returns | @@ -39,6 +40,7 @@ The `store` calls: create, connect, inspect, move and remove the databases that | [`store.push`](#storepush) | sqlite | `{ store, from, to, bytes, heads }` | | [`store.pull`](#storepull) | sqlite | `{ store, from, to, bytes, heads }` | | [`store.remove`](#storeremove) | sqlite | `{ removed: name }` | +| [`store.compact`](#storecompact) | sqlite | `{ kind: "Compacted", ... }`, the §11 report | | [`store.disconnect`](#storedisconnect) | PostgreSQL | `{ disconnected: name }` | A sqlite Store is made with the Disposable profile (WAL, @@ -128,7 +130,14 @@ leases are ignored, not removed password reference is resolved here and the server dialed once; a PostgreSQL Store is never `down` (an unreachable server shows in `server`) ([`status.rs`](../../../../control/crates/layerfs-store/src/status.rs)). It is -the call an `Unresolved` refusal names. Errors: `NotFound`; `InvalidInput` (a +the call an `Unresolved` refusal names. When a compaction runs on a sqlite +Store, `op` is read from `.compact-op`; on a PostgreSQL Store it is +the Store-side `gc_op` row with its `gc_intent`, read by one status query +with no Store session and no artifact read. Either way `state` is `busy` +while the owner lives and `unresolved` once it is gone, `uncertain` stays +the record's own fact, and `phases` names the backend's order (`mark`, +`build`, `verify`, `install` on sqlite; `mark`, `build`, `swap`, `cleanup` +on PostgreSQL). Errors: `NotFound`; `InvalidInput` (a record of unknown kind); `Unavailable` (a 1.1.0 Store whose machine is not known; `layerfs import-env` fixes it). @@ -372,6 +381,39 @@ uncertain. Errors: `InvalidInput` (a sqlite Store; use `store.remove`); recorded, or a call in this process is running on it, [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs)); `Failed`. +### store.compact + +Reclaims one sqlite Store's dead bytes by rebuilding it from its live history +(spec §3, §7): the only byte-reclaiming call, with no confirm, force or dry +run and no automatic collection. It writes its operation intent before the +first checkpoint, verifies the rebuilt Store, and installs it by rename; a +definite failure before the rename keeps the old Store and the generation `G`, +while an installed-but-uncertain reply stays at `G + 1` and is reported, never +deleted. A Store with less than 10% wholly dead packs takes the `none` path: +it marks only, writes nothing and reports `dead_bytes_refined == +dead_bytes_bound`. + +```text +JS await layerfs.store.compact(name) +Python layerfs.store.compact(name) +CLI layerfs store compact +``` + +Returns `{ kind: "Compacted", path, generation: {before, after}, ... }` +with the report's snake_case fields: `dead_bytes_bound`, +`dead_bytes_refined`, `logical_discarded`, `backend_reusable`, +`physical_shrink`, `kept_waste {dead_in_retained_packs_bytes, delta_bases +{bytes, references}, inode_blocks_bytes}`, `peak_temp_bytes`, `residue`, +`estimate` and `zero_reason`; a capped field is `{value, cap}`. + +| Error | When | Source | +| --- | --- | --- | +| `NoSuchStore`, `Unsupported` | a missing Store, a PostgreSQL Store, a non-private Store directory, or an unsupported filesystem | [`compact.rs`](../../../../control/crates/layerfs-store/src/compact.rs) | +| `MountsActive`, `Busy`, `Unresolved` | a mount, long-lived client or operation holds the Store, or a dead owner left an uncertain one | spec §5 | +| `StaleStore`, `Conflict` | a connection bypassed admission, or the sealed identity changed | spec §7, §10 | +| `Capacity` | a real allocation refusal during the rebuild; the old Store is kept | spec §7 | +| `Unknown` | the reply was lost around install; read `store.status` and the op before any second call | spec §9 | + ## Family errors | Error | When | Source | diff --git a/docs/architecture/engine/README.md b/docs/architecture/engine/README.md index 7031ed5431..a223f0cba6 100644 --- a/docs/architecture/engine/README.md +++ b/docs/architecture/engine/README.md @@ -46,7 +46,7 @@ crate except `layerfs-project`. | Topic | Crates | What it covers | Pages | | --- | --- | --- | --- | -| [Store](store/README.md) | `layerfs-persistence` | The one global Store: every project's history and immutable objects in one SQLite file or one PostgreSQL schema, written by one writer at a time. | [SQLite](store/sqlite.md), [PostgreSQL](store/postgresql.md), [Diagnostics](store/diagnostics.md) | +| [Store](store/README.md) | `layerfs-persistence` | The one global Store: every project's history and immutable objects in one SQLite file or one PostgreSQL schema, written by one writer at a time. | [SQLite](store/sqlite.md), [PostgreSQL](store/postgresql.md), [Diagnostics](store/diagnostics.md), [Testing](store/testing.md) | | [Store calls](store-calls/README.md) | `layerfs-store-service` | Typed reads, file changes and Branch history over the Store, run once in the process that opened it, and the SDK facade that names them. | [Reads](store-calls/reads.md) | | [Project Init](project-init/README.md) | `layerfs-project` | Turning one native directory into a project's first LayerStack and Branch: import, construction, then seal and install or a live add. | — | | [Overlay](overlay/README.md) | `layerfs-overlay`, `layerfs-store-service` `overlay/` | Each Workspace's own disposable SQLite file and in-memory open state, the Workspace roster and its one lock. | [Payload](overlay/payload.md), [Lifetimes](overlay/lifetimes.md) | diff --git a/docs/architecture/engine/store/README.md b/docs/architecture/engine/store/README.md index b2a670667e..9fd8da18d9 100644 --- a/docs/architecture/engine/store/README.md +++ b/docs/architecture/engine/store/README.md @@ -196,10 +196,16 @@ Source: [`records.rs`](../../../../engine/crates/layerfs-persistence/src/backend | [`binding_hint_blocks.rs`](../../../../engine/crates/layerfs-persistence/tests/binding_hint_blocks.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test binding_hint_blocks` | macOS, Linux | Hints merge across blocks without repeats | | [`postgresql_acquisition.rs`](../../../../engine/crates/layerfs-persistence/tests/postgresql_acquisition.rs) | `LAYERFS_TEST_POSTGRES_URL=… cargo +1.85.1 test --locked -p layerfs-persistence --features postgresql --test postgresql_acquisition` | a PostgreSQL server; prints `NOT_RUN` without the variable | Creating a PostgreSQL Store, refusing a second create, and Init's acquisition over it | +The commands above assume the test environment in [Testing](testing.md): `TMPDIR` and +`CARGO_TARGET_DIR` on ext4, xfs or btrfs (apfs or hfs on macOS). +[`tools/ci/test_store_ext4.sh`](../../../../tools/ci/test_store_ext4.sh) sets both +and refuses anything else before cargo starts. + ## Pages | Page | What it covers | | --- | --- | | [SQLite](sqlite.md) | The SQLite backend: schema, sessions, lane, profile, checkpoints, seal, snapshot, heads, acquisition and merge records. | | [PostgreSQL](postgresql.md) | The PostgreSQL backend: schema, open checks and authority, connections, transactions, publication, acquisition, access and status. | -| [Diagnostics](diagnostics.md) | Read-only Store audits: observations, the SQLite and PostgreSQL audits, table coverage and the structural fallback. | \ No newline at end of file +| [Diagnostics](diagnostics.md) | Read-only Store audits: observations, the SQLite and PostgreSQL audits, table coverage and the structural fallback. | +| [Testing](testing.md) | The filesystem the Store suites need, the harness that enforces it, and the PostgreSQL test server. | \ No newline at end of file diff --git a/docs/architecture/engine/store/postgresql.md b/docs/architecture/engine/store/postgresql.md index 688c129bbd..806926d1e8 100644 --- a/docs/architecture/engine/store/postgresql.md +++ b/docs/architecture/engine/store/postgresql.md @@ -70,7 +70,7 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`. | `Light` check | A Sandbox daemon's writer, Workspace and control readers, and `attach_read_only`; a hand-changed column fails the statement that uses it instead | [`bootstrap.rs`](../../../../engine/crates/layerfs-store-service/src/bootstrap.rs), [`owner.rs`](../../../../engine/crates/layerfs-daemon/src/application/owner.rs) | | `binding(selection)` | The binding key a schema records, read in one read-only session, so a home that did not create it can open it | [`store/postgresql.rs`](../../../../engine/crates/layerfs-persistence/src/store/postgresql.rs) | | `attach_read_only(selection)` | One read-only session under the recorded binding, returned for the caller to compare with its own | same | -| `postgresql_access(url, schema)` | Server version, one round trip's latency, `current_user`, whether the schema exists, and each privilege the role lacks: the schema's `USAGE`, the database's `TEMP` (merges stage in temporary tables), each table's `SELECT`/`INSERT`/`UPDATE`/`DELETE` and each sequence's `USAGE` | [`access.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/access.rs) | +| `postgresql_access(url, schema)` | Server version, one round trip's latency, `current_user`, whether the schema exists, and each privilege the role lacks: the schema's `USAGE` and `CREATE`, the database's `CREATE` (compaction creates the scratch schema) and `TEMP` (merges stage in temporary tables; compaction reads its visited set from them), `pg_read_all_stats` (compaction reads the recorded owner's `pg_stat_activity` row and calls `pg_terminate_backend`), each table's `SELECT`/`INSERT`/`UPDATE`/`DELETE` and each sequence's `USAGE` | [`access.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/access.rs) | | `postgresql_status(url, schemas)` | Server version, one round trip's latency and each named schema's size | [`status.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/status.rs) | `access` and `status` open one connection for the call and close it, run @@ -78,6 +78,11 @@ read-only queries only and never retry. An authentication or unknown database error is `Denied`; any other connection failure `Unreachable`. The session keeps the frozen policy and server version the open check read, so `PackPersistence::policy` costs no round trip afterwards. +`compaction_missing(session)` applies the same five Store-wide checks to the +compaction session in one read-only round trip, before the intent, the mark +and the `lfs_gc_` scratch schema; the first privilege the role lacks is +`Unsupported`, naming it (AC-44). It is reachable from a session because a +compaction holds one already and never retains the URL. `PostgresSelection`'s `Debug` redacts the URL. ## Session diff --git a/docs/architecture/engine/store/testing.md b/docs/architecture/engine/store/testing.md new file mode 100644 index 0000000000..d7c6ee0d9e --- /dev/null +++ b/docs/architecture/engine/store/testing.md @@ -0,0 +1,295 @@ +# Testing the Store + +> **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. + +The Store suites create real SQLite Stores, and a Store opens only on a local +filesystem that carries its locks. This page is the environment contract, and +the harness that enforces it before cargo spends a build. + +| | | +| --- | --- | +| Harness | [`tools/ci/test_store_ext4.sh`](../../../../tools/ci/test_store_ext4.sh) | +| Suites | `layerfs-persistence`, `layerfs-store-service`, `layerfs-store`, and the `store_*` targets of `layerfs-api` | +| Guard | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | +| CI | [`.github/workflows/store-ext4.yml`](../../../../.github/workflows/store-ext4.yml) | +| Related | [Store](README.md), [SQLite](sqlite.md), [PostgreSQL](postgresql.md), [Diagnostics](diagnostics.md) | + +## Why the tests need ext4 + +`kernel_guard::check` runs before a Store file is opened. It admits direct +opening only when the Store's own directory is a real directory (not a +symlink), owned by the current effective user, with mode `0700`, and the +filesystem holding it is `ext4`, `xfs` or `btrfs` on Linux, or `apfs` or `hfs` +on macOS. Anything else is `OpenRefusal::Unsupported` with the reason +`filesystem 0x… at … does not carry SQLite locks; direct Store access needs +ext4, xfs or btrfs` +([`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs)). + +The allowlist is not arbitrary. Every opener of one Store must share one kernel +and a filesystem with working advisory locks: SQLite's WAL locks and the writer +admission lane's `flock` on the Store's directory +([SQLite](sqlite.md)). `tmpfs`, `overlay`, FUSE mounts and network filesystems +do not coordinate those locks between the processes a suite starts, so a Store +there is refused rather than run without coordination. + +The suites keep their Stores in two directories, and both inherit the +filesystem of the test environment: + +| Directory | In the tests | Usual host | +| --- | --- | --- | +| `TMPDIR` | `std::env::temp_dir()` | `/tmp`, which is tmpfs in most containers | +| `$CARGO_TARGET_DIR/tmp` | `env!("CARGO_TARGET_TMPDIR")` | the checkout's `target/`, an overlay in a container | + +On a host whose `/tmp` is tmpfs, or in a container whose checkout is an +overlay, the guard refuses the Store and suites such as +`layerfs-store-service --test kernel_guard` fail at their "a private directory +on a supported filesystem passes" assertion. That is an environment failure, +not a product failure. Tests that must prove the refusal on purpose use +`/dev/shm` (tmpfs), so pointing `TMPDIR` at ext4 does not weaken them. + +## Run the suites + +The harness requires `LAYERFS_TEST_TMPDIR` on a supported filesystem, refuses +anything else with exit 2 before cargo starts, and points `TMPDIR` (through a +short symlink), `CARGO_TARGET_DIR` and `CARGO_HOME` at it. + +```bash +# A cross-crate subset: the first build is the cost, then minutes. +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick + +# Every Store suite. +LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh + +# Print the cargo commands and the environment, run nothing. +tools/ci/test_store_ext4.sh --list +``` + +`--quick` runs a few targets per suite: + +| Suite | Targets | What they exercise | +| --- | --- | --- | +| `layerfs-persistence` | `heads`, `sqlite_publication` | Branch heads and a publication round trip | +| `layerfs-store-service` | `kernel_guard`, `commit_pipeline` | the guard admits a private ext4 directory and refuses tmpfs; a Commit pipeline | +| `layerfs-store` | `create_directory` | `store.create`'s directory ownership and its guard refusal | +| `layerfs-api` | `store_family` | the Store family's call table | + +The full run is the unfiltered `layerfs-persistence`, `layerfs-store-service` +and `layerfs-store` suites, plus every `store_*.rs` target found under +`api/crates/layerfs-api/tests/`. Other API targets (machines, sandboxes, +Docker) are not Store tests and are not run here. + +## What the harness sets + +| Variable | Value | Why | +| --- | --- | --- | +| `LAYERFS_TEST_TMPDIR` | required, validated | the supported filesystem the run may use | +| `TMPDIR` | a short private wrapper under the system temporary directory, whose `tmp` entry links to `$LAYERFS_TEST_TMPDIR/tmp` | `std::env::temp_dir()` in the tests; Unix-domain socket paths are limited to about 108 bytes | +| `CARGO_TARGET_DIR` | `$LAYERFS_TEST_TMPDIR/target` | build output, and `CARGO_TARGET_TMPDIR` for the tests | +| `CARGO_HOME` | `$LAYERFS_TEST_TMPDIR/cargo-home` | the registry and git checkouts stay with the run, off the checkout's overlay | +| `-j` | `min(nproc, 16)` by default | keeps rustc within memory on a many-core host | +| `CARGO_NET_GIT_FETCH_WITH_CLI` | `true` | fetches the pinned `fuser` git dependency with the git CLI, which honours the host's proxy and credentials | +| `GIT_SSL_NO_VERIFY` | `true` | lets that one git fetch complete through TLS inspection; scoped to this process, not a production setting | +| `--features postgresql` | added to every suite when `LAYERFS_TEST_POSTGRES_URL` is set | the PostgreSQL test files are `#![cfg(feature = "postgresql")]` | + +The `TMPDIR` wrapper exists because a Sandbox run directory is long: a test that +binds a Unix-domain socket under `TMPDIR` otherwise fails with `path must be +shorter than SUN_LEN`. The wrapper is a `mktemp -d` directory under `/tmp` +(override with `LAYERFS_TEST_SYSTEM_TMP`), removed on exit, and holds only the +symlink; `statfs` follows it, so the Stores are still on the supported +filesystem. + +Overrides: `LAYERFS_TEST_JOBS`, `LAYERFS_TEST_CARGO_HOME`, +`LAYERFS_TEST_CARGO_TARGET_DIR` and `LAYERFS_TEST_SYSTEM_TMP`. Arguments after +`--` are appended to every `cargo test`, for example +`tools/ci/test_store_ext4.sh --quick -- --nocapture`. + +## Find a supported directory + +On Linux, ask the mount table: + +```bash +findmnt -n -o FSTYPE --target /path/to/candidate # ext4, xfs or btrfs +``` + +Inside a Docker Sandbox (the `namespaces` runtime), the sandbox run directory +is ext4 while `/tmp` is tmpfs and the checkout is an overlay: `$HOME` and +`/layerfs/data/machine/native/run/` report ext4, so a directory such +as `/layerfs/data/machine/native/run//tmp/layerfs-store-tests` is a +valid `LAYERFS_TEST_TMPDIR`. Never use `/tmp`. + +On macOS the corresponding local filesystems are `apfs` and `hfs`; the +harness reads the mount table there too. + +The build and the Stores live under `LAYERFS_TEST_TMPDIR`, so allow several +gigabytes on that mount. + +## PostgreSQL + +`LAYERFS_TEST_POSTGRES_URL` is a libpq connection string (despite the name), +not a URL: + +```bash +LAYERFS_TEST_POSTGRES_URL='host=127.0.0.1 port=55433 user=postgres password=… dbname=postgres sslmode=disable' \ + LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh +``` + +### Server and extensions + +Use PostgreSQL 17 or newer. The Store schema itself needs 16 or newer for +`STORAGE PLAIN` ([PostgreSQL](postgresql.md)), and the PostgreSQL suites are +qualified on 17.x, so 17 is the supported version for tests. + +The full-mode doctor calls `amcheck`'s `verify_heapam` and `bt_index_check` for +every heap and ready B-tree relation, and it never installs an extension itself +([Diagnostics](diagnostics.md)). Create the contrib extensions in the test +database before a full audit: + +```sql +CREATE EXTENSION IF NOT EXISTS amcheck; +CREATE EXTENSION IF NOT EXISTS pgstattuple; +``` + +```bash +psql "$LAYERFS_TEST_POSTGRES_URL" -c 'CREATE EXTENSION IF NOT EXISTS amcheck;' +``` + +`amcheck` is the required one: without it a full audit reports each relation as +`store.database.relations=Unknown`, because no supported function exists, and +`store.database.relations.summary` is `Unknown` too. That is missing coverage, +not corruption, but the PostgreSQL compaction tests assert that a rebuilt Store +audits cleanly with no `Fail` or `Unknown` observation. On a fresh cluster that +never created the extension, the five `clean()` assertions in +`postgresql_compact.rs` fail with exactly that finding; the +`assert_doctor_clean` assertions in `postgresql_compact_fixtures.rs` depend on +it too. `pgstattuple` is the companion contrib module; the current diagnostic +calls only `amcheck`, but the CI lane creates both so the scratch server +matches the qualified setup. + +The push lane is SQLite-only and does not need any of this. When +`LAYERFS_TEST_POSTGRES_URL` is set, the harness asks `psql` whether the +extensions exist and warns with the `CREATE EXTENSION` command when `amcheck` +is missing; an absent `psql` or an unreachable server downgrades that to a +message and never fails the run. `LAYERFS_TEST_EXPECT_AMCHECK` (any value) +makes `postgresql_diagnostic` assert `store.database.relations.summary` is +`Pass`; the `store-ext4` full job sets it after creating the extensions. + +### Credentials and TLS + +The PostgreSQL test fixtures and the doctor's admin client connect with +`postgres::Client::connect(…, NoTls)`, not through the Store's TLS policy. The +connection string must therefore carry its own password and must not require a +TLS handshake: use `sslmode=disable` on loopback or a private route, and +`password=…` unless the server uses trust authentication. `sslmode=require` or +`verify-full` is fine for the Store sessions, but the test fixtures reject it. +Do not put a password in a job log; pass the whole connection string through +the environment, as the workflow does with a scratch trust-authenticated +service. + +### The physical doctor's extra servers + +`postgresql_doctor_physical` needs two servers beyond the primary: + +| Variable | What it must be | +| --- | --- | +| `LAYERFS_TEST_POSTGRES_MISSING_URL` | a second database (or server) where `amcheck` is **not** installed; the test asserts that the missing extension is `Unknown` and that the doctor did not install it | +| `LAYERFS_TEST_POSTGRES_REPLICA_URL` | a physical standby of the primary, in recovery (`pg_is_in_recovery()` is true), with the test's schema replicated within 5 s | + +Create the amcheck-free database on the same scratch server, for example: + +```bash +psql "$LAYERFS_TEST_POSTGRES_URL" -c 'CREATE DATABASE layerfs_no_amcheck' +LAYERFS_TEST_POSTGRES_MISSING_URL='host=127.0.0.1 port=5432 user=postgres dbname=layerfs_no_amcheck sslmode=disable' +``` + +Both variables are optional: with the primary URL set, a test that needs one +and finds it unset prints `NOT_RUN: LAYERFS_TEST_POSTGRES_MISSING_URL is unset` +(or the replica variable) and returns. The `store-ext4` full job has no physical +standby and sets neither, so those extra cases print `NOT_RUN` and the target +stays green. Run the target separately with both for the physical coverage. + +Each PostgreSQL test creates and drops its own schema: point the variables at a +scratch server, never at a schema that holds a Store you care about, and make +sure the server is reachable from the process that runs the tests (inside a +Sandbox, `127.0.0.1` is the Sandbox). Without the variable the PostgreSQL test +files print `NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset` and check nothing; +the SQLite Store suites still run and still need the supported filesystem. + +### Compaction tests and per-test bindings + +The T9 PostgreSQL compaction suite (`postgresql_compact.rs`) gives every test +its own catalog binding, `postgresql-compact-`. PostgreSQL advisory locks +are server-wide, not schema-scoped, so two tests that shared a binding would +exclude each other while running in parallel even though each owns its schema. +The per-test binding fix is already in place; keep it when adding a case. Those +tests need only `LAYERFS_TEST_POSTGRES_URL`, `amcheck` and the supported +`LAYERFS_TEST_TMPDIR`, not the physical doctor's two extra URLs. + +## CI + +[`.github/workflows/store-ext4.yml`](../../../../.github/workflows/store-ext4.yml) +runs this harness in the repository's report-only CI. It adds no test +selection: every target it runs is already in the push or nightly lane that +[`tools/ci_tests.py`](../../../../tools/ci_tests.py) chooses; it runs them where +their guard admits a Store. Like the other lanes it is not a required check, an +agent does not wait for it, and a green run is not verification of a change +([AGENTS.md](../../../../AGENTS.md) §4). + +| Job | Trigger | Command | PostgreSQL | +| --- | --- | --- | --- | +| `store ext4 (--quick)` | push to `main` and pull requests touching `core/**`, `engine/**`, `control/**`, `api/crates/**` (not `layerfs-node`), `Cargo.*`, `.config/**`, `tools/ci/**`, `.cargo/**`; a markdown-only change does not run it | `tools/ci/test_store_ext4.sh --quick` | off | +| `store ext4 (--full, PostgreSQL 17)` | `schedule` (04:47 UTC) and manual dispatch | `tools/ci/test_store_ext4.sh --full` | a `postgres:17` service container | + +Both jobs set `LAYERFS_TEST_TMPDIR=/mnt/layerfs-store-tests`. On the +`ubuntu-24.04` runner `/mnt` is a separate ext4 data disk and `/tmp` is tmpfs; +the prepare step creates the directory, hands it to the runner user (it belongs +to `root` on the image) and asserts the filesystem with `findmnt`, so a runner +image without an ext4 `/mnt` fails in that step instead of as a guard refusal +inside a suite. The harness then sets `TMPDIR`, `CARGO_TARGET_DIR` and +`CARGO_HOME` itself, as it does locally. + +The scheduled job sets `LAYERFS_TEST_POSTGRES_URL` to +`host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=disable` +against that scratch service container, with trust authentication, so every +suite runs its PostgreSQL cases. Each PostgreSQL test creates and drops its own +schema, nothing outlives the run, and the runner reaches the container's +published port on `127.0.0.1`. The push job leaves the variable unset, so its +PostgreSQL cases print `NOT_RUN` and only the SQLite suites run. + +Before the suites, the full job installs `postgresql-client` and creates +`amcheck` and `pgstattuple` in the service container's database, then sets +`LAYERFS_TEST_EXPECT_AMCHECK=1` so `postgresql_diagnostic` asserts the +full-mode relation summary is `Pass`. A server that did not take the extension +fails in that step with its name, instead of as five `Unknown` findings inside +the compaction suite. The job also runs `postgresql_doctor_physical`. It needs +`LAYERFS_TEST_POSTGRES_MISSING_URL` and `LAYERFS_TEST_POSTGRES_REPLICA_URL` for +its extra cases; the full job provides neither, so those cases print `NOT_RUN` +and the target stays green. Run it separately with both for that coverage +(above). + +The harness moves `CARGO_HOME` and `CARGO_TARGET_DIR` onto +`LAYERFS_TEST_TMPDIR`, so these jobs do not restore the checkout-scoped +`Swatinem/rust-cache` the other workflows use: it would restore a `target/` +that the harness never reads. They install `mold` as the fast lane's `test` job +does, because they link the same test profile from a cold cache, and carry a +60-minute (`--quick`) or 120-minute (`--full`) job timeout. + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| `LAYERFS_TEST_TMPDIR is not set.` | the harness was run without the variable | set it to a directory on a supported filesystem | +| `… is on 'tmpfs', which cannot carry a Store.` | `/tmp` (or another tmpfs/overlay) was chosen | pick ext4, xfs or btrfs on Linux, apfs or hfs on macOS | +| a suite fails with `filesystem 0x… at … does not carry SQLite locks` | `TMPDIR` or `CARGO_TARGET_DIR` escaped the harness (exported by hand) | let the harness set both; do not override them to a scratch directory | +| `NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset` | no PostgreSQL server | expected; set the variable to run those cases | +| `store.database.relations=Unknown` and a compaction test fails on `clean` | `amcheck` was never created in the test database | `CREATE EXTENSION IF NOT EXISTS amcheck;`, then re-run; the harness warns when `psql` can see it is missing | +| `NOT_RUN: LAYERFS_TEST_POSTGRES_MISSING_URL is unset` (or `…_REPLICA_URL`) from `postgresql_doctor_physical` | the physical target's extra cases need an amcheck-free database and a physical standby; without them those cases skip | expected in the full sweep; set both (see [the physical doctor's extra servers](#the-physical-doctors-extra-servers)) and run that target separately for the physical coverage | +| `error: TLS support is disabled` or psql prompts for a password | the fixtures use `NoTls` and read the conninfo only | put `password=…` in the connection string and use `sslmode=disable` on a private route | +| `fatal: unable to access 'https://github.com/…/fuser.git'` or a TLS error | the pinned `fuser` git dependency could not be fetched | the harness sets `CARGO_NET_GIT_FETCH_WITH_CLI=true` and `GIT_SSL_NO_VERIFY=true`; check egress to `github.com` | +| `has mode 0755 … fix with chmod 700` | a left-over Store directory with group/other access | remove the named directory; the guard never changes permissions | +| `Updating crates.io index` stays silent for minutes | the sandbox network can stall cargo's multiplexed sparse-index connection | re-run with `CARGO_HTTP_MULTIPLEXING=false`; the download cache resumes where it stopped | +| `path must be shorter than SUN_LEN` | a Unix-domain socket path exceeded the kernel limit, usually because `TMPDIR` was set long by hand | use the harness (it keeps `TMPDIR` short) or pass a short `LAYERFS_TEST_TMPDIR` | +| `Store busy before effect` | another writer held the Store | not an environment failure; rerun the suite | + +The repository's test selection and CI lanes are separate: `tools/ci_tests.py` +decides which binaries the push and nightly runs take, and this harness only +gives the Store suites the filesystem they need. diff --git a/docs/architecture/flows/call.md b/docs/architecture/flows/call.md index 46905be63f..791ded8279 100644 --- a/docs/architecture/flows/call.md +++ b/docs/architecture/flows/call.md @@ -48,7 +48,7 @@ retries. A lost answer after the request was sent is `Unknown`, never resent. | # | What happens | Owning topic | | --- | --- | --- | | 1 | The front end makes one library call: the CLI parses `layerfs `; Node and Python pass JSON text. | [Front ends](../api/front-ends/README.md), [Bindings](../api/front-ends/bindings.md) | -| 2 | The call table finds `family.call` (73 calls in 10 families) and checks every argument before any effect. | [Calls](../api/calls/README.md#what-it-does), [SDK](../api/sdk/README.md) | +| 2 | The call table finds `family.call` (76 calls in 10 families) and checks every argument before any effect. | [Calls](../api/calls/README.md#what-it-does), [SDK](../api/sdk/README.md) | | 3 | A folder scope, when the client has one, guards and rewrites the checked arguments. | [Calls: folder scope](../api/calls/README.md#folder-scope) | | 4 | The home is opened (explicit path, else `LAYERFS_HOME`, else `~/.layerfs`). | [Home](../control/home/README.md) | | 5 | The family function reads the Store and project records, refuses a blocking intent and takes this process's lease on the Store. | [Stores and projects](../control/stores-and-projects/README.md#what-it-does), [Home: the `op` intent](../control/home/README.md#the-op-intent) | diff --git a/docs/architecture/glossary.md b/docs/architecture/glossary.md index 8654b30dc1..0a1703c9d1 100644 --- a/docs/architecture/glossary.md +++ b/docs/architecture/glossary.md @@ -18,7 +18,7 @@ LayerFS (backend, install, scope), the row gives both. | Branch | A named line of Commits in a project, with a base Layer and an optional head Commit. `main` is not a Branch. | [History: records](core/history/README.md#records) | | Bridge | `layerfs-bridge`: the wire contract between a controller and a Workspace daemon: the authenticated channel, every control and SDK record, the daemon's private setup and the Store install records. | [Bridge](engine/bridge/README.md) | | Bucket | A named place the home records where backups are kept, reached through a driver (local folder, MinIO, R2 or S3 API). Drivers know files by name and nothing about Stores. | [Backups: buckets](control/backups/buckets.md) | -| Call table | The 73 calls in 10 families that every front end exposes, with their arguments, and the one dispatcher they all go through. | [Calls](api/calls/README.md), [SDK](api/sdk/README.md) | +| Call table | The 76 calls in 10 families that every front end exposes, with their arguments, and the one dispatcher they all go through. | [Calls](api/calls/README.md), [SDK](api/sdk/README.md) | | Capture | The Commit step that seals a Workspace's active overlay generation as its published frontier and opens the next one. It copies no rows. | [Commit: what it does](engine/commit/README.md#what-it-does) | | Commit | An immutable history record: a complete filesystem root, its parent, the base Layer it was built from, an optional second parent and metadata. Also the call (`workspace.commit`) that makes one from a Workspace. | [History: records](core/history/README.md#records), [Commit](engine/commit/README.md) | | Construction | Building canonical file and tree objects from input and saving them: Init from a native folder, a Commit from its captured generation. A Commit has one construction producer. | [Commit: construction](engine/commit/construction.md), [Project Init: construction](engine/project-init/README.md#construction), [Files: construction](core/files/README.md#construction) | diff --git a/engine/crates/layerfs-bridge/src/control.rs b/engine/crates/layerfs-bridge/src/control.rs index 44a2822f4b..a82f5e6572 100644 --- a/engine/crates/layerfs-bridge/src/control.rs +++ b/engine/crates/layerfs-bridge/src/control.rs @@ -181,7 +181,7 @@ pub use crate::control_sdk::{ }; pub use crate::control_types::{ Activity, Answer, Call, CleanupObservation, ControlCode, ControlRefusal, LocalObservation, - Reply, Request, WorkspaceStatus, WorkspaceToken, HISTORY_WINDOW, + MountPlace, Reply, Request, WorkspaceStatus, WorkspaceToken, HISTORY_WINDOW, }; pub use crate::control_unmount::{DirtyReport, UnmountControl}; diff --git a/engine/crates/layerfs-bridge/src/control_request.rs b/engine/crates/layerfs-bridge/src/control_request.rs index 5c18098636..f49f9efdac 100644 --- a/engine/crates/layerfs-bridge/src/control_request.rs +++ b/engine/crates/layerfs-bridge/src/control_request.rs @@ -1,6 +1,6 @@ //! One correlated bounded control request, never an immutable-content upload. use crate::{ - control::{Call, ControlError, Request, HISTORY_WINDOW}, + control::{Call, ControlError, MountPlace, Request, HISTORY_WINDOW}, control_history::*, wire::{Reader, Writer}, }; @@ -51,6 +51,28 @@ impl Call { out.blob(project.as_bytes())?; out.blob(branch.as_bytes())?; } + Request::MountPlaced { + workspace, + branch, + place, + } => { + out.byte(16)?; + out.put(&workspace.to_bytes())?; + out.put(&branch.to_bytes())?; + put_place(&mut out, place)?; + } + Request::MountNamedPlaced { + workspace, + project, + branch, + place, + } => { + out.byte(17)?; + out.put(&workspace.to_bytes())?; + out.blob(project.as_bytes())?; + out.blob(branch.as_bytes())?; + put_place(&mut out, place)?; + } Request::Sdk(value) => { out.byte(20)?; crate::control_sdk::put_call(&mut out, value)?; @@ -169,6 +191,17 @@ impl Call { project: input.text(NAME_FIELD)?, branch: input.text(NAME_FIELD)?, }, + 16 => Request::MountPlaced { + workspace: history(WorkspaceId::from_authority(input.array()?))?, + branch: history(BranchId::from_bytes(input.array()?))?, + place: place(&mut input)?, + }, + 17 => Request::MountNamedPlaced { + workspace: history(WorkspaceId::from_authority(input.array()?))?, + project: input.text(NAME_FIELD)?, + branch: input.text(NAME_FIELD)?, + place: place(&mut input)?, + }, 2 => Request::Commit(crate::control_merge::commit_control(&mut input)?), 20 => Request::Sdk(crate::control_sdk::call(&mut input)?), 3 => Request::Status(token(&mut input)?), @@ -225,6 +258,38 @@ impl Call { Ok(Self { id, request }) } } +/// Writes the known Sandbox and machine names of one placed mount. +fn put_place(out: &mut Writer, place: &MountPlace) -> Result<(), ControlError> { + let flags = u8::from(place.sandbox.is_some()) | (u8::from(place.machine.is_some()) << 1); + out.byte(flags)?; + if let Some(sandbox) = &place.sandbox { + out.blob(sandbox.as_bytes())?; + } + if let Some(machine) = &place.machine { + out.blob(machine.as_bytes())?; + } + Ok(()) +} + +/// Reads the known Sandbox and machine names of one placed mount. +fn place(input: &mut Reader<'_>) -> Result { + let flags = input.byte()?; + if flags & !0b11 != 0 { + return Err(ControlError("mount place")); + } + let sandbox = if flags & 1 != 0 { + Some(input.text(NAME_FIELD)?) + } else { + None + }; + let machine = if flags & 2 != 0 { + Some(input.text(NAME_FIELD)?) + } else { + None + }; + Ok(MountPlace { sandbox, machine }) +} + fn window(limit: u16) -> Result<(), ControlError> { if limit == 0 || limit > HISTORY_WINDOW { Err(ControlError("history processing window")) diff --git a/engine/crates/layerfs-bridge/src/control_types.rs b/engine/crates/layerfs-bridge/src/control_types.rs index 141beadc5d..c6d6e3aa00 100644 --- a/engine/crates/layerfs-bridge/src/control_types.rs +++ b/engine/crates/layerfs-bridge/src/control_types.rs @@ -13,6 +13,37 @@ pub struct WorkspaceToken { /// Original local namespace returned by the owner; never reused by this daemon. pub namespace: i64, } +/// The Sandbox and machine a mount belongs to, when the caller knows them. +/// The holder report of a live mount shows them beside its Workspace; a +/// caller that knows neither leaves both absent and the report keeps the +/// Workspace alone. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct MountPlace { + /// The Sandbox the mounting daemon serves, when known. + pub sandbox: Option, + /// The machine that Sandbox runs on, when known. + pub machine: Option, +} + +impl MountPlace { + /// The place a caller names, by the names it knows (`None` when not). + /// An empty name is no name: only a real one is reported. + #[must_use] + pub fn new(sandbox: Option<&str>, machine: Option<&str>) -> Self { + let known = |name: Option<&str>| name.filter(|name| !name.is_empty()).map(str::to_owned); + Self { + sandbox: known(sandbox), + machine: known(machine), + } + } + + /// True when either name is known: a mount then reports what it knows. + #[must_use] + pub fn known(&self) -> bool { + self.sandbox.is_some() || self.machine.is_some() + } +} + /// One explicitly requested control operation. #[derive(Clone, Debug, Eq, PartialEq)] pub enum Request { @@ -58,6 +89,28 @@ pub enum Request { /// The Branch's name in that project, as the caller gave it. branch: String, }, + /// [`Request::Mount`] whose caller knows the Sandbox and machine its + /// daemon serves: a live mount's holders then show all three. + MountPlaced { + /// New authority incarnation. + workspace: WorkspaceId, + /// Selected Branch. + branch: BranchId, + /// The Sandbox and machine the mount belongs to, as known. + place: MountPlace, + }, + /// [`Request::MountNamed`] whose caller knows the Sandbox and machine + /// its daemon serves. + MountNamedPlaced { + /// New authority incarnation. + workspace: WorkspaceId, + /// The project's name, as the caller gave it. + project: String, + /// The Branch's name in that project, as the caller gave it. + branch: String, + /// The Sandbox and machine the mount belongs to, as known. + place: MountPlace, + }, /// One SDK call answered by a value or a typed SDK refusal. Sdk(crate::control_sdk::SdkCall), /// Capture the published frontier, construct it in the daemon, merge it diff --git a/engine/crates/layerfs-bridge/tests/control_records.rs b/engine/crates/layerfs-bridge/tests/control_records.rs index a708864895..7b6aa9cad5 100644 --- a/engine/crates/layerfs-bridge/tests/control_records.rs +++ b/engine/crates/layerfs-bridge/tests/control_records.rs @@ -57,6 +57,19 @@ fn commands_are_bounded_exact_and_reject_every_truncated_prefix() { project: "project".into(), branch: "Not A Name".into(), }, + // #G4: a mount that knows where it runs travels with both names + Request::MountPlaced { + workspace: token.workspace, + branch: binding.branch.id, + place: MountPlace::new(Some("sbx-1"), Some("docker")), + }, + // a name the caller knows alone still travels exactly + Request::MountNamedPlaced { + workspace: token.workspace, + project: "project".into(), + branch: "work".into(), + place: MountPlace::new(None, Some("machine-1")), + }, Request::Commit(CommitControl { token, force: false, diff --git a/engine/crates/layerfs-daemon/src/application/diagnostics.rs b/engine/crates/layerfs-daemon/src/application/diagnostics.rs index 6df30ad60c..68a1af0b41 100644 --- a/engine/crates/layerfs-daemon/src/application/diagnostics.rs +++ b/engine/crates/layerfs-daemon/src/application/diagnostics.rs @@ -397,5 +397,7 @@ fn operation_code(operation: &Request) -> u64 { Request::Cleanup(_) => 13, Request::Sdk(_) => 15, Request::MountNamed { .. } => 16, + Request::MountPlaced { .. } => 17, + Request::MountNamedPlaced { .. } => 18, } } diff --git a/engine/crates/layerfs-daemon/src/application/retained.rs b/engine/crates/layerfs-daemon/src/application/retained.rs index ff7011dd4d..75e24e72fc 100644 --- a/engine/crates/layerfs-daemon/src/application/retained.rs +++ b/engine/crates/layerfs-daemon/src/application/retained.rs @@ -278,6 +278,8 @@ fn kind(request: &layerfs_bridge::control::Request) -> &'static str { match request { Request::Mount { .. } => "mount", Request::MountNamed { .. } => "mount", + Request::MountPlaced { .. } => "mount", + Request::MountNamedPlaced { .. } => "mount", Request::Commit(_) => "commit", Request::Status(_) => "status", Request::Unmount(_) => "unmount", diff --git a/engine/crates/layerfs-daemon/src/control/operations.rs b/engine/crates/layerfs-daemon/src/control/operations.rs index 394d64e5e2..8188165975 100644 --- a/engine/crates/layerfs-daemon/src/control/operations.rs +++ b/engine/crates/layerfs-daemon/src/control/operations.rs @@ -9,7 +9,7 @@ use crate::store::{ CommitError, CommitFailure, CommitOptions, CommitSuccess, NamedBindRequest, }; use layerfs_bridge::control::{ - Activity, ConflictReport, ControlCode, MergedCommit, Reply, Request, WorkspaceToken, + Activity, ConflictReport, ControlCode, MergedCommit, MountPlace, Reply, Request, WorkspaceToken, }; use layerfs_content::filesystem::FilesystemRootId; use layerfs_content::merge::{Conflict, ConflictKind}; @@ -102,9 +102,20 @@ impl Service { workspace: *workspace, branch: *branch, }; - self.mount(BindIntent::Id(request), timing) + self.mount(BindIntent::Id(request), MountPlace::default(), timing) }) } + Request::MountPlaced { + workspace, + branch, + place, + } => crate::telemetry::timed("sandbox.mount.bind", |timing| { + let request = BindRequest { + workspace: *workspace, + branch: *branch, + }; + self.mount(BindIntent::Id(request), place.clone(), timing) + }), Request::MountNamed { workspace, project, @@ -115,7 +126,20 @@ impl Service { project: project.clone(), branch: branch.clone(), }; - self.mount(BindIntent::Named(request), timing) + self.mount(BindIntent::Named(request), MountPlace::default(), timing) + }), + Request::MountNamedPlaced { + workspace, + project, + branch, + place, + } => crate::telemetry::timed("sandbox.mount.bind", |timing| { + let request = NamedBindRequest { + workspace: *workspace, + project: project.clone(), + branch: branch.clone(), + }; + self.mount(BindIntent::Named(request), place.clone(), timing) }), Request::Status(token) => self.status(*token), Request::Cleanup(token) => self.cleanup(*token), @@ -146,6 +170,7 @@ impl Service { pub(crate) fn mount( &self, intent: BindIntent, + place: MountPlace, timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { let workspace = intent.workspace(); @@ -177,8 +202,11 @@ impl Service { entries.insert(workspace, Entry::Binding); } let bound = match crate::telemetry::phase(timing, "store.bind", || match intent { - BindIntent::Id(request) => self.store.bind(self.owner.clone(), request), - BindIntent::Named(request) => self.store.bind_named(self.owner.clone(), request), + BindIntent::Id(request) => self.store.bind_placed(self.owner.clone(), request, place), + BindIntent::Named(request) => { + self.store + .bind_named_placed(self.owner.clone(), request, place) + } }) { Ok(value) => value, Err(value) => { diff --git a/engine/crates/layerfs-daemon/tests/mount_holders.rs b/engine/crates/layerfs-daemon/tests/mount_holders.rs new file mode 100644 index 0000000000..a04962a3bb --- /dev/null +++ b/engine/crates/layerfs-daemon/tests/mount_holders.rs @@ -0,0 +1,127 @@ +//! #G4: a daemon mount registers the Sandbox and machine beside its +//! Workspace, and a mount that knows neither still registers the Workspace +//! alone. `MountsActive` reports exactly what the intent placed. +#[allow(dead_code)] +#[path = "support/control_fixture.rs"] +mod fixture; +#[allow(dead_code)] +#[path = "support/host_install.rs"] +mod support; + +use layerfs_bridge::control::{MountPlace, Reply, Request, SdkValue, WorkspaceToken}; +use layerfs_history::{BranchId, ForkRequest, ForkSource, HistoryName, LayerStackId, WorkspaceId}; +use layerfs_store_service::history::{lookup, removal}; + +/// The fixture's Store with a `work` Branch forked from the project's head. +fn work(f: &fixture::Fixture) -> (LayerStackId, BranchId) { + let store = &f.installed.opened.store; + let stack = LayerStackId::from_authority([54; 16]); + let base = store + .history() + .layer_stack(stack) + .unwrap() + .unwrap() + .head_layer; + let branch = BranchId::from_authority([64; 16]); + store + .history() + .fork(&ForkRequest { + stack, + branch, + name: HistoryName::new("work").unwrap(), + source: ForkSource::Layer(base), + }) + .unwrap(); + (stack, branch) +} + +/// Executes one mount request and returns the bound token. +fn bound(f: &fixture::Fixture, request: Request) -> WorkspaceToken { + match f.service.execute_control(&request).unwrap().reply { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + } +} + +/// A `workspace.mount`-shaped named bind that knows its Sandbox and machine. +fn bind_named_placed(f: &fixture::Fixture, tag: u8, place: MountPlace) -> WorkspaceToken { + bound( + f, + Request::MountNamedPlaced { + workspace: WorkspaceId::from_authority([tag; 32]).unwrap(), + project: "project".to_owned(), + branch: "work".to_owned(), + place, + }, + ) +} + +/// A named bind whose caller knows neither the Sandbox nor the machine. +fn bind_named(f: &fixture::Fixture, tag: u8) -> WorkspaceToken { + bound( + f, + Request::MountNamed { + workspace: WorkspaceId::from_authority([tag; 32]).unwrap(), + project: "project".to_owned(), + branch: "work".to_owned(), + }, + ) +} + +fn unmount(f: &fixture::Fixture, token: WorkspaceToken) { + let reply = f + .service + .execute_control(&Request::Unmount(layerfs_bridge::control::UnmountControl { + token, + discard: true, + })) + .unwrap() + .reply; + assert_eq!(reply, Reply::Unmounted(token)); +} + +/// The one holder a live mount leaves in `MountsActive`. +fn holder(f: &fixture::Fixture, stack: &layerfs_history::LayerStackRecord) -> SdkValue { + let refused = removal::remove(&f.installed.opened.store, stack, "work") + .expect_err("the live mount refuses the removal"); + assert_eq!(refused.code, "MountsActive", "{refused:?}"); + let SdkValue::List(holders) = refused.detail.get("holders").expect("holders") else { + panic!("{refused:?}"); + }; + assert_eq!(holders.len(), 1, "{holders:?}"); + holders[0].clone() +} + +/// The fixture project's live stack record, as the removal names it. +fn stack(f: &fixture::Fixture) -> layerfs_history::LayerStackRecord { + lookup::project(&f.installed.opened.store, "project").expect("project") +} + +#[test] +fn a_placed_daemon_mount_registers_sandbox_machine_and_workspace() { + let f = fixture::Fixture::new("g4-mount-place"); + work(&f); + let token = bind_named_placed(&f, 7, MountPlace::new(Some("g4sbx"), Some("docker"))); + let holder = holder(&f, &stack(&f)); + assert_eq!(holder.get_text("kind"), Some("mount")); + assert_eq!(holder.get_text("sandbox"), Some("g4sbx")); + assert_eq!(holder.get_text("machine"), Some("docker")); + let workspace = layerfs_bridge::sdk_values::hex(&[7; 32]); + assert_eq!(holder.get_text("workspace"), Some(workspace.as_str())); + unmount(&f, token); + f.cleanup(); +} + +#[test] +fn a_mount_with_no_known_context_registers_the_workspace_alone() { + let f = fixture::Fixture::new("g4-mount-unknown"); + work(&f); + let token = bind_named(&f, 8); + let holder = holder(&f, &stack(&f)); + assert_eq!(holder.get_text("kind"), Some("mount")); + assert_eq!(holder.get_text("sandbox"), None); + assert_eq!(holder.get_text("machine"), None); + assert!(holder.get_text("workspace").is_some()); + unmount(&f, token); + f.cleanup(); +} diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql index 42682b5053..3fb3e03c82 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_branch.sql @@ -4,7 +4,7 @@ FROM gc_intent WHERE state <> 'done' AND ( scope_kind = 'store' - OR ($1 IS NOT NULL AND scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'project' AND scope_id = $1) OR (scope_kind = 'branch' AND scope_id = $2) ) AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql index e9b5bdeac6..684af7e2ea 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_all_project.sql @@ -6,7 +6,7 @@ WHERE state <> 'done' scope_kind = 'store' OR (scope_kind = 'project' AND scope_id = $1) OR (scope_kind = 'branch' AND ( - ($2 IS NOT NULL AND project = $2) + (project = $2) OR scope_id IN (SELECT branch_id FROM branch WHERE layer_stack_id = $1) )) ) diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql index 124e372af9..cfce4edba7 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_branch.sql @@ -5,7 +5,7 @@ WHERE state <> 'done' AND kind IN ('removal', 'compaction') AND ( scope_kind = 'store' - OR ($1 IS NOT NULL AND scope_kind = 'project' AND scope_id = $1) + OR (scope_kind = 'project' AND scope_id = $1) OR (scope_kind = 'branch' AND scope_id = $2) ) AND op_id <> $3; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql index c427223f56..5ea9c9f72c 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/gc_intent_ops_project.sql @@ -7,7 +7,7 @@ WHERE state <> 'done' scope_kind = 'store' OR (scope_kind = 'project' AND scope_id = $1) OR (scope_kind = 'branch' AND ( - ($2 IS NOT NULL AND project = $2) + (project = $2) OR scope_id IN (SELECT branch_id FROM branch WHERE layer_stack_id = $1) )) ) diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs index 417d654060..488b0fee19 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/access.rs @@ -1,11 +1,13 @@ //! What a role may do in a Store's schema, for `store.attach`'s check of a -//! machine's connection: the server's version, one round trip's latency, -//! the role, and each privilege a Sandbox's writes need that it lacks. One -//! connection opened for the call and closed after it, read-only queries -//! only, no retry. -use crate::backend::postgresql::session::connection::Connector; +//! machine's connection and for a PostgreSQL compaction's preflight: the +//! server's version, one round trip's latency, the role, and each privilege +//! the Store's writes and compaction need that it lacks. One connection +//! opened for the call and closed after it, read-only queries only, no retry. +use crate::backend::postgresql::session::connection::{Connector, Session}; use crate::backend::postgresql::status::PostgresStatusError; +use crate::backend::records::{BackendError, Param}; use crate::PostgresSelection; +use layerfs_history::error::HistoryResult; use std::time::{Duration, Instant}; /// What the server answered for a role and a schema. @@ -20,18 +22,26 @@ pub struct PostgresAccess { /// Whether the database has the schema. pub schema: bool, /// Each privilege the role lacks, as ` on `, in a - /// fixed order: the schema's `USAGE`, the database's `TEMP`, then each - /// table's `SELECT`, `INSERT`, `UPDATE`, `DELETE` and each sequence's - /// `USAGE`. Empty when the role has them all (or there is no schema). + /// fixed order: the schema's `USAGE` and `CREATE`, the database's + /// `CREATE` and `TEMP`, `pg_read_all_stats`, then each table's `SELECT`, + /// `INSERT`, `UPDATE`, `DELETE` and each sequence's `USAGE`. Empty when + /// the role has them all (or there is no schema). pub missing: Vec, } -/// The role, the schema's `USAGE` (NULL without the schema) and the -/// database's `TEMP` (a merge stages in temporary tables). +/// The role, the schema's `USAGE` and `CREATE` (NULL without the schema), the +/// database's `CREATE` (a compaction creates the `lfs_gc_` scratch +/// schema) and `TEMP` (a merge stages in temporary tables, and a compaction +/// reads its visited set from them), and `pg_read_all_stats` (a compaction +/// reads another backend's `pg_stat_activity` row). const ROLE: &str = "SELECT current_user::text, \ CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = $1::text) \ THEN has_schema_privilege($1::text, 'USAGE') END, \ - has_database_privilege(current_database(), 'TEMP')"; + CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = $1::text) \ + THEN has_schema_privilege($1::text, 'CREATE') END, \ + has_database_privilege(current_database(), 'CREATE'), \ + has_database_privilege(current_database(), 'TEMP'), \ + pg_has_role('pg_read_all_stats', 'MEMBER')"; /// Each table and sequence privilege of the schema the role lacks. const MISSING: &str = "SELECT p.privilege || ' on ' || $1::text || '.' || c.relname \ @@ -44,6 +54,49 @@ const MISSING: &str = "SELECT p.privilege || ' on ' || $1::text || '.' || c.reln ELSE has_table_privilege(c.oid, p.privilege) END \ ORDER BY c.relkind DESC, c.relname, p.at"; +/// The five Store-wide privileges a compaction needs before it names or +/// creates its scratch schema, in the order [`compaction_missing`] reports a +/// missing one: the schema's `USAGE` and `CREATE`, the database's `CREATE` +/// and `TEMP`, and `pg_read_all_stats`. `CREATE` on the schema is the +/// `ALTER TABLE ... SET SCHEMA` of Phase B; `CREATE` on the database is the +/// scratch schema itself; `pg_read_all_stats` is the recorded owner's +/// `pg_stat_activity` row and its `pg_terminate_backend` proof. +const COMPACTION: &str = "SELECT \ + has_schema_privilege($1::text, 'USAGE')::int, \ + has_schema_privilege($1::text, 'CREATE')::int, \ + has_database_privilege(current_database(), 'CREATE')::int, \ + has_database_privilege(current_database(), 'TEMP')::int, \ + pg_has_role('pg_read_all_stats', 'MEMBER')::int"; + +/// The privilege named for each [`COMPACTION`] column, in its order. +const COMPACTION_NAMES: [&str; 5] = [ + "USAGE on the Store schema", + "CREATE on the Store schema", + "CREATE on the database", + "TEMP on the database", + "pg_read_all_stats", +]; + +/// The first Store-wide privilege `session`'s role lacks for a PostgreSQL +/// compaction, or `None` when it has them all: one read-only round trip, +/// taken before the mark and before the `lfs_gc_` scratch schema is +/// named or created. The caller reports the returned name as `Unsupported` +/// (AC-44). +pub(crate) fn compaction_missing(session: &Session) -> HistoryResult> { + session.run(false, |tx| { + let rows = tx.query(COMPACTION, vec![Param::Text(tx.schema.to_owned())])?; + let Some(row) = rows.first() else { + return Err(BackendError::Integrity.into()); + }; + for (at, privilege) in COMPACTION_NAMES.iter().enumerate() { + if row.get::(at)? == 0 { + return Ok(Some(*privilege)); + } + } + Ok(None) + }) +} + /// Reads what the role `url` logs in as may do in `schema`. pub fn access(url: &str, schema: &str) -> Result { let connector = Connector::new(&PostgresSelection::new(url, "public")) @@ -67,14 +120,26 @@ pub fn access(url: &str, schema: &str) -> Result = row.try_get(1).map_err(failed)?; - let temp: bool = row.try_get(2).map_err(failed)?; + let create_schema: Option = row.try_get(2).map_err(failed)?; + let create_database: bool = row.try_get(3).map_err(failed)?; + let temp: bool = row.try_get(4).map_err(failed)?; + let read_all_stats: bool = row.try_get(5).map_err(failed)?; let mut missing = Vec::new(); if usage == Some(false) { missing.push(format!("USAGE on schema {schema}")); } + if create_schema == Some(false) { + missing.push(format!("CREATE on schema {schema}")); + } + if !create_database { + missing.push("CREATE on the database".to_owned()); + } if !temp { missing.push("TEMP on the database".to_owned()); } + if !read_all_stats { + missing.push("pg_read_all_stats".to_owned()); + } if usage == Some(true) { for row in client.query(MISSING, &[&schema]).map_err(failed)? { missing.push(row.try_get(0).map_err(failed)?); diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs index a359f6cd6f..0ae5ebc265 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact.rs @@ -11,6 +11,7 @@ //! crash (or a lost COMMIT reply) is judged by the next `store.compact`: //! generation `G` removes the artifact, `G + 1` resumes cleanup. +use super::access::compaction_missing; use super::session::{ connection::Session, transaction::{Mode, Transaction}, @@ -20,13 +21,16 @@ use crate::backend::sqlite::compact::{ CompactFailpoint, CompactKeptWaste, CompactPath, CompactReceipt, CompactRequest, CompactResidue, CompactZeroReason, }; +use crate::compact_metrics::{ + self, CompactOutcome, CompactPhase, CompactRefusal, CompactRunObservation, +}; use crate::history::GcTuple; use crate::{MarkResult, PostgresHandles}; use layerfs_history::error::{HistoryError, HistoryResult}; use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; use layerfs_history::HistoryCatalog; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; /// The §3 rebuild threshold: rebuild when dead * 10 >= stored. const THRESHOLD: u64 = 10; @@ -68,6 +72,113 @@ struct Swapped { dead: DeadTotals, } +/// The phase clock and §11 values one measured PostgreSQL compaction observes. +/// +/// Phase M times the intent, the mark and the wholly-dead-pack decision, Phase +/// A the scratch copy, Phase B the swap, and Phase C the cleanup — also when a +/// later call resumes an installed `G + 1` leftover. The `pg_` phase values +/// keep these distinct from the SQLite §7 phases in the process totals. +struct RunMetrics { + started: Instant, + phase: Option<(CompactPhase, Instant)>, + phase_nanos: Vec<(CompactPhase, u64)>, + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + kept_waste: CompactKeptWaste, + residue_bytes: u64, + leftover_settled: bool, +} + +impl RunMetrics { + fn start() -> Self { + Self { + started: Instant::now(), + phase: None, + phase_nanos: Vec::new(), + generation_before: 0, + generation_after: 0, + dead_bytes_bound: 0, + dead_bytes_refined: 0, + kept_waste: CompactKeptWaste::default(), + residue_bytes: 0, + leftover_settled: false, + } + } + + /// Ends the current phase and starts `phase`; a phase is inclusive wall + /// time, never an exclusive CPU decomposition. + fn begin_phase(&mut self, phase: CompactPhase) { + self.end_phase(); + self.phase = Some((phase, Instant::now())); + } + + fn end_phase(&mut self) { + if let Some((phase, started)) = self.phase.take() { + self.phase_nanos.push(( + phase, + u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), + )); + } + } + + /// Records this run once, whatever its result. The outcome and refusal + /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted + /// without changing the error the caller receives. + fn finish(mut self, result: &HistoryResult) { + self.end_phase(); + let total_nanos = u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let (outcome, refusal) = match result { + Ok(receipt) => { + // A resumed `G + 1` round reports the original run's + // generation, so the receipt is the authority here. + self.generation_before = receipt.generation_before; + self.generation_after = receipt.generation_after; + self.dead_bytes_bound = receipt.dead_bytes_bound; + self.dead_bytes_refined = receipt.dead_bytes_refined; + self.kept_waste = receipt.kept_waste.clone(); + self.residue_bytes = receipt.residue.bytes; + let outcome = match receipt.path { + CompactPath::Rebuild => CompactOutcome::Rebuild, + CompactPath::None => CompactOutcome::None, + }; + (outcome, None) + } + Err(error) => { + if !error.unknown() { + self.generation_after = self.generation_before; + } + ( + if error.unknown() { + CompactOutcome::Uncertain + } else { + CompactOutcome::Failed + }, + match error { + HistoryError::Busy | HistoryError::BusyOp(_) => Some(CompactRefusal::Busy), + HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), + _ => None, + }, + ) + } + }; + compact_metrics::record_run(&CompactRunObservation { + outcome, + generation_before: self.generation_before, + generation_after: self.generation_after, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + kept_waste: self.kept_waste, + residue_bytes: self.residue_bytes, + phase_nanos: self.phase_nanos, + total_nanos, + leftover_settled: self.leftover_settled, + refusal, + }); + } +} + fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -699,6 +810,7 @@ fn settle_leftover( handles: &PostgresHandles, session: &Arc, schema: &str, + metrics: &mut RunMetrics, ) -> HistoryResult> { let Some(op) = read_op(session, schema)? else { return Ok(None); @@ -720,6 +832,7 @@ fn settle_leftover( } } } + metrics.begin_phase(CompactPhase::PostgresCleanup); if let Some(artifact) = &op.artifact { drop_scratch(session, artifact)?; } @@ -730,13 +843,18 @@ fn settle_leftover( ))?; Ok(()) })?; + metrics.end_phase(); + metrics.leftover_settled = true; return Ok(None); } if current != op.generation + 1 { return Err(HistoryError::Integrity("compaction op generation")); } + metrics.begin_phase(CompactPhase::PostgresCleanup); let op_id = compaction_intent(session, schema)?; cleanup(handles, session, schema, op.dead, op.totals_applied, op_id)?; + metrics.end_phase(); + metrics.leftover_settled = true; Ok(Some(report( CompactPath::Rebuild, u64::try_from(op.generation).unwrap_or(0), @@ -1153,7 +1271,9 @@ fn round( session: &Arc, schema: &str, request: &CompactRequest, + metrics: &mut RunMetrics, ) -> HistoryResult { + metrics.begin_phase(CompactPhase::PostgresMark); let (mark, tuple) = handles.history.mark_gc()?; load_visited(session, schema, &mark)?; let stored = u64::try_from(total_stored(session, schema)?).unwrap_or(0); @@ -1176,6 +1296,7 @@ fn round( }; let rebuild = stored > 0 && dead_bytes.saturating_mul(THRESHOLD) >= stored; if !rebuild { + metrics.end_phase(); clear_round(session, schema)?; let _ = handles.history.gc_release(&request.op_id); return Ok(report( @@ -1194,6 +1315,7 @@ fn round( mark.history().len() as u64, )); } + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterMark, @@ -1201,7 +1323,9 @@ fn round( schema, "mark", )?; + metrics.begin_phase(CompactPhase::PostgresBuild); let mut built = build(session, schema, &scratch, &mark, &tuple)?; + metrics.end_phase(); built.dead.logical = mark_dead.logical; stop( request, @@ -1210,7 +1334,9 @@ fn round( schema, "build", )?; + metrics.begin_phase(CompactPhase::PostgresSwap); let swapped = swap(handles, session, schema, &tuple, &mark, &built)?; + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterSwap, @@ -1219,6 +1345,7 @@ fn round( "swap", )?; let logical_discarded = u64::try_from(swapped.dead.logical).unwrap_or(0); + metrics.begin_phase(CompactPhase::PostgresCleanup); cleanup( handles, session, @@ -1227,6 +1354,7 @@ fn round( false, Some(request.op_id), )?; + metrics.end_phase(); stop( request, CompactFailpoint::PostgresAfterCleanup, @@ -1351,10 +1479,29 @@ impl PostgresHandles { /// deletes only the persisted dead set. The caller keeps its session; the /// Store is usable at `G + 1` from the swap on. pub fn compact(&self, request: &CompactRequest) -> HistoryResult { + let mut metrics = RunMetrics::start(); + let result = self.compact_measured(request, &mut metrics); + metrics.finish(&result); + result + } + + /// The measured body of [`Self::compact`]; `metrics` observes it. + fn compact_measured( + &self, + request: &CompactRequest, + metrics: &mut RunMetrics, + ) -> HistoryResult { let session = self.storage.session.clone(); let schema = session.settings.schema.clone(); - let _ = read_generation(&session, &schema)?; - if let Some(receipt) = settle_leftover(self, &session, &schema)? { + // AC-44: a restricted role is refused before the intent, the mark and + // the scratch schema; the missing privilege is named, never a raw + // `42501` from whichever statement happened to run first. + if let Some(missing) = compaction_missing(&session)? { + return Err(HistoryError::Unsupported(missing)); + } + let generation = read_generation(&session, &schema)?; + metrics.generation_before = u64::try_from(generation).unwrap_or(0); + if let Some(receipt) = settle_leftover(self, &session, &schema, metrics)? { return Ok(receipt); } let intent = GcIntent::operation( @@ -1372,7 +1519,7 @@ impl PostgresHandles { let result = (|| { let rank = effective_sync_rank(&session)?; standby_precheck(&session, rank)?; - round(self, &session, &schema, request) + round(self, &session, &schema, request, metrics) })(); if result.is_err() { let _ = self.history.gc_release(&request.op_id); diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs index c9b7d7a7a2..69eeb53eb6 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/status.rs @@ -2,7 +2,12 @@ //! its version, one round trip's latency and the size of each named schema //! (tables and materialized views with their indexes and TOAST), on one //! connection opened for the call and closed after it, with no retry. +//! +//! It also reads a Store's compaction op — the `gc_op` row with its +//! `gc_intent` — for `store.status` (T10.5): one query, with no Store open, +//! schema change, artifact read or scratch-schema read. use crate::backend::postgresql::session::connection::Connector; +use crate::report::{CompactStatus, POSTGRES_COMPACT_PHASES}; use crate::PostgresSelection; use std::time::{Duration, Instant}; @@ -80,3 +85,182 @@ pub fn status(url: &str, schemas: &[String]) -> Result, + /// The owning backend's `backend_start`, when the row records one. + pub owner_backend_start: Option, + /// The owning process runtime `gc_intent` recorded, when it left one. + pub owner_runtime: Option, + /// The owning process boot id `gc_intent` recorded, when it left one. + pub owner_boot: Option, + /// The `gc_intent` row's own state (`live`, `uncertain` or `done`), when + /// it is still there; a crash leaves `live`, so this is evidence, not + /// liveness. + pub intent_state: Option, + /// The mark's history serial, when recorded. + pub mark_serial: Option, + /// The mark's next pack id, when recorded. + pub mark_pack_id: Option, + /// The mark's live-heads digest, when recorded. + pub mark_digest: Option>, + /// True while the row remains: the run did not settle, so its outcome is + /// unknown. `store.status` reports `busy` while the owner lives and + /// `unresolved` once it is gone. + pub uncertain: bool, + /// True when the recorded owner backend is still alive and not `idle`, + /// the same §9 test a resolving compaction applies. + pub alive: bool, +} + +impl PostgresCompactOpStatus { + /// The `store.status` projection of this operation: `busy` while the + /// owner lives, `unresolved` once it is gone. `uncertain` stays true for + /// every remaining row, live or leftover; `state` separates the two. The + /// phases are PostgreSQL's §8 ones: `mark`, `build`, `swap`, `cleanup`. + #[must_use] + pub fn status(&self) -> CompactStatus { + CompactStatus { + id: format!("op-{}", hex(&self.op_id)), + name: "store.compact", + phase: self.phase.clone(), + started: self.started_ms, + uncertain: self.uncertain, + state: if self.alive { "busy" } else { "unresolved" }, + phases: &POSTGRES_COMPACT_PHASES, + } + } +} + +/// `gc_op` plus its compaction `gc_intent` and the owner's live state, in one +/// query on one status connection. Returns `None` when the Store has no +/// compaction row (idle), or when a Store written before the GC tables has no +/// `gc_op` at all. No Store open, schema mutation, artifact or scratch read +/// happens here. +pub fn compact_status( + url: &str, + schema: &str, +) -> Result, PostgresStatusError> { + let connector = Connector::new(&PostgresSelection::new(url, schema)) + .map_err(|_| PostgresStatusError::Unreachable("the URL does not parse".to_owned()))?; + let mut client = connector + .connect_raw() + .map_err(|error| match error.code() { + Some(code) if code.code().starts_with("28") || code.code() == "3D000" => { + PostgresStatusError::Denied(error.to_string()) + } + _ => PostgresStatusError::Unreachable(error.to_string()), + })?; + let table = quote_ident(schema); + let sql = format!( + "SELECT o.generation,o.phase,o.started_ms,o.pid,o.backend_start,o.artifact,\ + o.mark_serial,o.mark_pack_id,o.mark_digest,\ + i.op_id,i.owner_runtime,i.owner_boot,i.state,\ + EXISTS (SELECT 1 FROM pg_stat_activity a \ + WHERE a.datname=current_database() AND a.pid=o.pid \ + AND a.backend_start::text=o.backend_start \ + AND a.state IS DISTINCT FROM 'idle') \ + FROM {table}.gc_op o \ + LEFT JOIN LATERAL (SELECT op_id,owner_runtime,owner_boot,state \ + FROM {table}.gc_intent WHERE kind='compaction' \ + ORDER BY updated_ms LIMIT 1) i ON TRUE \ + WHERE o.id=1" + ); + let rows = match client.query(&sql, &[]) { + Ok(rows) => rows, + // A Store opened before the GC tables has no row to read: idle. + Err(error) + if matches!( + error.code().map(postgres::error::SqlState::code), + Some("42P01" | "3F000") + ) => + { + let _ = client.close(); + return Ok(None); + } + Err(error) => return Err(PostgresStatusError::Failed(error.to_string())), + }; + let Some(row) = rows.first() else { + let _ = client.close(); + return Ok(None); + }; + let failed = |error: postgres::Error| PostgresStatusError::Failed(error.to_string()); + let intent: Option> = row.try_get(9).map_err(failed)?; + let artifact: Option = row.try_get(5).map_err(failed)?; + let op = PostgresCompactOpStatus { + op_id: op_identity(intent, artifact.as_deref()), + generation: row.try_get(0).map_err(failed)?, + phase: row.try_get(1).map_err(failed)?, + started_ms: row.try_get(2).map_err(failed)?, + owner_pid: row.try_get(3).map_err(failed)?, + owner_backend_start: row.try_get(4).map_err(failed)?, + owner_runtime: row.try_get(10).map_err(failed)?, + owner_boot: row.try_get(11).map_err(failed)?, + intent_state: row.try_get(12).map_err(failed)?, + mark_serial: row.try_get(6).map_err(failed)?, + mark_pack_id: row.try_get(7).map_err(failed)?, + mark_digest: row.try_get(8).map_err(failed)?, + // A remaining row is an unsettled run: the record's own fact. It is + // never inferred from the owner's liveness. + uncertain: true, + alive: row.try_get(13).map_err(failed)?, + }; + let _ = client.close(); + Ok(Some(op)) +} + +/// The `lfs_gc_` scratch name `gc_op.artifact` records carries the +/// operation identity, so a row whose `gc_intent` was already released still +/// projects a stable `op.id`. +fn op_identity(intent: Option>, artifact: Option<&str>) -> [u8; 16] { + if let Some(bytes) = intent { + if let Ok(op) = <[u8; 16]>::try_from(bytes.as_slice()) { + return op; + } + } + artifact + .and_then(|name| name.strip_prefix("lfs_gc_")) + .and_then(unhex) + .unwrap_or([0; 16]) +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn unhex(text: &str) -> Option<[u8; 16]> { + if text.len() != 32 { + return None; + } + let mut out = [0u8; 16]; + for (index, byte) in out.iter_mut().enumerate() { + *byte = u8::from_str_radix(text.get(index * 2..index * 2 + 2)?, 16).ok()?; + } + Some(out) +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs index 61c3b99aef..e3f07fe462 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact.rs @@ -22,6 +22,9 @@ use super::schema; use super::session::connection::{Session, SqlWork}; use super::session::rows; use crate::backend::records::BackendError; +use crate::compact_metrics::{ + self, CompactOutcome, CompactPhase, CompactRefusal, CompactRunObservation, +}; use crate::{KeptWaste, MarkResult, SqliteHandles}; use layerfs_content::ObjectId; use layerfs_history::error::{HistoryError, HistoryResult}; @@ -33,7 +36,7 @@ use std::cell::RefCell; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; /// The sidecar record suffix (§11: SQLite keeps its phase outside the sealed /// database). @@ -42,6 +45,11 @@ const OP_SUFFIX: &str = ".compact-op"; const COMPACT_SUFFIX: &str = ".compact"; /// The §6.3 visited scratch database. const VISITED_SUFFIX: &str = ".compact-visited"; +/// Upper bound on the rollback-journal header: SQLite pads it to the +/// filesystem's sector size and writes a header for each transaction. Two +/// sectors cover the prune and rewrite headers. +const JOURNAL_HEADER_BOUND: u64 = 8192; + /// The full SQLite artifact set of §9, without the variable `-mj*` journals. const ARTIFACT_SUFFIXES: [&str; 7] = [ COMPACT_SUFFIX, @@ -123,6 +131,12 @@ pub enum CompactFailpoint { AfterSeal, /// Inside the build, before the visited scratch or the copy is written. BeforeCopy, + /// Inside the build, immediately before `VACUUM INTO` starts the copy. + /// SQLite runs that copy as one uninterruptible call, so a real-process + /// `SIGKILL` cannot land inside it; the real-kill arm stops here with the + /// visited scratch present and the staged file not yet created. The + /// in-process `DuringCopy` arm keeps the torn-target state. + BeforeVacuum, /// Inside the build, after `VACUUM INTO` produced `.compact` and /// before the copy is pruned: the state a crash during the copy leaves. DuringCopy, @@ -161,6 +175,7 @@ impl CompactFailpoint { match self { Self::BeforeSeal | Self::BeforeSealRead | Self::AfterSeal => "mark", Self::BeforeCopy + | Self::BeforeVacuum | Self::DuringCopy | Self::AfterBuild | Self::CapacityAtBuild @@ -736,6 +751,31 @@ fn live_page_metrics(session: &Arc) -> HistoryResult<(u64, u64)> { .map_err(Into::into) } +/// The rollback-journal upper bound of one transaction on a `pages`-page file: +/// one record per modified page (`page_size + 8` bytes: page number, page image +/// and checksum) plus the header allowance. A page is journaled at most once +/// per transaction, so the staged copy's page count bounds both the prune and +/// the rewrite journal, and the bound never under-reports the observed file. +const fn journal_bound_bytes(pages: u64, page_size: u64) -> u64 { + pages + .saturating_mul(page_size.saturating_add(8)) + .saturating_add(JOURNAL_HEADER_BOUND) +} + +/// The staged file's page count and page size, before the prune frees pages. +fn file_page_extent(connection: &Connection) -> HistoryResult<(u64, u64)> { + let pages: i64 = connection + .query_row("PRAGMA page_count", params![], |row| row.get(0)) + .map_err(db_error)?; + let page: i64 = connection + .query_row("PRAGMA page_size", params![], |row| row.get(0)) + .map_err(db_error)?; + Ok(( + u64::try_from(pages).map_err(|_| HistoryError::Integrity("page count"))?, + u64::try_from(page).map_err(|_| HistoryError::Integrity("page size"))?, + )) +} + fn file_page_metrics(connection: &Connection) -> HistoryResult<(u64, u64)> { let free: i64 = connection .query_row("PRAGMA freelist_count", params![], |row| row.get(0)) @@ -852,6 +892,10 @@ CREATE TABLE live_pack (pack_id INTEGER PRIMARY KEY) STRICT, WITHOUT ROWID;", struct Built { target: PathBuf, target_bytes: u64, + /// The staged copy's extent right after `VACUUM INTO`, before the prune. + staged_bytes: u64, + /// The rollback-journal upper bound the staged page count implies. + journal_bound_bytes: u64, visited_bytes: u64, logical: u64, canonical: u64, @@ -861,8 +905,24 @@ struct Built { } impl Built { + /// §11 peak temporary bytes: the run's whole artifact footprint below the + /// rename, as a documented upper bound rather than a sampled peak. + /// + /// The staged copy is the whole sealed Store (`VACUUM INTO`), so it keeps + /// the source extent until the rewrite's final truncate. The rewrite then + /// writes the new Store into SQLite's temp database (the process temp + /// directory, unlinked) while rewriting the staged file in place, and both + /// the prune and the rewrite journal the pages they overwrite. The journal + /// term bounds either journal by one record per staged page + /// (`page_size + 8` bytes each: page number, image and checksum) plus the + /// header allowance, so it never under-reports the observed journal. The + /// sum over-reports the measured 50 GB peak by a few percent; §13 already + /// marks the report an estimate. fn peak_temp_bytes(&self) -> u64 { - self.target_bytes.saturating_add(self.visited_bytes) + self.staged_bytes + .saturating_add(self.journal_bound_bytes) + .saturating_add(self.target_bytes) + .saturating_add(self.visited_bytes) } } @@ -882,19 +942,32 @@ fn table_present(connection: &Connection, name: &str) -> HistoryResult { Ok(found == 1) } +/// The sealed Store and the mark a staged build is measured against. +struct BuildInput<'a> { + facts: &'a SealedFacts, + mark: &'a MarkResult, + tuple: &'a MarkTuple, + packs: &'a BTreeMap, + layout: crate::SqlitePackLayout, + synchronous: i64, +} + /// The §7 steps 2–5: build the staged Store, copy only retained rows, compact /// it, and switch it to WAL. fn build( - facts: &SealedFacts, - mark: &MarkResult, - tuple: &MarkTuple, - packs: &BTreeMap, - layout: crate::SqlitePackLayout, - synchronous: i64, + input: BuildInput<'_>, capacity: bool, request: &CompactRequest, op: &RunOp, ) -> HistoryResult { + let BuildInput { + facts, + mark, + tuple, + packs, + layout, + synchronous, + } = input; let store = &facts.path; let target = compact_path(store); let visited = visited_path(store); @@ -913,6 +986,9 @@ fn build( return Err(io_error(std::io::Error::from_raw_os_error(28))); } + // A real-process crash arm stops here: the visited scratch is written and + // `VACUUM INTO` has not started. See `BeforeVacuum`. + stop_at(request, CompactFailpoint::BeforeVacuum, op, store)?; let source = open_immutable(store)?; let target_text = target .to_str() @@ -925,6 +1001,9 @@ fn build( .map_err(|(_, error)| db_error(error))?; // The copy produced the staged file but has not pruned or verified it. stop_at(request, CompactFailpoint::DuringCopy, op, store)?; + // The staged copy holds the source's whole extent until the final + // `VACUUM` truncates it; §11 counts it in the peak temp. + let staged_bytes = std::fs::metadata(&target).map_err(io_error)?.len(); let connection = Connection::open_with_flags( &target, @@ -939,6 +1018,10 @@ fn build( "PRAGMA journal_mode=DELETE; PRAGMA synchronous={synchronous};" )) .map_err(db_error)?; + // The prune and the rewrite journal the pages they overwrite; one record + // per staged page bounds either journal before the staged file shrinks. + let (staged_pages, page_size) = file_page_extent(&connection)?; + let journal_bound_bytes = journal_bound_bytes(staged_pages, page_size); // The copied published-pack rows may be deleted in the artifact; the live // immutability triggers are restored before the file is verified. // `sqlite_sequence` is the acquisition operation counter: the copy keeps @@ -1157,6 +1240,8 @@ metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", Ok(Built { target, target_bytes, + staged_bytes, + journal_bound_bytes, visited_bytes, logical: u64::try_from(totals.0).unwrap_or(0), canonical: u64::try_from(totals.1).unwrap_or(0), @@ -1166,20 +1251,33 @@ metadata=(SELECT COALESCE(SUM(length),0) FROM pack WHERE domain=0) WHERE id=1;", }) } +/// The staged Store and the mark a verification reads it against. +struct VerifyInput<'a> { + store: &'a Path, + built: &'a Built, + mark: &'a MarkResult, + tuple: &'a MarkTuple, + layout: crate::SqlitePackLayout, + acquisition: crate::SqliteAcquisitionSchema, +} + /// The §7 step 6 verification through an `immutable=1` reader: schema identity, /// foreign keys, row counts against the mark, both live-heads digests, the /// allocator tuple and the recomputed totals. `quick_check` and the full doctor /// stay separate commands. fn verify( - store: &Path, - built: &Built, - mark: &MarkResult, - tuple: &MarkTuple, - layout: crate::SqlitePackLayout, - acquisition: crate::SqliteAcquisitionSchema, + input: VerifyInput<'_>, request: &CompactRequest, op: &RunOp, ) -> HistoryResult<()> { + let VerifyInput { + store, + built, + mark, + tuple, + layout, + acquisition, + } = input; let connection = open_immutable(&built.target)?; schema::check( &connection, @@ -1461,6 +1559,7 @@ fn settle_leftover( history: &crate::HistoryProvider, store: &Path, _generation: u64, + metrics: &mut RunMetrics, ) -> HistoryResult<()> { let Some(op) = RunOp::read(store) else { if artifact_paths(store) @@ -1482,6 +1581,7 @@ fn settle_leftover( if std::fs::symlink_metadata(op_path(store)).is_ok() { std::fs::remove_file(op_path(store)).map_err(io_error)?; } + metrics.leftover_settled = true; Ok(()) } @@ -1517,6 +1617,30 @@ impl Drop for WriterGuard { } } +/// Set by the real-process crash harness (T12.2's second arm): when present, +/// a reached failpoint connects to this Unix socket, writes one readiness +/// byte and blocks until the harness SIGKILLs the process. Production callers +/// never set it, so a failpoint keeps its in-process behaviour. +const CRASH_GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; + +/// Holds a reached failpoint until the test harness kills this process. The +/// variable being absent is a no-op: the in-process stop applies. +#[cfg(unix)] +fn crash_gate() -> HistoryResult<()> { + use std::io::{Read, Write}; + use std::os::unix::net::UnixStream; + let Some(path) = std::env::var_os(CRASH_GATE_ENV) else { + return Ok(()); + }; + let mut gate = UnixStream::connect(path).map_err(io_error)?; + gate.write_all(b"R").map_err(io_error)?; + gate.flush().map_err(io_error)?; + // The harness never releases the gate: it SIGKILLs this process here. + let mut release = [0u8; 1]; + let _ = gate.read(&mut release); + Ok(()) +} + /// A test-only crash stop: records the phase and leaves the state a crash /// would leave. fn stop_at( @@ -1531,6 +1655,8 @@ fn stop_at( let mut stopped = op.clone(); stopped.phase = point.phase().to_owned(); stopped.write(store)?; + #[cfg(unix)] + crash_gate()?; Err(HistoryError::UnknownOutcome) } @@ -1660,9 +1786,21 @@ impl SqliteHandles { /// directory-fsync failure is installed-but-uncertain: the new file and the /// op record are kept for the next `store.compact` to judge (§9). pub fn compact(self, request: &CompactRequest) -> HistoryResult { + let mut metrics = RunMetrics::start(); + let result = self.compact_measured(request, &mut metrics); + metrics.finish(&result); + result + } + + /// The measured body of [`Self::compact`]; `metrics` observes it. + fn compact_measured( + self, + request: &CompactRequest, + metrics: &mut RunMetrics, + ) -> HistoryResult { #[cfg(not(any(target_os = "linux", target_os = "macos")))] { - let _ = request; + let _ = (request, metrics); return Err(HistoryError::Unsupported("SQLite Store compaction")); } #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -1689,7 +1827,8 @@ impl SqliteHandles { let live = live_tuple(&session)?; let generation = u64::try_from(live.generation) .map_err(|_| HistoryError::Integrity("Store generation"))?; - settle_leftover(&history, &store, generation)?; + metrics.generation_before = generation; + settle_leftover(&history, &store, generation, metrics)?; let op_id = request.op_id; let mut op = RunOp { @@ -1715,11 +1854,17 @@ impl SqliteHandles { request.owner_boot.as_deref(), ) .with_phase(Some("mark")); - history.gc_register(&intent)?; + if let Err(error) = history.gc_register(&intent) { + // A definite admission refusal started nothing: the run's op + // record must not outlive it as a false leftover (§9). + let _ = std::fs::remove_file(op_path(&store)); + return Err(error); + } // §8: the registration, the mark and the totals read are one // admitted sequence. A definite failure here releases the intent // before the artifact set and the op record go, so the Store is // left exactly as it was and no later call meets BusyOp. + metrics.begin_phase(CompactPhase::Mark); let admitted = (|| -> HistoryResult<(MarkResult, MarkTuple, u64, u64, CompactKeptWaste, u64)> { let mark = history.mark()?; @@ -1750,6 +1895,7 @@ impl SqliteHandles { } }; let rebuild = total > 0 && dead.saturating_mul(10) >= total; + metrics.end_phase(); if !rebuild { // §3 `none`: mark only. The intent is cleared and the op record @@ -1792,9 +1938,7 @@ impl SqliteHandles { // a live intent would refuse every later GC operation. The held // writer lane still excludes writers through the install, and a // crash leaves the op record for the next run to settle. - if let Err(error) = history.gc_release(&op_id) { - return Err(error); - } + history.gc_release(&op_id)?; drop(history); drop(storage); let mut guard = WriterGuard::new(lane.into_writer()); @@ -1836,16 +1980,19 @@ impl SqliteHandles { } stop_at(request, CompactFailpoint::AfterSeal, &op, &store)?; + metrics.begin_phase(CompactPhase::Build); op.phase = "build".to_owned(); op.write(&store)?; let capacity = request.failpoint == Some(CompactFailpoint::CapacityAtBuild); let built = match build( - &facts, - &mark, - &tuple, - &object_packs, - layout, - synchronous, + BuildInput { + facts: &facts, + mark: &mark, + tuple: &tuple, + packs: &object_packs, + layout, + synchronous, + }, capacity, request, &op, @@ -1861,19 +2008,23 @@ impl SqliteHandles { }; stop_at(request, CompactFailpoint::AfterBuild, &op, &store)?; stop_at(request, CompactFailpoint::AfterWalSwitch, &op, &store)?; + metrics.end_phase(); + metrics.begin_phase(CompactPhase::Verify); op.phase = "verify".to_owned(); if let Err(error) = op.write(&store) { abort_before_install(&store); return Err(error); } if let Err(error) = verify( - &store, - &built, - &mark, - &tuple, - layout, - acquisition_schema, + VerifyInput { + store: &store, + built: &built, + mark: &mark, + tuple: &tuple, + layout, + acquisition: acquisition_schema, + }, request, &op, ) { @@ -1884,7 +2035,9 @@ impl SqliteHandles { return Err(error); } stop_at(request, CompactFailpoint::AfterVerify, &op, &store)?; + metrics.end_phase(); + metrics.begin_phase(CompactPhase::Install); if let Some(hook) = request.before_install_hook { hook(&facts.path); } @@ -1940,9 +2093,109 @@ impl SqliteHandles { } } +/// The phase clock and §11 values one measured `compact` run observes. +struct RunMetrics { + started: Instant, + phase: Option<(CompactPhase, Instant)>, + phase_nanos: Vec<(CompactPhase, u64)>, + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + kept_waste: CompactKeptWaste, + residue_bytes: u64, + leftover_settled: bool, +} + +impl RunMetrics { + fn start() -> Self { + Self { + started: Instant::now(), + phase: None, + phase_nanos: Vec::new(), + generation_before: 0, + generation_after: 0, + dead_bytes_bound: 0, + dead_bytes_refined: 0, + kept_waste: CompactKeptWaste::default(), + residue_bytes: 0, + leftover_settled: false, + } + } + + /// Ends the current phase and starts `phase`; a phase is inclusive wall + /// time, never an exclusive CPU decomposition. + fn begin_phase(&mut self, phase: CompactPhase) { + self.end_phase(); + self.phase = Some((phase, Instant::now())); + } + + fn end_phase(&mut self) { + if let Some((phase, started)) = self.phase.take() { + self.phase_nanos.push(( + phase, + u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), + )); + } + } + + /// Records this run once, whatever its result. The outcome and refusal + /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted + /// without changing the error the caller receives. + fn finish(mut self, result: &HistoryResult) { + self.end_phase(); + let total_nanos = + u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let (outcome, refusal) = match result { + Ok(receipt) => { + self.generation_after = receipt.generation_after; + self.dead_bytes_bound = receipt.dead_bytes_bound; + self.dead_bytes_refined = receipt.dead_bytes_refined; + self.kept_waste = receipt.kept_waste.clone(); + self.residue_bytes = receipt.residue.bytes; + let outcome = match receipt.path { + CompactPath::Rebuild => CompactOutcome::Rebuild, + CompactPath::None => CompactOutcome::None, + }; + (outcome, None) + } + Err(error) => { + if !error.unknown() { + self.generation_after = self.generation_before; + } + ( + if error.unknown() { + CompactOutcome::Uncertain + } else { + CompactOutcome::Failed + }, + match error { + HistoryError::Busy => Some(CompactRefusal::Busy), + HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), + _ => None, + }, + ) + } + }; + compact_metrics::record_run(&CompactRunObservation { + outcome, + generation_before: self.generation_before, + generation_after: self.generation_after, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + kept_waste: self.kept_waste, + residue_bytes: self.residue_bytes, + phase_nanos: self.phase_nanos, + total_nanos, + leftover_settled: self.leftover_settled, + refusal, + }); + } +} + #[cfg(test)] mod tests { - use super::io_error; + use super::{journal_bound_bytes, io_error, Built}; use layerfs_history::HistoryError; /// §7 step 3 on a bounded filesystem: `ENOSPC`, `EDQUOT` and `EFBIG` are @@ -1964,4 +2217,44 @@ mod tests { let error = io_error(std::io::Error::from_raw_os_error(5)); assert!(matches!(error, HistoryError::Integrity(_)), "{error:?}"); } + + /// §11 `peakTempBytes` on the 50 GB arm (T-G66): the staged copy holds the + /// source extent, the rewrite holds that copy beside the new Store, and the + /// rollback journal is bounded by the staged page count. The receipt must + /// cover the measured 143.1 GB pre-rebuild peak and the 44.6 GB journal the + /// old `new + visited` formula dropped. + #[test] + fn peak_temp_covers_the_staged_copy_and_the_rollback_journal() { + let page_size = 4096; + // 50,862,854,144 B staged copy = the sealed Store's page count. + let staged_pages = 12_417_689; + let staged_bytes = staged_pages * page_size; + let target_bytes = 45_763_203_072; + let visited_bytes = 20_365_312; + let journal = journal_bound_bytes(staged_pages, page_size); + assert_eq!(journal, 50_962_203_848); + let built = Built { + target: std::path::PathBuf::new(), + target_bytes, + staged_bytes, + journal_bound_bytes: journal, + visited_bytes, + logical: 0, + canonical: 0, + stored: 0, + metadata: 0, + free_bytes: 0, + }; + let peak = built.peak_temp_bytes(); + assert_eq!(peak, 147_608_626_376); + assert!(peak >= target_bytes + journal + visited_bytes); + assert!( + peak >= target_bytes + 44_630_503_928 + visited_bytes, + "the bound must cover the 50 GB arm's observed 44.6 GB journal" + ); + assert!( + peak >= 143_100_000_000, + "the bound must cover the measured 143.1 GB sampled 50 GB peak" + ); + } } diff --git a/engine/crates/layerfs-persistence/src/compact_metrics.rs b/engine/crates/layerfs-persistence/src/compact_metrics.rs new file mode 100644 index 0000000000..96b42bde6c --- /dev/null +++ b/engine/crates/layerfs-persistence/src/compact_metrics.rs @@ -0,0 +1,581 @@ +//! Bounded metrics and SLO signals for SQLite (§7) and PostgreSQL (§8) +//! Store compaction. +//! +//! Every metric a real `store.compact` or `store.status` path records goes +//! through this module. Label values come from closed enums ([`CompactOutcome`], +//! [`CompactPhase`], [`CompactRefusal`], [`CompactStatusState`]), so the metric +//! cardinality is fixed: no Store name, path, operation id or artifact name is +//! ever a label. A finished run is observed once, in [`record_run`], and +//! recording never changes its result. +//! +//! Production reads the process totals through [`snapshot`]. A test or a +//! diagnostic probe can capture only its own thread with +//! [`CompactMetricsCapture`], so concurrent work on other threads does not +//! pollute the observation. + +use crate::backend::sqlite::compact::CompactKeptWaste; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; + +/// The outcome of one Store compaction call. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactOutcome { + /// A rebuild installed at `G -> G + 1`. + Rebuild, + /// The `none` path: mark only, generation unchanged. + None, + /// A definite failure that left the live Store unchanged. + Failed, + /// A failed or stopped run whose persistence outcome is unknown (§9). + Uncertain, +} + +impl CompactOutcome { + /// The bounded `outcome` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Rebuild => "rebuild", + Self::None => "none", + Self::Failed => "failed", + Self::Uncertain => "uncertain", + } + } +} + +/// One phase a run may time. +/// +/// SQLite (§7) reports `mark`, `build`, `verify` and `install`. PostgreSQL +/// (§8) reports its own Phase M (`pg_mark`), Phase A (`pg_build`), Phase B +/// (`pg_swap`) and Phase C (`pg_cleanup`). The two backends' values are +/// distinct, so a process total never blends a SQLite phase with a PostgreSQL +/// phase of a different meaning, and every value stays a closed enum string. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactPhase { + /// The §7 step 1 mark and the wholly-dead-pack decision. + Mark, + /// The staged rebuild's build. + Build, + /// The immutable verification. + Verify, + /// The install and its settle. + Install, + /// PostgreSQL §8 Phase M: the intent, the T7.2 mark and the persisted + /// `gc_dead` set. + PostgresMark, + /// PostgreSQL §8 Phase A: copy the retained payload into the scratch + /// schema. + PostgresBuild, + /// PostgreSQL §8 Phase B: swap `pack`/`object_location` at `G -> G + 1`. + PostgresSwap, + /// PostgreSQL §8 Phase C: delete the persisted dead set and settle the + /// totals, also for a resumed `G + 1` leftover. + PostgresCleanup, +} + +impl CompactPhase { + /// The bounded `phase` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Mark => "mark", + Self::Build => "build", + Self::Verify => "verify", + Self::Install => "install", + Self::PostgresMark => "pg_mark", + Self::PostgresBuild => "pg_build", + Self::PostgresSwap => "pg_swap", + Self::PostgresCleanup => "pg_cleanup", + } + } +} + +/// A refusal that stops a compaction before or during its admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactRefusal { + /// A live operation, writer or opener holds the Store. + Busy, + /// A dead or uncertain owner left an operation unresolved. + Unresolved, +} + +impl CompactRefusal { + /// The bounded `code` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Busy => "busy", + Self::Unresolved => "unresolved", + } + } +} + +/// The compaction state `store.status` reads from an unsettled op record. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactStatusState { + /// The owning process still lives. + Busy, + /// The recording owner is gone, so the outcome is unresolved. + Unresolved, +} + +impl CompactStatusState { + /// The bounded `state` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Busy => "busy", + Self::Unresolved => "unresolved", + } + } +} + +/// An SLO objective one finished run evaluates. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactSloObjective { + /// A successful settle left no artifact and zero residue bytes. + ResidueFree, + /// A rebuild advanced exactly one generation; `none` kept it. + Generation, +} + +impl CompactSloObjective { + /// The bounded `objective` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ResidueFree => "residue_free", + Self::Generation => "generation", + } + } +} + +/// One SLO check's result. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum CompactSloResult { + /// The run met the objective. + Pass, + /// The run missed the objective. + Fail, +} + +impl CompactSloResult { + /// The bounded `result` label value. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Fail => "fail", + } + } +} + +/// A duration aggregate: how many observations, their sum and their maximum, +/// all in nanoseconds. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct DurationStat { + /// Observations counted. + pub count: u64, + /// Saturating sum of the observations. + pub sum_ns: u64, + /// The longest observation. + pub max_ns: u64, +} + +impl DurationStat { + fn observe(&mut self, nanos: u64) { + self.count = self.count.saturating_add(1); + self.sum_ns = self.sum_ns.saturating_add(nanos); + self.max_ns = self.max_ns.max(nanos); + } +} + +/// Everything one finished compaction run reports to the metrics. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompactRunObservation { + /// The run's outcome. + pub outcome: CompactOutcome, + /// The generation the mark ran at. + pub generation_before: u64, + /// The generation the run left; equal on `none` and on a failure. + pub generation_after: u64, + /// §11 `deadBytesBound`. + pub dead_bytes_bound: u64, + /// §11 `deadBytesRefined`. + pub dead_bytes_refined: u64, + /// §11 `keptWaste`, whose delta-base references are not labels. + pub kept_waste: CompactKeptWaste, + /// §11 `residue.bytes`. + pub residue_bytes: u64, + /// The completed phases and their inclusive durations. + pub phase_nanos: Vec<(CompactPhase, u64)>, + /// The run's whole duration. + pub total_nanos: u64, + /// Whether this run settled an earlier run's leftover record. + pub leftover_settled: bool, + /// The admission refusal, when the run was refused. + pub refusal: Option, +} + +/// One rendered metric sample: a stable name, bounded label pairs, a value. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MetricSample { + /// The metric name, for example `compact.operations`. + pub name: &'static str, + /// The bounded label pairs, in sorted key order. + pub labels: Vec<(&'static str, &'static str)>, + /// The sample's counter or gauge value. + pub value: u64, +} + +impl MetricSample { + /// The value of label `key`, when the sample carries it. + #[must_use] + pub fn label(&self, key: &str) -> Option<&'static str> { + self.labels + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| *value) + } +} + +/// The last successful run's §11 gauges, the values a report names. +#[derive(Clone, Debug, Eq, PartialEq)] +struct LastRun { + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + kept_waste: CompactKeptWaste, + residue_bytes: u64, +} + +/// One process's compaction metric totals. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CompactMetrics { + runs: u64, + outcomes: BTreeMap<&'static str, u64>, + refusals: BTreeMap<&'static str, u64>, + settles: BTreeMap<&'static str, u64>, + retries: BTreeMap<&'static str, u64>, + statuses: BTreeMap<&'static str, u64>, + slo: BTreeMap<(&'static str, &'static str), u64>, + phases: BTreeMap<&'static str, DurationStat>, + duration: DurationStat, + last: Option, +} + +impl CompactMetrics { + /// Adds one finished run. + fn observe(&mut self, run: &CompactRunObservation) { + self.runs = self.runs.saturating_add(1); + increment(&mut self.outcomes, run.outcome.as_str()); + if let Some(refusal) = run.refusal { + increment(&mut self.refusals, refusal.as_str()); + } + if run.leftover_settled { + increment(&mut self.settles, "leftover"); + increment(&mut self.retries, "leftover"); + } + for (phase, nanos) in &run.phase_nanos { + self.phases + .entry(phase.as_str()) + .or_default() + .observe(*nanos); + } + self.duration.observe(run.total_nanos); + if !matches!(run.outcome, CompactOutcome::Rebuild | CompactOutcome::None) { + return; + } + increment(&mut self.settles, "run"); + let generation = match run.outcome { + CompactOutcome::Rebuild => { + run.generation_after == run.generation_before.saturating_add(1) + } + CompactOutcome::None => run.generation_after == run.generation_before, + CompactOutcome::Failed | CompactOutcome::Uncertain => false, + }; + increment_pair( + &mut self.slo, + ( + CompactSloObjective::Generation.as_str(), + if generation { + CompactSloResult::Pass.as_str() + } else { + CompactSloResult::Fail.as_str() + }, + ), + ); + increment_pair( + &mut self.slo, + ( + CompactSloObjective::ResidueFree.as_str(), + if run.residue_bytes == 0 { + CompactSloResult::Pass.as_str() + } else { + CompactSloResult::Fail.as_str() + }, + ), + ); + self.last = Some(LastRun { + generation_before: run.generation_before, + generation_after: run.generation_after, + dead_bytes_bound: run.dead_bytes_bound, + dead_bytes_refined: run.dead_bytes_refined, + kept_waste: run.kept_waste.clone(), + residue_bytes: run.residue_bytes, + }); + } + + /// Adds one refusal that stopped a call before a run could be observed. + fn observe_refusal(&mut self, refusal: CompactRefusal) { + increment(&mut self.refusals, refusal.as_str()); + } + + /// Adds one `store.status` read of an unsettled compaction record. + fn observe_status(&mut self, state: CompactStatusState) { + increment(&mut self.statuses, state.as_str()); + } + + /// Every sample this process or capture holds, in stable order. + /// + /// Counters are cumulative since the process started (or the capture + /// started); the gauges are the last successful run's report values. + #[must_use] + pub fn samples(&self) -> Vec { + let mut samples = Vec::new(); + samples.push(sample("compact.runs", &[], self.runs)); + for (outcome, value) in &self.outcomes { + samples.push(sample( + "compact.operations", + &[("outcome", *outcome)], + *value, + )); + } + for (code, value) in &self.refusals { + samples.push(sample("compact.refusals", &[("code", *code)], *value)); + } + for (kind, value) in &self.settles { + samples.push(sample("compact.settles", &[("kind", *kind)], *value)); + } + for (reason, value) in &self.retries { + samples.push(sample("compact.retries", &[("reason", *reason)], *value)); + } + for (state, value) in &self.statuses { + samples.push(sample("compact.status", &[("state", *state)], *value)); + } + for ((objective, result), value) in &self.slo { + samples.push(sample( + "compact.slo", + &[("objective", *objective), ("result", *result)], + *value, + )); + } + for (phase, stat) in &self.phases { + for (statistic, value) in [ + ("count", stat.count), + ("sum", stat.sum_ns), + ("max", stat.max_ns), + ] { + samples.push(sample( + "compact.phase.duration_ns", + &[("phase", *phase), ("statistic", statistic)], + value, + )); + } + } + for (statistic, value) in [ + ("count", self.duration.count), + ("sum", self.duration.sum_ns), + ("max", self.duration.max_ns), + ] { + samples.push(sample( + "compact.duration_ns", + &[("statistic", statistic)], + value, + )); + } + if let Some(last) = &self.last { + samples.push(sample( + "compact.dead_bytes", + &[("kind", "bound")], + last.dead_bytes_bound, + )); + samples.push(sample( + "compact.dead_bytes", + &[("kind", "refined")], + last.dead_bytes_refined, + )); + samples.push(sample( + "compact.kept_waste", + &[("part", "dead_in_retained_packs")], + last.kept_waste.dead_in_retained_packs_bytes, + )); + samples.push(sample( + "compact.kept_waste", + &[("part", "delta_bases")], + last.kept_waste.delta_bases_bytes, + )); + samples.push(sample( + "compact.kept_waste", + &[("part", "inode_blocks")], + last.kept_waste.inode_blocks_bytes, + )); + samples.push(sample( + "compact.kept_waste_bytes", + &[], + last.kept_waste + .dead_in_retained_packs_bytes + .saturating_add(last.kept_waste.delta_bases_bytes) + .saturating_add(last.kept_waste.inode_blocks_bytes), + )); + samples.push(sample("compact.residue_bytes", &[], last.residue_bytes)); + samples.push(sample( + "compact.generation", + &[("which", "before")], + last.generation_before, + )); + samples.push(sample( + "compact.generation", + &[("which", "after")], + last.generation_after, + )); + } + samples + } +} + +fn increment(map: &mut BTreeMap<&'static str, u64>, key: &'static str) { + let value = map.entry(key).or_insert(0); + *value = value.saturating_add(1); +} + +fn increment_pair( + map: &mut BTreeMap<(&'static str, &'static str), u64>, + key: (&'static str, &'static str), +) { + let value = map.entry(key).or_insert(0); + *value = value.saturating_add(1); +} + +fn sample(name: &'static str, labels: &[(&'static str, &'static str)], value: u64) -> MetricSample { + MetricSample { + name, + labels: labels.to_vec(), + value, + } +} + +thread_local! { + /// This thread's armed capture, if any. + static CAPTURE: RefCell> = const { RefCell::new(None) }; +} + +fn registry() -> &'static Mutex { + static METRICS: OnceLock> = OnceLock::new(); + METRICS.get_or_init(|| Mutex::new(CompactMetrics::default())) +} + +/// Records one finished compaction run in the process totals and, when this +/// thread has an armed [`CompactMetricsCapture`], in that capture too. +pub fn record_run(run: &CompactRunObservation) { + registry() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .observe(run); + CAPTURE.with_borrow_mut(|capture| { + if let Some(metrics) = capture.as_mut() { + metrics.observe(run); + } + }); +} + +/// Records one admission refusal that stopped a call before any run. +pub fn record_refusal(refusal: CompactRefusal) { + registry() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .observe_refusal(refusal); + CAPTURE.with_borrow_mut(|capture| { + if let Some(metrics) = capture.as_mut() { + metrics.observe_refusal(refusal); + } + }); +} + +/// Records one `store.status` read of an unsettled compaction record. +pub fn record_status(state: CompactStatusState) { + registry() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .observe_status(state); + CAPTURE.with_borrow_mut(|capture| { + if let Some(metrics) = capture.as_mut() { + metrics.observe_status(state); + } + }); +} + +/// The process's cumulative compaction metrics. +#[must_use] +pub fn snapshot() -> CompactMetrics { + registry() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .clone() +} + +/// Clears the process's cumulative compaction metrics. +pub fn reset() { + *registry() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = CompactMetrics::default(); +} + +/// Captures the metrics this thread records, leaving the process totals alone. +/// +/// Build one with [`Self::start`], run the compaction or status call on the +/// same thread, and read [`Self::finish`]. A capture nests: starting another +/// restores the outer one when it finishes, and a dropped capture restores it +/// too. +pub struct CompactMetricsCapture { + previous: Option, + finished: bool, +} + +impl CompactMetricsCapture { + /// Arms a capture on this thread, replacing (and saving) an outer one. + #[must_use] + pub fn start() -> Self { + let previous = + CAPTURE.with_borrow_mut(|capture| capture.replace(CompactMetrics::default())); + Self { + previous, + finished: false, + } + } + + /// The metrics this thread has recorded since the capture started. + #[must_use] + pub fn snapshot(&self) -> CompactMetrics { + CAPTURE.with_borrow(Clone::clone).unwrap_or_default() + } + + /// Ends the capture, restoring the outer one, and returns its metrics. + #[must_use] + pub fn finish(mut self) -> CompactMetrics { + let metrics = self.snapshot(); + self.finished = true; + CAPTURE.with_borrow_mut(|capture| *capture = self.previous.take()); + metrics + } +} + +impl Drop for CompactMetricsCapture { + fn drop(&mut self) { + if !self.finished { + CAPTURE.with_borrow_mut(|capture| *capture = self.previous.take()); + } + } +} diff --git a/engine/crates/layerfs-persistence/src/history/admission.rs b/engine/crates/layerfs-persistence/src/history/admission.rs index f45a48eef0..53e0996e97 100644 --- a/engine/crates/layerfs-persistence/src/history/admission.rs +++ b/engine/crates/layerfs-persistence/src/history/admission.rs @@ -404,19 +404,30 @@ fn get_in(tx: &Transaction<'_>, op_id: &[u8; 16]) -> HistoryResult)>, + shared: bool, +} + #[cfg(feature = "postgresql")] fn keys_for_release( tx: &Transaction<'_>, catalog: CatalogId, intent: &GcIntent, -) -> HistoryResult<(Vec<(u8, Vec)>, bool)> { +) -> HistoryResult { let mut intent = intent.clone(); if intent.scope_kind == GcScopeKind::Branch && intent.project_id.is_none() { intent.project_id = one(tx, BRANCH_STACK, [intent.scope_id.as_slice()], |row| { cell::>(row, 0) })?; } - Ok((advisory_keys(catalog, &intent), intent.kind.shared())) + Ok(ReleaseKeys { + keys: advisory_keys(catalog, &intent), + shared: intent.kind.shared(), + }) } /// Registers one shared (`mount`, `init`, `merge`) or exclusive @@ -453,7 +464,7 @@ pub(crate) fn release(provider: &HistoryProvider, op_id: &[u8; 16]) -> HistoryRe release_in(tx, op_id)?; Ok(Some(keys_for_release(tx, provider.catalog_id, &intent)?)) })?; - if let Some((keys, shared)) = found { + if let Some(ReleaseKeys { keys, shared }) = found { let _ = session_unlock(session, provider.catalog_id, &keys, shared); } Ok(()) diff --git a/engine/crates/layerfs-persistence/src/history/catalog.rs b/engine/crates/layerfs-persistence/src/history/catalog.rs index f4cb1cf748..64d01cc814 100644 --- a/engine/crates/layerfs-persistence/src/history/catalog.rs +++ b/engine/crates/layerfs-persistence/src/history/catalog.rs @@ -59,10 +59,23 @@ impl HistoryCatalog for HistoryProvider { self.read_statement(|tx| layerstack::layer_stack_named(tx, name)) } + // The collision statement reads the retired/deleted holder on purpose. + fn layer_stack_name_taken( + &self, + name: &HistoryName, + ) -> HistoryResult> { + self.read(|tx| layerstack::name_taken(tx, name)) + } + fn branch(&self, id: BranchId) -> HistoryResult> { self.read(|tx| branch::branch(tx, id)) } + // One statement, no visibility filter: rendering keeps a retired Branch. + fn branch_any(&self, id: BranchId) -> HistoryResult> { + self.read_statement(|tx| branch::branch_any(tx, id)) + } + fn branch_named( &self, stack: LayerStackId, diff --git a/engine/crates/layerfs-persistence/src/history/mod.rs b/engine/crates/layerfs-persistence/src/history/mod.rs index 219d7b16e8..fec388af57 100644 --- a/engine/crates/layerfs-persistence/src/history/mod.rs +++ b/engine/crates/layerfs-persistence/src/history/mod.rs @@ -23,7 +23,9 @@ mod rows; mod transaction; pub use kept::{KeptHistory, KeptNode, KeptNodeId}; pub(crate) use lineage_backfill::backfill_lineage; -pub use objects::{GcTuple, KeptObjects, KeptWaste, MarkResult}; +#[cfg(feature = "postgresql")] +pub use objects::GcTuple; +pub use objects::{KeptObjects, KeptWaste, MarkResult}; pub use provider::HistoryProvider; pub(crate) use rows::{ branch_row as diagnostic_branch, commit_graph_row as diagnostic_commit_graph, diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index fe158ec820..f0ff344f80 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -3,6 +3,7 @@ #![deny(unsafe_code)] #![deny(missing_docs)] mod backend; +pub mod compact_metrics; pub mod diagnostic; mod storage; mod store; @@ -13,7 +14,8 @@ mod report; pub use backend::postgresql::access::{access as postgresql_access, PostgresAccess}; #[cfg(feature = "postgresql")] pub use backend::postgresql::status::{ - status as postgresql_status, PostgresStatus, PostgresStatusError, + compact_status as postgresql_compact_status, status as postgresql_status, + PostgresCompactOpStatus, PostgresStatus, PostgresStatusError, }; pub use backend::sqlite::acquisition::AcquisitionProvider; pub use backend::sqlite::compact::{ @@ -22,7 +24,8 @@ pub use backend::sqlite::compact::{ }; pub use backend::sqlite::heads::{heads_digest, read_heads}; pub use report::{ - CompactGeneration, CompactReport, CompactReportSource, CompactStatus, SQLITE_COMPACT_PHASES, + CompactGeneration, CompactReport, CompactReportSource, CompactStatus, POSTGRES_COMPACT_PHASES, + SQLITE_COMPACT_PHASES, }; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use backend::sqlite::lifecycle::headroom::STORE_HEADROOM_BYTES; diff --git a/engine/crates/layerfs-persistence/src/report.rs b/engine/crates/layerfs-persistence/src/report.rs index 0c6a1fa80e..becb1eef5c 100644 --- a/engine/crates/layerfs-persistence/src/report.rs +++ b/engine/crates/layerfs-persistence/src/report.rs @@ -13,6 +13,11 @@ use crate::backend::sqlite::compact::{ /// leaves its record at the phase it reached. pub const SQLITE_COMPACT_PHASES: [&str; 4] = ["mark", "build", "verify", "install"]; +/// The PostgreSQL §8 compaction phases, in order: Phase M mark, Phase A +/// build, Phase B swap and Phase C cleanup. These are the values a +/// PostgreSQL `gc_op` row records, distinct from the SQLite §7 phases. +pub const POSTGRES_COMPACT_PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + /// The generation a compaction started at and, when it installed a rebuild, /// the one it left (`none` keeps both equal). #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/engine/crates/layerfs-persistence/tests/compact_metrics.rs b/engine/crates/layerfs-persistence/tests/compact_metrics.rs new file mode 100644 index 0000000000..585a5f8088 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/compact_metrics.rs @@ -0,0 +1,371 @@ +//! T10.3: the bounded metrics a real `store.compact` path records. +//! +//! The three observations are a successful rebuild, the `none` path and the +//! leftover a stopped run leaves for the next call. The capture is +//! thread-local, so the other tests in this binary cannot pollute it. +#![cfg(any(target_os = "macos", target_os = "linux"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::HistoryError; +use layerfs_persistence::compact_metrics::CompactMetricsCapture; +use layerfs_persistence::{ + CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_storage::{Storage, StoragePolicy}; +use std::path::{Path, PathBuf}; +use support::metrics::{assert_bounded, metric, value}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-metrics-test-binding"; + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-metrics"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &layerfs_history::HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn compact_path(store: &Path) -> PathBuf { + let mut name = store.as_os_str().to_os_string(); + name.push(".compact"); + PathBuf::from(name) +} + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + save.accept(object.clone()).unwrap(); + save.finish().unwrap(); + object.id() +} + +#[test] +fn a_successful_rebuild_emits_phases_outcome_and_report_gauges() { + let (_temp, handles, path) = create_store(); + let _dead = whole_file(&handles, b"a pack no retained root reaches"); + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(40)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::Rebuild, "the dead pack rebuilds"); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "rebuild")]), + 1 + ); + for phase in ["mark", "build", "verify", "install"] { + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "{phase} was timed once" + ); + assert!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "sum")] + ) > 0, + "{phase} took time" + ); + assert!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "max")] + ) > 0, + "{phase} has a duration" + ); + } + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert!( + value(&metrics, "compact.duration_ns", &[("statistic", "sum")]) > 0, + "the run took time" + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + let kept = receipt + .kept_waste + .dead_in_retained_packs_bytes + .saturating_add(receipt.kept_waste.delta_bases_bytes) + .saturating_add(receipt.kept_waste.inode_blocks_bytes); + assert_eq!(value(&metrics, "compact.kept_waste_bytes", &[]), kept); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + receipt.residue.bytes, + "a successful settle leaves no residue" + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + assert_eq!(value(&metrics, "compact.settles", &[("kind", "run")]), 1); + assert_eq!( + metric(&metrics, "compact.retries", &[("reason", "leftover")]), + None + ); + + let reopened = reopen(&path); + clean(&reopened); +} + +#[test] +fn the_none_path_emits_only_the_mark_phase_and_keeps_the_generation() { + let (_temp, handles, path) = create_store(); + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(41)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::None); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "none")]), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", "mark"), ("statistic", "count")] + ), + 1 + ); + for phase in ["build", "verify", "install"] { + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "the none path never enters {phase}" + ); + } + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + 0, + "none leaves no artifact" + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after, + "none keeps the generation" + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + + let reopened = reopen(&path); + clean(&reopened); +} + +#[test] +fn a_stopped_run_then_the_leftover_settle_emits_failure_retry_and_settle_metrics() { + let (_temp, handles, path) = create_store(); + let _dead = whole_file(&handles, b"a pack no retained root reaches"); + + let capture = CompactMetricsCapture::start(); + let error = handles + .compact(&request_at(42, CompactFailpoint::AfterBuild)) + .unwrap_err(); + let stopped = capture.finish(); + assert!(matches!(error, HistoryError::UnknownOutcome), "{error:?}"); + assert_bounded(&stopped); + assert_eq!(value(&stopped, "compact.runs", &[]), 1); + assert_eq!( + value(&stopped, "compact.operations", &[("outcome", "uncertain")]), + 1, + "a stop point leaves an unknown outcome" + ); + for phase in ["mark", "build"] { + assert_eq!( + value( + &stopped, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "the stopped run reached {phase}" + ); + } + assert_eq!( + metric(&stopped, "compact.generation", &[("which", "after")]), + None, + "an unknown outcome publishes no report gauge" + ); + assert_eq!( + metric(&stopped, "compact.settles", &[("kind", "leftover")]), + None, + "the stopped run settles no leftover" + ); + + let capture = CompactMetricsCapture::start(); + let reopened = reopen(&path); + let receipt = reopened.compact(&request(43)).unwrap(); + let settled = capture.finish(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_bounded(&settled); + assert_eq!( + value(&settled, "compact.operations", &[("outcome", "rebuild")]), + 1 + ); + assert_eq!( + value(&settled, "compact.retries", &[("reason", "leftover")]), + 1 + ); + assert_eq!( + value(&settled, "compact.settles", &[("kind", "leftover")]), + 1 + ); + assert_eq!(value(&settled, "compact.settles", &[("kind", "run")]), 1); + assert_eq!(value(&settled, "compact.residue_bytes", &[]), 0); + + let final_handles = reopen(&path); + clean(&final_handles); +} + +#[test] +fn a_definite_failure_emits_a_failed_outcome_without_report_gauges() { + let (_temp, handles, path) = create_store(); + // A stray artifact no record names makes settle_leftover refuse, before + // the mark: a definite failure, not an unknown outcome. + std::fs::write(compact_path(&path), b"stray").unwrap(); + let capture = CompactMetricsCapture::start(); + let error = handles.compact(&request(44)).unwrap_err(); + let metrics = capture.finish(); + assert!(matches!(error, HistoryError::Exists(_)), "{error:?}"); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "failed")]), + 1 + ); + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert!(value(&metrics, "compact.duration_ns", &[("statistic", "sum")]) > 0); + assert_eq!( + metric(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + None + ); + assert_eq!( + metric( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + None + ); +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} diff --git a/engine/crates/layerfs-persistence/tests/concurrent_first_installs.rs b/engine/crates/layerfs-persistence/tests/concurrent_first_installs.rs index 121e06ecbb..5c01efb8da 100644 --- a/engine/crates/layerfs-persistence/tests/concurrent_first_installs.rs +++ b/engine/crates/layerfs-persistence/tests/concurrent_first_installs.rs @@ -1,6 +1,13 @@ //! F-036: two writers, each with its own opened Store handle as a daemon has, //! save the same new content at the same time. Both Saves complete and every //! object reads back, whichever writer won each first-wins registration. +//! +//! Delta selection can propose a copy the other writer just published, so the +//! candidate's chain can name the very object this Save is storing: the two +//! writers can each pick the other's copy as a base, and storing both would +//! close a delta cycle (`Integrity("dependency cycle")` at the first read). +//! Selection refuses that edge (`DepthCache::cost_avoiding`) and stores the +//! object FULL, so the graph stays acyclic whichever registrations win. #![cfg(target_os = "linux")] mod support; use layerfs_content::{AdvisoryPredecessors, FinalizedObject, ObjectRole}; diff --git a/engine/crates/layerfs-persistence/tests/diagnostic_history.rs b/engine/crates/layerfs-persistence/tests/diagnostic_history.rs index 46cc3021e7..dbd394fa91 100644 --- a/engine/crates/layerfs-persistence/tests/diagnostic_history.rs +++ b/engine/crates/layerfs-persistence/tests/diagnostic_history.rs @@ -191,9 +191,9 @@ fn a_retained_commit_branch_reference_must_exist_in_its_stack() { let f = graph(); clean(&f.audit(true)); f.raw() - .execute_batch(&format!( - "PRAGMA foreign_keys=OFF; UPDATE \"commit\" SET branch_id=CAST(X'11'||zeroblob(16) AS BLOB)" - )) + .execute_batch( + "PRAGMA foreign_keys=OFF; UPDATE \"commit\" SET branch_id=CAST(X'11'||zeroblob(16) AS BLOB)", + ) .unwrap(); let out = f.audit(true); failure(&out, "store.history.branches"); diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json new file mode 100644 index 0000000000..6848f4ca47 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json @@ -0,0 +1,89 @@ +{ + "schema": "t12.3a-sqlite-compact-capacity/1", + "actualBytes": 9986795475, + "actualGib": "9.987", + "packCount": 76870, + "packLengthMin": 561, + "packLengthMax": 131108, + "objectBytes": 131071, + "groups": 701, + "groupSize": 99, + "liveObjects": 69399, + "initialDeadObjects": 6770, + "deadFractionPermille": 88, + "fixtureBuildSeconds": 150.0, + "noneDeadBytes": 887601160, + "noneTotalBytes": 9986795475, + "rebuildDeadBytes": 1011104896, + "rebuildTotalBytes": 10110299211, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)", + "cpus": 88, + "memBytes": 135033307136, + "none": { + "path": "none", + "runs": 3, + "micros": [148519470, 149703782, 149318362], + "medianMicros": 149318362, + "minMicros": 148519470, + "maxMicros": 149703782, + "deadBytesBound": [887601160, 887601160, 887601160], + "deadBytesRefined": [887601160, 887601160, 887601160], + "physicalShrink": [0, 0, 0], + "peakTempBytes": [0, 0, 0], + "oldFileBytes": [10034614272, 10034696192, 10034696192], + "newFileBytes": [10034696192, 10034696192, 10034696192], + "generationBefore": [7, 7, 7], + "generationAfter": [7, 7, 7], + "packCountBefore": [76870, 76870, 76870], + "packCountAfter": [76870, 76870, 76870], + "addedDeadObjects": [0, 0, 0], + "phasesNanos": { + "mark": [148512076807, 149699478879, 149314062028], + "build": [0, 0, 0], + "verify": [0, 0, 0], + "install": [0, 0, 0] + } + }, + "rebuild": { + "path": "rebuild", + "runs": 3, + "micros": [246370906, 255548454, 243305751], + "medianMicros": 246370906, + "minMicros": 243305751, + "maxMicros": 255548454, + "deadBytesBound": [1011104896, 1011104896, 1011104896], + "deadBytesRefined": [1011104896, 1011104896, 1011104896], + "physicalShrink": [1018978304, 1016524800, 1016504320], + "peakTempBytes": [9143750656, 9143750656, 9143750656], + "oldFileBytes": [10158751744, 10156294144, 10156273664], + "newFileBytes": [9139773440, 9139773440, 9139773440], + "generationBefore": [7, 8, 9], + "generationAfter": [8, 9, 10], + "packCountBefore": [77812, 77812, 77812], + "packCountAfter": [70100, 70100, 70100], + "addedDeadObjects": [942, 7712, 7712], + "phasesNanos": { + "mark": [149758815060, 148589673893, 149088791304], + "build": [94316711529, 104423746881, 91945016315], + "verify": [311503029, 431132794, 286831239], + "install": [1913022088, 2035474103, 1915532558] + } + }, + "durableRebuild": { + "path": "rebuild", + "runs": 1, + "micros": [284078209], + "deadBytesBound": [1011104896], + "physicalShrink": [1016483840], + "peakTempBytes": [9143750656], + "addedDeadObjects": [7712], + "phasesNanos": { + "mark": [149687630706], + "build": [132235308703], + "verify": [267889954], + "install": [1816879571] + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json new file mode 100644 index 0000000000..b2b13e713f --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json @@ -0,0 +1,74 @@ +{ + "schema": "t12.3a-sqlite-compact-capacity/1", + "actualBytes": 49998990030, + "actualGib": "49.999", + "packCount": 384850, + "packLengthMin": 561, + "packLengthMax": 131108, + "objectBytes": 131071, + "groups": 3510, + "groupSize": 99, + "liveObjects": 347490, + "initialDeadObjects": 33850, + "deadFractionPermille": 88, + "fixtureBuildSeconds": 761.2, + "noneDeadBytes": 4438005800, + "noneTotalBytes": 49998990030, + "rebuildDeadBytes": 5062342096, + "rebuildTotalBytes": 50623326326, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)", + "cpus": 88, + "memBytes": 135033307136, + "none": { + "path": "none", + "runs": 1, + "micros": [1633585041], + "medianMicros": 1633585041, + "minMicros": 1633585041, + "maxMicros": 1633585041, + "deadBytesBound": [4438005800], + "deadBytesRefined": [4438005800], + "physicalShrink": [0], + "peakTempBytes": [0], + "oldFileBytes": [50235379712], + "newFileBytes": [50235826176], + "generationBefore": [7], + "generationAfter": [7], + "packCountBefore": [384850], + "packCountAfter": [384850], + "addedDeadObjects": [0], + "phasesNanos": { + "mark": [1633578666486], + "build": [0], + "verify": [0], + "install": [0] + } + }, + "rebuild": { + "path": "rebuild", + "runs": 1, + "micros": [5131834971], + "medianMicros": 5131834971, + "minMicros": 5131834971, + "maxMicros": 5131834971, + "deadBytesBound": [5062342096], + "deadBytesRefined": [5062342096], + "physicalShrink": [5099651072], + "peakTempBytes": [45783568384], + "oldFileBytes": [50862854144], + "newFileBytes": [45763203072], + "generationBefore": [7], + "generationAfter": [8], + "packCountBefore": [389612], + "packCountAfter": [351000], + "addedDeadObjects": [4762], + "phasesNanos": { + "mark": [784250627182], + "build": [4254371054960], + "verify": [85556998080], + "install": [7311035596] + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json index c67cf099d2..1167973121 100644 --- a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json @@ -377,7 +377,7 @@ }, "logicalDiscarded": 160, "path": "rebuild", - "peakTempBytes": 389120, + "peakTempBytes": 1537112, "physicalShrink": 221184, "residue": { "artifacts": [], diff --git a/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json new file mode 100644 index 0000000000..12a2c3be9c --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json @@ -0,0 +1,16 @@ +{ + "schema": "t12.3a-sqlite-compact-measure/1", + "runs": 3, + "micros": [26738, 28201, 28681], + "medianMicros": 28201, + "minMicros": 26738, + "maxMicros": 28681, + "deadBytesBound": 1100, + "physicalShrink": 20480, + "peakTempBytes": 758240, + "liveRoots": 20, + "deadPacks": 20, + "profile": "SqlitePersistenceProfile::Disposable (WAL/OFF)", + "platform": "linux-x86_64", + "rustc": "rustc 1.85.1 (4eb161250 2025-03-15)" +} diff --git a/engine/crates/layerfs-persistence/tests/gc_backfill.rs b/engine/crates/layerfs-persistence/tests/gc_backfill.rs index aa65bd1518..eb30210ae7 100644 --- a/engine/crates/layerfs-persistence/tests/gc_backfill.rs +++ b/engine/crates/layerfs-persistence/tests/gc_backfill.rs @@ -314,7 +314,7 @@ fn commits_layers_and_epochs_are_backfilled_in_deterministic_order() { // One epoch row per project, both zero. let rows = epochs(&sql); assert_eq!(rows.len(), 2); - let mut expected = vec![stack_one, stack_two]; + let mut expected = [stack_one, stack_two]; expected.sort_by(|a, b| a.as_slice().cmp(b.as_slice())); for (row, expected) in rows.iter().zip(expected.iter()) { assert_eq!(row.0, expected.as_slice()); diff --git a/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs b/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs new file mode 100644 index 0000000000..4b222d5067 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/pg_doctor/conninfo.rs @@ -0,0 +1,242 @@ +//! Connection-string rewriting shared by the diagnostic and doctor suites. + +/// Add one libpq connection option to a connection string. +/// +/// A URL (`…://…`) gets a query parameter whose value is percent-encoded. The +/// libpq keyword form that the docs and CI use gets `name='value'`, because +/// `?`/`&` appended to it is parsed as part of the preceding value and the +/// server refuses the connection. +pub fn with_option(url: &str, name: &str, value: &str) -> String { + if url.contains("://") { + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{name}={}", percent_encode(value)) + } else { + format!("{url} {name}='{value}'") + } +} + +/// Put an unresolvable host in front of the real fixture host. +/// +/// The diagnostic driver tries every configured host in order and treats a +/// failed lookup as "try the next host", so `missing.invalid` exercises the +/// failover path. Rewriting only the URL form would leave the keyword form +/// unchanged and the assertion would silently stop exercising failover. +pub fn with_failing_first_host(conninfo: &str) -> String { + const HOST: &str = "127.0.0.1"; + const MISSING: &str = "missing.invalid"; + if conninfo.contains("://") { + return conninfo.replacen(HOST, &format!("{MISSING},{HOST}"), 1); + } + for (from, to) in [ + (format!("host={HOST}"), format!("host={MISSING},{HOST}")), + (format!("host='{HOST}'"), format!("host='{MISSING},{HOST}'")), + ] { + if conninfo.contains(&from) { + return conninfo.replacen(&from, &to, 1); + } + } + conninfo.to_owned() +} + +/// Rewrite a connection string's endpoint to `host:port`. +/// +/// The physical doctor's byte-counting proxy forwards to the fixture, so the +/// selection it opens must reach the proxy's port in whichever connection +/// string form the environment supplied. A URL keeps its user information, +/// path and query and has its authority's host and port replaced; the libpq +/// keyword form has its `host=` and `port=` values replaced (and added when +/// absent), leaving every other keyword — quoted values with spaces included — +/// untouched. +/// +/// Rewriting only `:{port}/` would match the URL form alone: the keyword form +/// would keep connecting straight to the fixture, and the proxy's byte count +/// would silently stop measuring the wire. +pub fn with_host(conninfo: &str, host: &str, port: u16) -> String { + if conninfo.contains("://") { + url_with_host(conninfo, host, port) + } else { + keyword_with_host(conninfo, host, port) + } +} + +/// [`with_host`] for the physical doctor's loopback proxy, whose listener and +/// forwarder are both `127.0.0.1`. +pub fn with_local_proxy(conninfo: &str, proxy_port: u16) -> String { + with_host(conninfo, "127.0.0.1", proxy_port) +} + +fn url_with_host(url: &str, host: &str, port: u16) -> String { + let Some(scheme_end) = url.find("://").map(|at| at + 3) else { + return url.to_owned(); + }; + let (scheme, rest) = url.split_at(scheme_end); + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + let userinfo_end = authority.rfind('@').map(|at| at + 1).unwrap_or(0); + let userinfo = &authority[..userinfo_end]; + format!("{scheme}{userinfo}{host}:{port}{tail}") +} + +fn keyword_with_host(conninfo: &str, host: &str, port: u16) -> String { + let bytes = conninfo.as_bytes(); + let mut out = String::with_capacity(conninfo.len() + host.len() + 16); + let mut seen_host = false; + let mut seen_port = false; + let mut at = 0; + while at < bytes.len() { + let space_start = at; + while at < bytes.len() && bytes[at].is_ascii_whitespace() { + at += 1; + } + out.push_str(&conninfo[space_start..at]); + if at == bytes.len() { + break; + } + let key_start = at; + while at < bytes.len() && (bytes[at] == b'_' || bytes[at].is_ascii_alphanumeric()) { + at += 1; + } + let key_end = at; + if key_end == key_start || bytes.get(key_end) != Some(&b'=') { + // Not `keyword=value`: keep the tail exactly as it came in. + out.push_str(&conninfo[key_start..]); + break; + } + at += 1; + if bytes.get(at) == Some(&b'\'') { + at += 1; + while at < bytes.len() { + match bytes[at] { + b'\\' => at = (at + 2).min(bytes.len()), + b'\'' if bytes.get(at + 1) == Some(&b'\'') => at += 2, + b'\'' => { + at += 1; + break; + } + _ => at += 1, + } + } + } else { + while at < bytes.len() && !bytes[at].is_ascii_whitespace() { + at += 1; + } + } + match &conninfo[key_start..key_end] { + "host" if !seen_host => { + out.push_str("host="); + out.push_str(host); + seen_host = true; + } + "port" if !seen_port => { + out.push_str("port="); + out.push_str(&port.to_string()); + seen_port = true; + } + _ => out.push_str(&conninfo[key_start..at]), + } + } + if !seen_host { + out.push_str(&format!(" host={host}")); + } + if !seen_port { + out.push_str(&format!(" port={port}")); + } + out +} + +fn percent_encode(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for &byte in value.as_bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +#[cfg(test)] +mod tests { + use super::{with_failing_first_host, with_host, with_local_proxy, with_option}; + + #[test] + fn url_form_rewrites_options_and_endpoints() { + assert_eq!( + with_option("postgresql://u@h/db", "options", "-c role=r"), + "postgresql://u@h/db?options=-c%20role%3Dr" + ); + assert_eq!( + with_option( + "postgresql://u@h/db?sslmode=disable", + "options", + "-c role=r" + ), + "postgresql://u@h/db?sslmode=disable&options=-c%20role%3Dr" + ); + assert_eq!( + with_host( + "postgresql://u:p@127.0.0.1:5432/db?sslmode=disable", + "127.0.0.1", + 9000 + ), + "postgresql://u:p@127.0.0.1:9000/db?sslmode=disable" + ); + assert_eq!( + with_host("postgres://127.0.0.1/db", "127.0.0.1", 9000), + "postgres://127.0.0.1:9000/db" + ); + assert_eq!( + with_host("postgresql://u@127.0.0.1:5432", "127.0.0.1", 9000), + "postgresql://u@127.0.0.1:9000" + ); + assert_eq!( + with_local_proxy("postgresql://u@127.0.0.1:5432/db", 9000), + "postgresql://u@127.0.0.1:9000/db" + ); + } + + #[test] + fn keyword_form_rewrites_options_and_endpoints() { + assert_eq!( + with_option("host=h port=1 sslmode=disable", "options", "-c role=r"), + "host=h port=1 sslmode=disable options='-c role=r'" + ); + assert_eq!( + with_host("host=127.0.0.1 port=5432 sslmode=disable", "127.0.0.1", 9000), + "host=127.0.0.1 port=9000 sslmode=disable" + ); + assert_eq!( + with_host("host='127.0.0.1' port='5432'", "127.0.0.1", 9000), + "host=127.0.0.1 port=9000" + ); + assert_eq!( + with_host( + "host=127.0.0.1 port=5432 options='-c role=r'", + "127.0.0.1", + 9000 + ), + "host=127.0.0.1 port=9000 options='-c role=r'" + ); + assert_eq!( + with_local_proxy("host=127.0.0.1 port=5432", 9000), + "host=127.0.0.1 port=9000" + ); + } + + #[test] + fn url_failover_puts_the_missing_host_first() { + assert_eq!( + with_failing_first_host("postgresql://u:p@127.0.0.1:5432/db?sslmode=disable"), + "postgresql://u:p@missing.invalid,127.0.0.1:5432/db?sslmode=disable" + ); + } + + #[test] + fn keyword_failover_puts_the_missing_host_first() { + assert_eq!( + with_failing_first_host("host=127.0.0.1 port=5432 sslmode=disable"), + "host=missing.invalid,127.0.0.1 port=5432 sslmode=disable" + ); + } +} diff --git a/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs b/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs index cecca20de2..31ef6e1384 100644 --- a/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs +++ b/engine/crates/layerfs-persistence/tests/pg_doctor/mod.rs @@ -10,9 +10,21 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, time::{Duration, Instant}, }; +pub mod conninfo; pub mod namespace; pub mod proxy; pub const BINDING: &[u8] = b"doctor-postgres-acceptance"; +/// An auxiliary server the physical doctor's extra cases need, or `None` +/// with the usual NOT_RUN note. The primary `LAYERFS_TEST_POSTGRES_URL` +/// still drives the rest of the target. +pub fn auxiliary_url(variable: &str) -> Option { + let url = std::env::var(variable).ok(); + if url.is_none() { + eprintln!("NOT_RUN: {variable} is unset"); + } + url +} + pub struct Fixture { pub selection: PostgresSelection, pub writer: Option, @@ -70,9 +82,10 @@ impl Fixture { self.roles.push(role.clone()); role } + /// The same selection with `role` applied to every connection. pub fn restricted(&self, role: &str) -> PostgresSelection { let mut result = self.selection.clone(); - result.url.push_str(&format!("&options=-c%20role%3D{role}")); + result.url = conninfo::with_option(&result.url, "options", &format!("-c role={role}")); result } pub fn fingerprint(&mut self) -> Vec<(String, i64, String)> { diff --git a/engine/crates/layerfs-persistence/tests/postgresql_commit_round_trips.rs b/engine/crates/layerfs-persistence/tests/postgresql_commit_round_trips.rs index 99c0df929a..d43fad8377 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_commit_round_trips.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_commit_round_trips.rs @@ -364,8 +364,9 @@ fn commit_store_calls_are_counted_on_both_engines() { assert_eq!(pg("reserve").round_trips, 3); // The lane: its BEGIN, the head read, commit_branch's BEGIN, joined // snapshot, Commit insert, parent line, lineage insert, the position - // read, advance and COMMIT, and the lane's ROLLBACK. - assert_eq!(pg("lane + head + commit_branch").round_trips, 11); + // read, advance and COMMIT, the T6.5 epoch READ and its FOR SHARE + // re-check, and the lane's ROLLBACK. + assert_eq!(pg("lane + head + commit_branch").round_trips, 13); // #70: descriptors and bodies in one statement, found or missing. assert_eq!(pg("read_packs 2").round_trips, 1); assert_eq!(pg("read_packs missing").round_trips, 1); diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs index acdcebce6a..9b12630626 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs @@ -5,6 +5,18 @@ //! scratch schema or op row. A leftover at `G` is removed; one at `G + 1` //! finishes Phase C (lost-reply resume). A conflicting lock is `Busy` (55P03). //! +//! T10.4 adds the bounded `compact.*` capture tests: a successful rebuild, the +//! `none` path, and the Busy/Unresolved refusals. They assert the same closed +//! labels and values as the SQLite capture test. +//! +//! AC-44 adds the kill arms: a real `pg_terminate_backend` of the recorded +//! `(pid, backend_start)` before the swap COMMIT (wholly at `G`) and during +//! Phase C after the swap COMMIT (wholly at `G + 1`), plus the deterministic +//! `PostgresAfterSwap` lost-reply arm whose owner is then really killed. A +//! named failpoint alone is a simulated lost reply: the process survives and +//! releases its intent. A killed backend leaves the op, the intent and (before +//! COMMIT) the scratch for the next call to resolve. +//! //! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for //! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres //! sslmode=disable`). Without it each test reports NOT_RUN. @@ -12,39 +24,54 @@ mod support; use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; use layerfs_history::{ AddLayerOutcome, AddLayerRequest, BranchId, CommitMeta, CommitOutcome, CommitRequest, CommitVia, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, LayerStackId, LayerStackRecord, StackInitialization, }; +use layerfs_persistence::compact_metrics::CompactMetricsCapture; use layerfs_persistence::{ CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, }; use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; -use support::{branch_id, name, stack, BINDING}; +use support::metrics::{assert_bounded, metric, value}; +use support::{branch_id, name, stack}; /// Drops the test's schema when the test ends, pass or fail. struct Schema { url: String, selection: PostgresSelection, + /// This test's own catalog identity. The Store-scoped admission advisory + /// key derives from it, and PostgreSQL advisory locks are server-wide + /// (not schema-scoped), so tests that shared one binding would exclude + /// each other while running in parallel. + binding: Vec, } impl Schema { fn new(url: &str, tag: &str) -> Self { let name = format!("lfs_t9_{tag}_{}", std::process::id()); let selection = PostgresSelection::new(url, name); + let binding = binding(tag); let schema = Self { url: url.to_owned(), selection, + binding, }; schema.drop_schema(); schema } fn open(&self) -> PostgresHandles { - PostgresHandles::open_writable(&self.selection, BINDING, [71; 32]).expect("Store open") + PostgresHandles::open_writable(&self.selection, &self.binding, [71; 32]) + .expect("Store open") } fn create(&self) -> PostgresHandles { - PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) - .expect("Store creation") + PostgresHandles::create( + &self.selection, + StoragePolicy::frozen_default(), + &catalog(&self.binding), + ) + .expect("Store creation") } fn client(&self) -> postgres::Client { let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); @@ -95,9 +122,16 @@ impl Drop for Schema { } } -fn catalog() -> HistoryCatalogConfig { +/// Every test opens its own catalog identity, as a real Store does: the +/// PostgreSQL advisory keys are server-global, so two tests that shared a +/// binding could see each other's intents while running in parallel. +fn binding(tag: &str) -> Vec { + format!("postgresql-compact-{tag}").into_bytes() +} + +fn catalog(binding: &[u8]) -> HistoryCatalogConfig { HistoryCatalogConfig { - binding_key: BINDING.to_vec(), + binding_key: binding.to_vec(), cursor_key: [71; 32], incarnation: 7, } @@ -297,6 +331,345 @@ fn db_generation(schema: &Schema) -> u64 { .unwrap() } +/// The phase counts one PostgreSQL rebuild times: Phase M/A/B/C, each once. +fn assert_pg_rebuild_phases(metrics: &layerfs_persistence::compact_metrics::CompactMetrics) { + for phase in ["pg_mark", "pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "PostgreSQL {phase} was timed once" + ); + assert!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "sum")] + ) > 0, + "PostgreSQL {phase} took time" + ); + assert!( + value( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "max")] + ) > 0, + "PostgreSQL {phase} has a duration" + ); + } + for phase in ["mark", "build", "verify", "install"] { + assert_eq!( + metric( + metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "a PostgreSQL run never emits the SQLite phase {phase}" + ); + } +} + +#[test] +fn a_rebuild_records_bounded_metrics_with_pg_phases() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_rebuild"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the metrics rebuild collects"); + + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(21)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "rebuild")]), + 1 + ); + assert_pg_rebuild_phases(&metrics); + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert!(value(&metrics, "compact.duration_ns", &[("statistic", "sum")]) > 0); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + let kept = receipt + .kept_waste + .dead_in_retained_packs_bytes + .saturating_add(receipt.kept_waste.delta_bases_bytes) + .saturating_add(receipt.kept_waste.inode_blocks_bytes); + assert_eq!(value(&metrics, "compact.kept_waste_bytes", &[]), kept); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + receipt.residue.bytes + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + assert_eq!(value(&metrics, "compact.settles", &[("kind", "run")]), 1); + assert_eq!( + metric(&metrics, "compact.settles", &[("kind", "leftover")]), + None, + "a fresh rebuild settles no leftover" + ); + assert_eq!( + metric(&metrics, "compact.retries", &[("reason", "leftover")]), + None + ); + + clean(&schema.open()); +} + +#[test] +fn a_none_path_records_only_the_pg_mark_phase() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_none"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + + let capture = CompactMetricsCapture::start(); + let receipt = handles.compact(&request(22)).unwrap(); + let metrics = capture.finish(); + + assert_eq!(receipt.path, CompactPath::None); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "none")]), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", "pg_mark"), ("statistic", "count")] + ), + 1 + ); + for phase in ["pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "the PostgreSQL none path never enters {phase}" + ); + } + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + receipt.dead_bytes_bound + ); + assert_eq!( + value(&metrics, "compact.dead_bytes", &[("kind", "refined")]), + receipt.dead_bytes_refined + ); + assert_eq!( + value(&metrics, "compact.residue_bytes", &[]), + 0, + "none leaves no artifact" + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "before")]), + receipt.generation_before + ); + assert_eq!( + value(&metrics, "compact.generation", &[("which", "after")]), + receipt.generation_after, + "none keeps the generation" + ); + assert_eq!(receipt.generation_after, receipt.generation_before); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "generation"), ("result", "pass")] + ), + 1 + ); + assert_eq!( + value( + &metrics, + "compact.slo", + &[("objective", "residue_free"), ("result", "pass")] + ), + 1 + ); + assert_eq!(value(&metrics, "compact.settles", &[("kind", "run")]), 1); + + clean(&schema.open()); +} + +#[test] +fn a_busy_swap_records_a_failed_outcome_and_a_busy_refusal() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_busy"); + schema.drop_scratch(23); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the metrics busy run would collect"); + // Phase A never touches store_totals; the swap's ACCESS EXCLUSIVE NOWAIT + // does, so this lock is met exactly at Phase B (SQLSTATE 55P03). + let mut blocker = schema.client(); + blocker + .batch_execute(&format!( + "BEGIN; LOCK TABLE {}.binding_hint_block IN ACCESS EXCLUSIVE MODE;", + schema.selection.schema + )) + .unwrap(); + + let capture = CompactMetricsCapture::start(); + let error = handles.compact(&request(23)).unwrap_err(); + let metrics = capture.finish(); + blocker.batch_execute("ROLLBACK").unwrap(); + + assert!( + matches!(error, layerfs_history::HistoryError::Busy), + "{error:?}" + ); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "failed")]), + 1 + ); + assert_eq!(value(&metrics, "compact.refusals", &[("code", "busy")]), 1); + for phase in ["pg_mark", "pg_build", "pg_swap"] { + assert_eq!( + value( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + 1, + "the refused run reached {phase}" + ); + } + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", "pg_cleanup"), ("statistic", "count")] + ), + None, + "the refused run never reaches Phase C" + ); + assert_eq!( + metric(&metrics, "compact.dead_bytes", &[("kind", "bound")]), + None, + "a failed run publishes no report gauge" + ); + assert_eq!( + metric(&metrics, "compact.generation", &[("which", "after")]), + None + ); + assert_eq!( + metric(&metrics, "compact.settles", &[("kind", "run")]), + None + ); + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); +} + +#[test] +fn an_uncertain_intent_records_an_unresolved_refusal() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "metrics_unresolved"); + let handles = schema.create(); + // A dead owner's uncertain compaction intent with no `gc_op` row: the + // next call meets it at admission, before Phase M writes anything. + let dead = [0x44u8; 16]; + let intent = GcIntent::operation( + dead, + GcIntentKind::Compaction, + GcScopeKind::Store, + vec![0u8; 16], + ) + .with_phase(Some("mark")); + handles.history.gc_register(&intent).unwrap(); + handles.history.gc_mark_uncertain(&dead).unwrap(); + + let capture = CompactMetricsCapture::start(); + let error = handles.compact(&request(24)).unwrap_err(); + let metrics = capture.finish(); + + assert!( + matches!(error, layerfs_history::HistoryError::Unresolved(_)), + "{error:?}" + ); + assert_bounded(&metrics); + assert_eq!(value(&metrics, "compact.runs", &[]), 1); + assert_eq!( + value(&metrics, "compact.operations", &[("outcome", "failed")]), + 1 + ); + assert_eq!( + value(&metrics, "compact.refusals", &[("code", "unresolved")]), + 1 + ); + for phase in ["pg_mark", "pg_build", "pg_swap", "pg_cleanup"] { + assert_eq!( + metric( + &metrics, + "compact.phase.duration_ns", + &[("phase", phase), ("statistic", "count")] + ), + None, + "the refusal stops before {phase}" + ); + } + assert_eq!( + value(&metrics, "compact.duration_ns", &[("statistic", "count")]), + 1 + ); + assert_eq!( + metric(&metrics, "compact.generation", &[("which", "after")]), + None + ); + + handles.history.gc_release(&dead).unwrap(); +} + fn clean(handles: &PostgresHandles) { let mut out = Vec::new(); handles @@ -580,3 +953,716 @@ fn an_exclusive_table_lock_makes_the_swap_busy() { "a 55P03 lock at the swap is Busy, got {error:?}" ); } + +// --- AC-44: a killed backend and the lost COMMIT reply --------------------- +// +// The failpoints above stop a live session and return `UnknownOutcome`: the +// process survives and releases its own intent, so they are a *simulated* lost +// reply. The tests below kill the PostgreSQL backend that owns the round, at a +// point where the server's own `pg_terminate_backend(pid, timeout)` is scoped to +// the `(pid, backend_start)` recorded in `gc_op` (the §12 proof). The next call +// then meets a real leftover whose recorded owner is gone and must resolve it: +// +// * `a_killed_backend_before_the_swap_commit_leaves_the_store_at_g` parks the +// backend inside Phase B's own `UPDATE gc_op`, before COMMIT, and terminates +// it. The Store is wholly at `G`; the next compact proves the owner ended and +// adopts the `G` leftover. +// * `a_killed_backend_during_phase_c_resumes_at_g_plus_1` parks the backend in +// Phase C's `store_totals` settle, after COMMIT committed the swap, and +// terminates it. The Store is wholly at `G + 1`; the next compact reads the +// incarnation and finishes Phase C. +// * `a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1` uses the named +// `PostgresAfterSwap` failpoint as the deterministic stand-in for the instant +// a COMMIT reply is lost (the caller sees `UnknownOutcome` while the swap is +// committed), then kills that owner backend so the owner is provably ended +// before a fresh connection resolves the reply. +// +// The only injected condition is a test-only `BEFORE UPDATE` trigger on the +// statement whose commit must be interrupted; it calls `pg_sleep` so a second +// connection can find the backend in `pg_stat_activity` and terminate exactly +// that process. Nothing in the Store is modified by the injection. + +/// The four `store_totals` columns, so a killed round can be shown not to have +/// settled them. +fn totals(schema: &Schema) -> (i64, i64, i64, i64) { + let mut client = schema.client(); + let row = client + .query_one( + &format!( + "SELECT logical,canonical,stored,metadata FROM {}.store_totals WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("store totals"); + (row.get(0), row.get(1), row.get(2), row.get(3)) +} + +/// One `gc_op` row: `(pid, backend_start, phase)`. +fn op_owner(schema: &Schema) -> Option<(i64, String, String)> { + let mut client = schema.client(); + let row = client + .query_opt( + &format!( + "SELECT pid, backend_start::text, phase::text FROM {}.gc_op WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("the gc_op row"); + row.map(|row| (row.get(0), row.get(1), row.get(2))) +} + +/// Waits for the interrupted round to record its `(pid, backend_start)`. +fn wait_for_op_owner(schema: &Schema) -> (i64, String, String) { + for _ in 0..2400 { + if let Some(owner) = op_owner(schema) { + return owner; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the interrupted round never wrote its gc_op row"); +} + +/// Installs a test-only trigger that parks the backend in `pg_sleep` on the +/// next `UPDATE` of `table`, before the update becomes visible or is committed. +fn park_backend(schema: &Schema, table: &str) { + let name = format!("lfs_test_park_{table}"); + let mut client = schema.client(); + client + .batch_execute(&format!( + "CREATE OR REPLACE FUNCTION {schema}.{name}() RETURNS trigger LANGUAGE plpgsql AS \ + $fn$ BEGIN PERFORM pg_sleep(120); RETURN NEW; END $fn$;\ + CREATE TRIGGER {name} BEFORE UPDATE ON {schema}.{table} \ + FOR EACH ROW EXECUTE FUNCTION {schema}.{name}();", + schema = schema.selection.schema, + table = table, + name = name, + )) + .expect("the test parking trigger"); +} + +/// Removes the test-only parking trigger and its function. +fn unpark_backend(schema: &Schema, table: &str) { + let name = format!("lfs_test_park_{table}"); + let mut client = schema.client(); + let _ = client.batch_execute(&format!( + "DROP TRIGGER IF EXISTS {name} ON {schema}.{table};\ + DROP FUNCTION IF EXISTS {schema}.{name}();", + schema = schema.selection.schema, + table = table, + name = name, + )); +} + +/// Waits for the child backend parked in the unique schema-qualified +/// statement the test injected, and returns `(pid, backend_start)`. Matching on +/// the statement avoids reading `gc_op` while Phase B holds every base table +/// `ACCESS EXCLUSIVE`, so there is no race between the parent's poll and the +/// parked swap transaction. +fn wait_for_parked_statement(schema: &Schema, table: &str) -> (i64, String) { + let pattern = format!("%{}.{table}%", schema.selection.schema); + let mut client = schema.client(); + for _ in 0..4800 { + let rows = client + .query( + "SELECT pid, backend_start::text FROM pg_stat_activity \ + WHERE datname=current_database() AND state='active' \ + AND wait_event='PgSleep' AND query LIKE $1 AND pid<>pg_backend_pid()", + &[&pattern], + ) + .expect("pg_stat_activity"); + if let Some(row) = rows.first() { + let pid: i32 = row.get(0); + return (i64::from(pid), row.get(1)); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("no backend parked in a {table} statement"); +} + +/// Kills exactly the recorded owner: the two-argument +/// `pg_terminate_backend(pid, timeout)`, scoped by `(pid, backend_start)`, never +/// the caller and never another database. This is the §12 proof the next call +/// relies on. +fn kill_recorded_owner(schema: &Schema, pid: i64, backend_start: &str) { + let mut client = schema.client(); + let killed: bool = client + .query_one( + "SELECT COALESCE(bool_or(pg_terminate_backend(a.pid, 30000)), false) \ + FROM pg_stat_activity a \ + WHERE a.pid::bigint=$1 AND a.backend_start::text=$2 \ + AND a.datname=current_database() AND a.pid<>pg_backend_pid()", + &[&pid, &backend_start], + ) + .expect("pg_terminate_backend") + .get(0); + assert!(killed, "the recorded owner backend {pid} was terminated"); +} + +/// Proves the recorded owner ended: no `pg_stat_activity` row keeps the same +/// `(pid, backend_start)` pair. +fn assert_owner_ended(schema: &Schema, pid: i64, backend_start: &str) { + let mut client = schema.client(); + for _ in 0..2400 { + let alive: i64 = client + .query_one( + "SELECT count(*) FROM pg_stat_activity \ + WHERE pid::bigint=$1 AND backend_start::text=$2", + &[&pid, &backend_start], + ) + .expect("pg_stat_activity") + .get(0); + if alive == 0 { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend {pid} did not end"); +} + +/// The compaction intents a Store's admission table still holds. +fn compaction_intents(handles: &PostgresHandles, schema: &Schema) -> Vec { + let catalog = layerfs_history::CatalogId::derive(&schema.binding).expect("catalog identity"); + handles + .history + .gc_intents(GcScopeKind::Store, catalog.as_slice()) + .expect("compaction intents") +} + +/// Runs one compact on a connection of its own, so the parent can terminate +/// exactly that backend. +fn spawn_round( + selection: PostgresSelection, + binding: Vec, + op: u8, + failpoint: Option, +) -> std::thread::JoinHandle> { + std::thread::spawn(move || { + let handles = + PostgresHandles::open_writable(&selection, &binding, [71; 32]).expect("child open"); + handles.compact(&CompactRequest { + op_id: [op; 16], + failpoint, + ..CompactRequest::default() + }) + }) +} + +#[test] +fn a_killed_backend_before_the_swap_commit_leaves_the_store_at_g() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "kill_before_commit"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the killed round would collect"); + let before = generation(&setup); + let totals_before = totals(&schema); + + // Park Phase B's own `UPDATE gc_op` before COMMIT. The kill therefore lands + // after Phase A built the scratch and before the swap transaction can + // commit, which is the "killed before COMMIT" arm. + park_backend(&schema, "gc_op"); + let child = spawn_round(schema.selection.clone(), schema.binding.clone(), 41, None); + let (pid, start) = wait_for_parked_statement(&schema, "gc_op"); + kill_recorded_owner(&schema, pid, &start); + let error = child + .join() + .expect("the killed round's thread") + .unwrap_err(); + unpark_backend(&schema, "gc_op"); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "a killed backend loses the reply: {error:?}" + ); + + // Wholly old at `G`: the swap transaction rolled back, so the Store is not + // mixed and the old totals still hold. + assert_eq!(db_generation(&schema), before); + assert_eq!(totals(&schema), totals_before); + assert!(located(&setup, live), "the retained object survives"); + assert!(located(&setup, dead), "the swap did not commit"); + // The documented leftover of an interrupted round: Phase M's op row keeps + // the owner, Phase A's scratch schema and the dead set remain. + let (op_pid, op_start, op_phase) = op_owner(&schema).expect("the op row survives the kill"); + assert_eq!((op_pid, op_phase), (pid, "mark".to_owned())); + assert_eq!(op_start, start); + assert!( + schema.scratch_exists(41), + "Phase A's scratch schema remains" + ); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + + // The owner is provably ended, and the admission intent the killed process + // never released is still visible: a compaction intent is not an uncertain + // one, so the resume adopts it instead of reporting `Unresolved`. + assert_owner_ended(&schema, pid, &start); + let intents = compaction_intents(&setup, &schema); + assert_eq!( + intents.len(), + 1, + "the killed process never released its intent" + ); + assert_eq!(intents[0].kind, GcIntentKind::Compaction); + assert_eq!(intents[0].phase.as_deref(), Some("mark")); + assert!( + !intents[0].uncertain, + "a compaction intent carries no uncertain owner" + ); + + // The next compact proves the owner ended, adopts the `G` leftover and + // rebuilds once: `G -> G + 1`. + let resumed = schema.open(); + let receipt = resumed.compact(&request(42)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 42); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the next round collects the dead pack" + ); + assert!( + located(&resumed, live), + "the retained object survives the resume" + ); + clean(&schema.open()); +} + +#[test] +fn a_killed_backend_during_phase_c_resumes_at_g_plus_1() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "kill_phase_c"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the killed cleanup would collect"); + let before = generation(&setup); + let totals_before = totals(&schema); + + // Park Phase C's `store_totals` settle. The swap transaction already + // committed `G + 1` before Phase C starts, so the kill lands on the + // "installed but cleanup unfinished" arm. + park_backend(&schema, "store_totals"); + let child = spawn_round(schema.selection.clone(), schema.binding.clone(), 43, None); + let (pid, start) = wait_for_parked_statement(&schema, "store_totals"); + kill_recorded_owner(&schema, pid, &start); + let error = child + .join() + .expect("the killed round's thread") + .unwrap_err(); + unpark_backend(&schema, "store_totals"); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "a killed backend loses the reply: {error:?}" + ); + + // Wholly new at `G + 1`, but Phase C rolled back: the committed swap has + // already replaced the payload tables (the dead pack is gone), while the + // persisted dead history rows, the op row and the old totals remain. + assert_eq!(db_generation(&schema), before + 1); + assert_eq!(totals(&schema), totals_before); + assert!( + !located(&setup, dead), + "the committed swap already replaced the payload tables" + ); + assert!(located(&setup, live)); + let (op_pid, op_start, op_phase) = op_owner(&schema).expect("the op row survives the kill"); + assert_eq!((op_pid, op_phase), (pid, "swap".to_owned())); + assert_eq!(op_start, start); + assert!( + !schema.scratch_exists(43), + "the swap transaction dropped the scratch before COMMIT" + ); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + assert_owner_ended(&schema, pid, &start); + // A real kill runs no error path: the intent is still registered. + let intents = compaction_intents(&setup, &schema); + assert_eq!( + intents.len(), + 1, + "the killed process never released its intent" + ); + assert_eq!(intents[0].kind, GcIntentKind::Compaction); + assert!(!intents[0].uncertain); + + // The next compact reads the incarnation (`G + 1`) and finishes Phase C, + // reporting the outcome the lost reply would have reported. + let resumed = schema.open(); + let receipt = resumed.compact(&request(44)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 44); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the resumed Phase C collects the dead pack" + ); + assert!(located(&resumed, live)); + clean(&schema.open()); +} + +#[test] +fn a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "lost_commit"); + let setup = schema.create(); + let live = filesystem_root(&setup, 1); + initialize(&setup, 1, live); + let dead = whole_file(&setup, b"a pack the lost COMMIT would collect"); + let before = generation(&setup); + + // The named failpoint is the deterministic stand-in for the instant the + // swap's COMMIT reply is lost: the swap committed at `G + 1`, the caller + // saw `UnknownOutcome`, and its backend is still alive. The child holds its + // handles open so the parent can end that owner with a real kill. + let (started_tx, started_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let selection = schema.selection.clone(); + let binding = schema.binding.clone(); + let child = std::thread::spawn(move || { + let handles = + PostgresHandles::open_writable(&selection, &binding, [71; 32]).expect("child open"); + let result = handles.compact(&request_at(45, CompactFailpoint::PostgresAfterSwap)); + started_tx + .send(()) + .expect("the child reports its lost reply"); + release_rx.recv().expect("the parent releases the child"); + drop(handles); + result + }); + started_rx + .recv_timeout(std::time::Duration::from_secs(60)) + .expect("the failpoint returned"); + + // The committed incarnation says the swap happened; the op still names + // Phase C and the scratch is already gone. The child is still holding its + // handles open, so this backend is alive and idle: the real kill below + // proves the owner ended. + assert_eq!(db_generation(&schema), before + 1); + let (pid, start, phase) = wait_for_op_owner(&schema); + assert_eq!(phase, "swap"); + assert!(!schema.scratch_exists(45)); + assert!(schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0); + // A simulated lost reply keeps the process alive and releases its intent; + // the real kill below then proves the owner ended. + assert!( + compaction_intents(&setup, &schema).is_empty(), + "the failpoint path released its own intent" + ); + kill_recorded_owner(&schema, pid, &start); + assert_owner_ended(&schema, pid, &start); + release_tx.send(()).expect("release the lost-reply child"); + let error = child.join().expect("the lost-reply thread").unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "the failpoint simulates the lost reply: {error:?}" + ); + + // A fresh connection resolves the committed swap: it reads `G + 1` and + // finishes Phase C without replaying the swap. + let resumed = schema.open(); + let receipt = resumed.compact(&request(46)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 46); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&resumed, dead), + "the resumed Phase C collects the dead pack" + ); + assert!(located(&resumed, live)); + clean(&schema.open()); +} + +// --- AC-44: the privilege half --------------------------------------------- + +/// The base URL with `role` as the login user, for either the keyword or the +/// URL form. +fn role_url(url: &str, role: &str) -> String { + if let Some((scheme, tail)) = url.split_once("://") { + let authority_end = tail.find('/').unwrap_or(tail.len()); + let authority = &tail[..authority_end]; + let host = authority.rsplit('@').next().unwrap_or(authority); + format!("{scheme}://{role}@{host}{}", &tail[authority_end..]) + } else { + let mut parts = Vec::new(); + let mut replaced = false; + for part in url.split_whitespace() { + if part.starts_with("user=") { + parts.push(format!("user={role}")); + replaced = true; + } else { + parts.push(part.to_owned()); + } + } + if !replaced { + parts.push(format!("user={role}")); + } + parts.join(" ") + } +} + +/// Serializes the role and database ACL writes of the AC-44 matrix: the +/// tests run in parallel, and two `GRANT`/`REVOKE` statements on the same +/// `pg_database` row can fail with `tuple concurrently updated`. The lock is +/// held only while a fixture writes, never through a compaction. +static ROLE_SETUP: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn role_setup() -> std::sync::MutexGuard<'static, ()> { + ROLE_SETUP + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// A login role of the AC-44 privilege matrix. It owns the Store's tables and +/// sequences, so a swap may drop, move and re-create them, while the schema +/// and database stay the fixture administrator's; each test grants exactly +/// the Store-wide privileges its arm needs. The suite's PostgreSQL server is +/// this test binary's own, so it revokes the PUBLIC database grants once per +/// role: a superuser bypasses them. +struct Role { + name: String, + url: String, + admin_url: String, + database: String, +} +impl Role { + fn new(schema: &Schema, tag: &str) -> Self { + let _setup = role_setup(); + let name = format!("lfs_g54_{tag}_{}", std::process::id()); + let mut admin = + postgres::Client::connect(&schema.url, postgres::NoTls).expect("role admin"); + let _ = admin.batch_execute(&format!("DROP ROLE IF EXISTS {name}")); + admin + .batch_execute(&format!("CREATE ROLE {name} LOGIN")) + .expect("role creation"); + let database: String = admin + .query_one("SELECT current_database()", &[]) + .expect("database") + .get(0); + admin + .batch_execute(&format!( + "REVOKE CREATE, TEMP ON DATABASE {database} FROM PUBLIC" + )) + .expect("database public grants"); + admin + .batch_execute(&format!( + "GRANT USAGE ON SCHEMA {} TO {name}", + schema.selection.schema + )) + .expect("schema usage"); + let owned: Vec<(String, String)> = admin + .query( + "SELECT c.relname::text, c.relkind::text FROM pg_class c \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = $1 AND (c.relkind IN ('r', 'p') \ + OR (c.relkind = 'S' AND NOT EXISTS (SELECT 1 FROM pg_depend d \ + WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid \ + AND d.refclassid = 'pg_class'::regclass \ + AND d.deptype IN ('a', 'i'))))", + &[&schema.selection.schema], + ) + .expect("schema relations") + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + for (relation, kind) in owned { + let relation = relation.replace('"', "\"\""); + let keyword = if kind == "S" { "SEQUENCE" } else { "TABLE" }; + admin + .batch_execute(&format!( + "ALTER {keyword} \"{}\".\"{relation}\" OWNER TO {name}", + schema.selection.schema + )) + .expect("relation ownership"); + } + Self { + url: role_url(&schema.url, &name), + name, + admin_url: schema.url.clone(), + database, + } + } + fn admin(&self) -> postgres::Client { + postgres::Client::connect(&self.admin_url, postgres::NoTls).expect("role admin") + } + fn grant_schema_create(&self, schema: &Schema) { + let _setup = role_setup(); + self.admin() + .batch_execute(&format!( + "GRANT CREATE ON SCHEMA {} TO {}", + schema.selection.schema, self.name + )) + .expect("schema create"); + } + fn grant_database(&self, create: bool, temp: bool) { + let _setup = role_setup(); + let mut sql = String::new(); + if create { + sql.push_str(&format!( + "GRANT CREATE ON DATABASE {} TO {};", + self.database, self.name + )); + } + if temp { + sql.push_str(&format!( + "GRANT TEMP ON DATABASE {} TO {};", + self.database, self.name + )); + } + if !sql.is_empty() { + self.admin().batch_execute(&sql).expect("database grants"); + } + } + fn grant_read_all_stats(&self) { + let _setup = role_setup(); + self.admin() + .batch_execute(&format!("GRANT pg_read_all_stats TO {}", self.name)) + .expect("stats grant"); + } + fn open(&self, schema: &Schema) -> PostgresHandles { + let selection = PostgresSelection::new(&self.url, schema.selection.schema.clone()); + PostgresHandles::open_writable(&selection, &schema.binding, [71; 32]).expect("role open") + } +} +impl Drop for Role { + fn drop(&mut self) { + let _setup = role_setup(); + if let Ok(mut admin) = postgres::Client::connect(&self.admin_url, postgres::NoTls) { + let _ = admin.batch_execute(&format!("DROP OWNED BY {}", self.name)); + let _ = admin.batch_execute(&format!("DROP ROLE IF EXISTS {}", self.name)); + } + } +} + +/// A Store with a live root and a wholly dead pack, so an arm that got past +/// the privilege preflight would take the rebuild path and create the scratch +/// schema. +fn seed_rebuild(schema: &Schema) -> (PostgresHandles, ObjectId, ObjectId) { + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack a privileged compaction collects"); + (handles, live, dead) +} + +#[test] +fn a_role_without_create_on_the_database_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_database_create"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_database_create"); + role.grant_schema_create(&schema); + role.grant_database(false, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(61)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("CREATE on the database")), + "{error:?}" + ); + assert!(!schema.scratch_exists(61), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_create_on_the_store_schema_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_schema_create"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_schema_create"); + role.grant_database(true, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(62)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("CREATE on the Store schema")), + "{error:?}" + ); + assert!(!schema.scratch_exists(62), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_temp_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_temp"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_temp"); + role.grant_schema_create(&schema); + role.grant_database(true, false); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let error = handles.compact(&request(63)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("TEMP on the database")), + "{error:?}" + ); + assert!(!schema.scratch_exists(63), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_role_without_pg_read_all_stats_is_refused_as_unsupported() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_stats"); + let (setup, _live, _dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "no_stats"); + role.grant_schema_create(&schema); + role.grant_database(true, true); + let handles = role.open(&schema); + let error = handles.compact(&request(64)).unwrap_err(); + assert!( + matches!(&error, layerfs_history::HistoryError::Unsupported(m) if m.contains("pg_read_all_stats")), + "{error:?}" + ); + assert!(!schema.scratch_exists(64), "no scratch schema was created"); + assert_eq!(schema.count("gc_op"), 0, "no compaction op row"); + assert_eq!(db_generation(&schema), before, "nothing was marked"); +} + +#[test] +fn a_fully_privileged_role_compacts() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "privilege_full"); + let (setup, live, dead) = seed_rebuild(&schema); + let before = generation(&setup); + + let role = Role::new(&schema, "full"); + role.grant_schema_create(&schema); + role.grant_database(true, true); + role.grant_read_all_stats(); + let handles = role.open(&schema); + let receipt = handles.compact(&request(65)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert_settled(&schema, 65); + assert_eq!(db_generation(&schema), before + 1); + assert!( + !located(&handles, dead), + "the rebuild collected the dead pack" + ); + assert!( + located(&handles, live), + "the retained root survives the role's rebuild" + ); + drop(handles); + clean(&schema.open()); +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs new file mode 100644 index 0000000000..6d7a6fc643 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact_fixtures.rs @@ -0,0 +1,1790 @@ +//! #41 T12.3b: the PostgreSQL half of the §12 acceptance matrix. +//! +//! One fixture, built through the public APIs only, mirrors the T12.1 SQLite +//! fixture on PostgreSQL so the cross-backend rows have the same evidence on +//! both engines: the §6.4 live-heads digest is compared against the *SQLite +//! golden literal* (AC-40, AC-56), and the §11 report, §7 worklist and §8 +//! swap/cleanup are asserted on the Store rows (AC-02, AC-05, AC-17, AC-18, +//! AC-25, AC-41, AC-52). Two failpoint tests cover the §9 leftovers the T9 +//! note left open: a save after the swap survives the resumed Phase C +//! (AC-28) and the resumed Store is usable with totals, ACL, trigger and +//! hints checked (AC-43). A marker test covers the PostgreSQL `project_epoch` +//! bump and fence (AC-24, AC-39). +//! +//! Genuinely unreachable arms are named in +//! `core/docs/issues/project-41/gc-t12.3b-postgresql-acceptance.md` with the +//! reason (real standby, killed-backend lost COMMIT, a save racing the swap). +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=postgres +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{ + encode_whole_file_payload, + file::mapping::{self, ExtentNode, ExtentSlice, FileState}, + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + root::profile_id, + scope_for_seed, FilesystemRoot, PathName, + }, + inode_leaf::{ + encode_inode_value, InodeKind, InodeLeaf, InodeLeafRow, InodeMetadata, InodeValue, + LEAF_ROW_BYTES, + }, + FinalizedObject, ObjectId, ObjectRole, +}; +use layerfs_history::error::{HistoryError, Missing}; +use layerfs_history::path::NormalizedPath; +use layerfs_history::{ + AddLayerOutcome, AddLayerRequest, BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, HistoryCatalog, + HistoryCatalogConfig, LayerId, LayerRecord, LayerStackId, LayerStackRecord, PathRemovalKind, + PathRemovalRequest, StackInitialization, +}; +use layerfs_persistence::{ + diagnostic::{Observation, Options, Outcome}, + CompactFailpoint, CompactPath, CompactRequest, PostgresHandles, PostgresSelection, +}; +use layerfs_storage::{ + encoding::{encode_full, encode_prefix, raw_payload, CompressionWorkspace}, + location::{InodeValueRow, ObjectLocation, PackDomain}, + pack::{assemble, build_group, layout, PackLane}, + port::{PackPersistence, Publication, PublishedPack, Reserve}, + save::SaveProfile, + Storage, StorageCapacities, StoragePolicy, +}; +use std::sync::Arc; + +const BINDING: &[u8] = b"doctor-adversarial"; +const CURSOR: [u8; 32] = [71; 32]; +const SCOPE_SEED: [u8; 32] = [11; 32]; +/// The T12.1 SQLite transcript: one literal vector for both backends. +const SQLITE_GOLDEN: &str = include_str!("fixtures/sqlite_compact_golden.json"); + +/// Every string value the SQLite golden holds under `key`, in file order. +fn sqlite_literals(key: &str) -> Vec { + let needle = format!("\"{key}\": \""); + let mut out = Vec::new(); + let mut from = 0; + while let Some(offset) = SQLITE_GOLDEN[from..].find(&needle) { + let start = from + offset + needle.len(); + let rest = &SQLITE_GOLDEN[start..]; + let end = rest.find('"').expect("the literal ends"); + out.push(rest[..end].to_owned()); + from = start + end + 1; + } + out +} + +fn sqlite_literal(key: &str) -> String { + let mut values = sqlite_literals(key); + assert!( + values.len() == 1, + "the SQLite golden names {key} once, found {}", + values.len() + ); + values.remove(0) +} + +fn sqlite_commit_literal() -> String { + let needle = "\"commit\": ["; + let start = SQLITE_GOLDEN + .find(needle) + .expect("the SQLite golden commit") + + needle.len(); + let rest = &SQLITE_GOLDEN[start..]; + let first = rest.find('"').expect("the commit literal opens") + 1; + let rest = &rest[first..]; + rest[..rest.find('"').expect("the commit literal closes")].to_owned() +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(DIGITS[(byte >> 4) as usize] as char); + out.push(DIGITS[(byte & 15) as usize] as char); + } + out +} + +fn catalog() -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: CURSOR, + incarnation: 7, + } +} + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +/// One schema, dropped when the test ends, pass or fail. +struct Schema { + url: String, + selection: PostgresSelection, +} + +impl Schema { + fn new(url: &str, tag: &str) -> Self { + let name = format!("lfs123b_{tag}_{}", std::process::id()); + let selection = PostgresSelection::new(url, name); + let schema = Self { + url: url.to_owned(), + selection, + }; + schema.drop_schema(); + schema + } + fn open(&self) -> PostgresHandles { + PostgresHandles::open_writable(&self.selection, BINDING, CURSOR).expect("Store open") + } + fn create(&self) -> PostgresHandles { + PostgresHandles::create(&self.selection, StoragePolicy::frozen_default(), &catalog()) + .expect("Store creation") + } + fn client(&self) -> postgres::Client { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).expect("client"); + client + .batch_execute(&format!("SET search_path TO {}", self.selection.schema)) + .expect("search path"); + client + } + fn count(&self, table: &str) -> i64 { + let sql = format!("SELECT count(*) FROM {}.{table}", self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn count_where(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("count").get(0) + } + fn scalar(&self, sql: &str) -> i64 { + let sql = sql.replace("{s}", &self.selection.schema); + self.client().query_one(&sql, &[]).expect("scalar").get(0) + } + fn exists(&self, sql: &str) -> bool { + self.scalar(sql) != 0 + } + fn scratch_exists(&self, op: u8) -> bool { + let scratch = format!("lfs_gc_{}", format!("{op:02x}").repeat(16)); + self.exists(&format!( + "SELECT (CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname='{scratch}') THEN 1 ELSE 0 END)::bigint" + )) + } + fn drop_schema(&self) { + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!( + "DROP SCHEMA IF EXISTS {} CASCADE", + self.selection.schema + )); + } + } +} + +impl Drop for Schema { + fn drop(&mut self) { + self.drop_schema(); + } +} + +/// A Store plus the storage helpers the fixture needs (`Fixture::put` and +/// `inode` publish directly, like the SQLite doctor fixture). +struct Fixture { + schema: Schema, + handles: PostgresHandles, + policy: StoragePolicy, +} + +impl Fixture { + fn new(url: &str, tag: &str) -> Self { + let schema = Schema::new(url, tag); + let handles = schema.create(); + Self { + schema, + handles, + policy: StoragePolicy::frozen_default(), + } + } + fn put(&self, role: ObjectRole, canonical: Vec) -> FinalizedObject { + let object = FinalizedObject::new(role, canonical).expect("object"); + let record = encode_full( + object.canonical(), + role, + &StorageCapacities::from_policy(self.policy).expect("capacities"), + &mut CompressionWorkspace::new().expect("workspace"), + &mut SaveProfile::default(), + ) + .expect("record"); + self.record(&object, record.lane, record.record, Vec::new()); + object + } + fn record( + &self, + object: &FinalizedObject, + lane: PackLane, + record: Vec, + values: Vec, + ) { + let body = + assemble(lane, &[build_group(lane, &[record], None).expect("group")]).expect("body"); + self.body(object, lane, body, values); + } + fn body( + &self, + object: &FinalizedObject, + lane: PackLane, + body: Vec, + values: Vec, + ) { + let reserved = self + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .expect("reserve"); + self.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::for_lane(lane), + Arc::new(body), + )], + objects: vec![ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: 0, + record_number: 0, + }], + inode_values: values, + ..Publication::default() + }) + .expect("publication"); + } + fn inode(&self, canonical: Vec, values: Vec<[u8; 73]>) -> ObjectId { + let object = FinalizedObject::new(ObjectRole::InodeLeaf, canonical).expect("inode"); + let reserved = self + .handles + .storage + .reserve(Reserve { + packs: 0, + ordinals: values.len(), + }) + .expect("ordinals"); + let ordinals: Vec<_> = + (reserved.first_ordinal..reserved.first_ordinal + values.len() as u32).collect(); + let pooled = layerfs_content::inode_leaf::pooled_body(object.canonical(), &ordinals) + .expect("pooled"); + let record = layerfs_storage::encoding::pool::leaf::encode_full(&pooled).expect("leaf"); + self.record( + &object, + PackLane::Ordinary, + record, + ordinals + .into_iter() + .zip(values) + .map(|(ordinal, value)| InodeValueRow { ordinal, value }) + .collect(), + ); + object.id() + } +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: u8, failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +// ---- fixture objects ------------------------------------------------------- + +fn pseudo_bytes(len: usize, seed: u8) -> Vec { + let mut state = u64::from(seed).wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.to_le_bytes()); + } + out.truncate(len); + out +} + +fn whole_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + f.put( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole file"), + ) + .id() +} + +/// One object published through the real Save path, so its +/// `content_signature` slot exists and the mark can list it in `gc_dead`. +fn saved_whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).expect("storage"); + let save = storage.begin_save().expect("begin save"); + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole file"), + ) + .expect("object"); + let id = object.id(); + save.accept(object).expect("accept"); + save.finish().expect("finish"); + id +} + +/// One chunked file's content root (a FileState): the only kind whose +/// `file_digest` row the Store records (a WholeFile root is below the +/// small-file threshold). +fn chunked_file(f: &Fixture, bytes: &[u8]) -> ObjectId { + let mut extents = Vec::new(); + for part in bytes.chunks(32_768) { + let chunk = f + .put( + ObjectRole::Chunk, + mapping::encode_chunk_object(part).expect("chunk"), + ) + .id(); + extents.push(ExtentSlice::new(chunk, 0, part.len() as u32).expect("slice")); + } + let leaf = f + .put( + ObjectRole::ExtentLeaf, + mapping::encode_node( + &ExtentNode::Leaf { + subtree_logical_bytes: bytes.len() as u64, + extents, + }, + true, + ) + .expect("node"), + ) + .id(); + f.put( + ObjectRole::FileState, + mapping::encode_file_state(FileState { + logical_len: bytes.len() as u64, + extent_count: (bytes.len() as u64).div_ceil(32_768), + tree_level: 0, + profile_id: mapping::profile_id(), + mapping_root: leaf, + }) + .expect("file state"), + ) + .id() +} + +/// One valid filesystem tree: a root directory whose only entry is a regular +/// file whose content root is `content`. The inode leaf is pooled. +fn tree(f: &Fixture, content: ObjectId, seed: u8) -> ObjectId { + let directory = f + .put( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { + entries: vec![(PathName::new(&format!("file-{seed}")).expect("name"), 2)], + }) + .expect("directory"), + ) + .id(); + let mut values = Vec::new(); + let mut rows = Vec::new(); + for (serial, kind, count, root, mode) in [ + (1u64, InodeKind::Directory, 0u64, directory, 0o755), + (2u64, InodeKind::RegularFile, 1u64, content, 0o644), + ] { + let value = encode_inode_value(InodeValue { + kind, + namespace_ref_count: count, + content_root: root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode, + mtime_seconds: 0, + mtime_nanoseconds: 0, + }), + }); + values.push(value); + rows.push(InodeLeafRow { serial, value }); + } + let inode = f.inode( + InodeLeaf { + subtree_bytes: LEAF_ROW_BYTES as u64 * rows.len() as u64, + rows, + } + .encode() + .expect("inode leaf"), + values, + ); + f.put( + ObjectRole::FilesystemRoot, + FilesystemRoot::new(profile_id(), scope_for_seed(SCOPE_SEED), 1, inode) + .expect("root") + .encode() + .expect("root bytes"), + ) + .id() +} + +/// Publishes one pack holding each object as its own group, so a retained pack +/// can hold one reached and one unreached record. +fn publish_groups(f: &Fixture, objects: &[FinalizedObject]) -> (i64, Vec) { + let capacities = StorageCapacities::from_policy(f.policy).expect("capacities"); + let mut workspace = CompressionWorkspace::new().expect("workspace"); + let mut profile = SaveProfile::default(); + let groups: Vec<_> = objects + .iter() + .map(|object| { + let record = encode_full( + object.canonical(), + object.role(), + &capacities, + &mut workspace, + &mut profile, + ) + .expect("record"); + build_group(PackLane::WholeFile, &[record.record], None).expect("group") + }) + .collect(); + let body = assemble(PackLane::WholeFile, &groups).expect("body"); + let reserved = f + .handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .expect("reserve"); + f.handles + .storage + .publish(&Publication { + packs: vec![PublishedPack::seal( + reserved.first_pack_id, + PackDomain::Payload, + Arc::new(body.clone()), + )], + objects: objects + .iter() + .enumerate() + .map(|(group, object)| ObjectLocation { + object_id: object.id(), + role: object.role(), + canonical_length: object.canonical_len(), + pack_id: reserved.first_pack_id, + group_number: group, + record_number: 0, + }) + .collect(), + ..Publication::default() + }) + .expect("publication"); + (reserved.first_pack_id, body) +} + +/// Publishes `target` as a PREFIX record over `base`, asserting the encoder +/// kept the delta (a full record would hide the base from the mark). +fn prefix_record(f: &Fixture, target: &FinalizedObject, base: &FinalizedObject, base_id: ObjectId) { + let capacities = StorageCapacities::from_policy(f.policy).expect("capacities"); + let record = encode_prefix( + target.canonical(), + target.role(), + base_id, + raw_payload(base.canonical(), base.role()).expect("payload"), + &capacities, + &mut CompressionWorkspace::new().expect("workspace"), + &mut SaveProfile::default(), + ) + .expect("prefix"); + assert_eq!( + record.base, + Some(base_id), + "the fixture needs a PREFIX record, not a full one" + ); + f.record(target, record.lane, record.record, Vec::new()); +} + +// ---- fixture history ------------------------------------------------------ + +fn initialize(handles: &PostgresHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: scope_for_seed(SCOPE_SEED).object(), + profile: profile_id(), + genesis_root: genesis, + }) + .expect("stack") +} + +fn fork(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId, tag: u8) { + let record = handles.history.layer_stack(stack).unwrap().unwrap(); + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name(&format!("branch-{tag}")), + source: ForkSource::Layer(record.head_layer), + }) + .expect("fork"); +} + +fn expected_root( + handles: &PostgresHandles, + branch: BranchId, +) -> (Option, LayerId, ObjectId) { + let snapshot = handles.history.branch(branch).unwrap().unwrap(); + let root = match snapshot.head_commit { + Some(head) => handles.history.commit(head).unwrap().unwrap().root, + None => { + handles + .history + .layer(snapshot.base_layer) + .unwrap() + .unwrap() + .root + } + }; + (snapshot.head_commit, snapshot.base_layer, root) +} + +fn commit_request( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, +) -> CommitRequest { + let (head, base, root) = expected_root(handles, branch); + CommitRequest { + branch, + expected_head: head, + expected_base: base, + expected_root: root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + } +} + +fn commit( + handles: &PostgresHandles, + branch: BranchId, + candidate_root: ObjectId, + logical: u64, +) -> CommitId { + let (head, base, root) = expected_root(handles, branch); + let meta = CommitMeta { + statistics: CommitStatistics { + logical, + ..CommitStatistics::ZERO + }, + ..CommitMeta::new(CommitVia::Files) + }; + match handles + .history + .commit_branch(&CommitRequest { + branch, + expected_head: head, + expected_base: base, + expected_root: root, + candidate_root, + profile: profile_id(), + scope: scope_for_seed(SCOPE_SEED).object(), + second_parent: None, + meta, + force: false, + }) + .expect("commit") + { + CommitOutcome::Committed { record, .. } => record.id, + other => panic!("commit: {other:?}"), + } +} + +fn publish(handles: &PostgresHandles, stack: LayerStackId, branch: BranchId) -> LayerRecord { + let branch_record = handles.history.branch(branch).unwrap().unwrap(); + let stack_record = handles.history.layer_stack(stack).unwrap().unwrap(); + match handles + .history + .add_layer(&AddLayerRequest { + stack, + branch, + commit: branch_record.head_commit.expect("a committed Branch"), + expected_stack_head: stack_record.head_layer, + expected_branch_base: branch_record.base_layer, + }) + .expect("publish") + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +// ---- raw Store facts ------------------------------------------------------ + +#[derive(Clone, Copy, Debug, Default)] +struct Totals { + logical: u64, + canonical: u64, + stored: u64, + metadata: u64, +} + +fn totals(schema: &Schema) -> Totals { + let row = schema + .client() + .query_one( + &format!( + "SELECT logical,canonical,stored,metadata FROM {}.store_totals WHERE id=1", + schema.selection.schema + ), + &[], + ) + .expect("totals"); + Totals { + logical: u64::try_from(row.get::<_, i64>(0)).unwrap(), + canonical: u64::try_from(row.get::<_, i64>(1)).unwrap(), + stored: u64::try_from(row.get::<_, i64>(2)).unwrap(), + metadata: u64::try_from(row.get::<_, i64>(3)).unwrap(), + } +} + +#[derive(Debug)] +struct PackFacts { + total: u64, + live_bytes: u64, + dead_bytes: u64, + live: Vec, + dead: Vec, +} + +fn pack_facts(schema: &Schema, live_packs: &[i64]) -> PackFacts { + let rows = schema + .client() + .query( + &format!( + "SELECT pack_id, length FROM {}.pack ORDER BY pack_id", + schema.selection.schema + ), + &[], + ) + .expect("packs"); + let mut facts = PackFacts { + total: 0, + live_bytes: 0, + dead_bytes: 0, + live: Vec::new(), + dead: Vec::new(), + }; + for row in rows { + let pack: i64 = row.get(0); + let length = u64::try_from(row.get::<_, i64>(1)).unwrap(); + facts.total += length; + if live_packs.contains(&pack) { + facts.live.push(pack); + facts.live_bytes += length; + } else { + facts.dead.push(pack); + } + } + facts.dead_bytes = facts.total - facts.live_bytes; + facts +} + +fn pack_of(schema: &Schema, id: ObjectId) -> Option { + let object = id.as_bytes(); + let row = schema + .client() + .query_opt( + &format!( + "SELECT pack_id FROM {}.object_location WHERE object_id=$1", + schema.selection.schema + ), + &[&object.as_slice()], + ) + .expect("object location"); + row.map(|row| row.get(0)) +} + +fn object_count(schema: &Schema, id: ObjectId) -> i64 { + let object = id.as_bytes(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.object_location WHERE object_id=$1", + schema.selection.schema + ), + &[&object.as_slice()], + ) + .expect("object count") + .get(0) +} + +fn pack_length(schema: &Schema, pack: i64) -> u64 { + let length = schema + .client() + .query_one( + &format!( + "SELECT length FROM {}.pack WHERE pack_id=$1", + schema.selection.schema + ), + &[&pack], + ) + .expect("pack length") + .get::<_, i64>(0); + u64::try_from(length).unwrap() +} + +fn pack_row(schema: &Schema, pack: i64) -> i64 { + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.pack WHERE pack_id=$1", + schema.selection.schema + ), + &[&pack], + ) + .expect("pack row") + .get(0) +} + +fn branch_row(schema: &Schema, id: BranchId) -> i64 { + let bytes = id.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.branch WHERE branch_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("branch row") + .get(0) +} + +fn stack_row(schema: &Schema, id: LayerStackId) -> i64 { + let bytes = id.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.layer_stack WHERE layer_stack_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("stack row") + .get(0) +} + +fn retired_branches(schema: &Schema) -> Vec { + let rows = schema + .client() + .query( + &format!( + "SELECT branch_id FROM {}.branch_retired ORDER BY branch_id", + schema.selection.schema + ), + &[], + ) + .expect("retired branches"); + rows.iter() + .map(|row| { + let bytes: Vec = row.get(0); + BranchId::from_slice(&bytes).unwrap() + }) + .collect() +} + +fn retired_stacks(schema: &Schema) -> i64 { + schema.count("layer_stack_retired") +} + +fn digest_rows(schema: &Schema, digest: &[u8; 32]) -> i64 { + schema + .client() + .query_one( + &format!( + "SELECT count(*) FROM {}.file_digest WHERE digest=$1", + schema.selection.schema + ), + &[&digest.as_slice()], + ) + .expect("digest rows") + .get(0) +} + +fn epoch(schema: &Schema, stack: LayerStackId) -> i64 { + let bytes = stack.as_slice(); + schema + .client() + .query_one( + &format!( + "SELECT epoch FROM {}.project_epoch WHERE layer_stack_id=$1", + schema.selection.schema + ), + &[&bytes], + ) + .expect("project epoch") + .get(0) +} + +fn epoch_rows(schema: &Schema) -> i64 { + schema.count("project_epoch") +} + +fn next_serial(schema: &Schema) -> i64 { + schema.scalar("SELECT next_history_serial FROM {s}.history_meta WHERE id=1") +} + +fn inode_block_bytes(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(octet_length(body)),0) FROM {s}.inode_value_block"), + ) + .unwrap() +} + +fn sum_pack_lengths(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack")).unwrap() +} + +fn sum_canonical(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"), + ) + .unwrap() +} + +fn sum_metadata_packs(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"), + ) + .unwrap() +} + +fn sum_history_logical(schema: &Schema) -> u64 { + u64::try_from( + schema.scalar("SELECT COALESCE(SUM(logical),0)::bigint FROM {s}.\"commit\"") + + schema.scalar("SELECT COALESCE(SUM(logical),0)::bigint FROM {s}.layer"), + ) + .unwrap() +} + +fn generation(handles: &PostgresHandles) -> u64 { + handles.history.incarnation() +} + +fn db_generation(schema: &Schema) -> u64 { + u64::try_from(schema.scalar("SELECT catalog_incarnation FROM {s}.history_meta WHERE id=1")) + .unwrap() +} + +fn located(handles: &PostgresHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn audit(handles: &PostgresHandles) -> Vec { + let mut out = Vec::new(); + handles + .diagnose( + Options { + quick: false, + ..Options::default() + }, + &mut |observation| out.push(observation), + ) + .expect("diagnose"); + out +} + +fn assert_doctor_clean(handles: &PostgresHandles) { + let out = audit(handles); + assert!( + !out.iter() + .any(|observation| matches!(observation.outcome, Outcome::Fail | Outcome::Unknown)), + "the compacted Store must pass the full doctor: {out:?}" + ); +} + +/// §9/§8: after a settle no op row, dead set, intent or scratch schema remains +/// and every `store_totals` row equals the sum of the rows it summarises. +fn assert_settled(schema: &Schema, op: u8) { + assert_eq!(schema.count("gc_op"), 0, "gc_op is cleared on settle"); + assert_eq!(schema.count("gc_dead"), 0, "gc_dead is cleared on settle"); + assert_eq!( + schema.count_where("SELECT count(*) FROM {s}.gc_intent WHERE kind='compaction'"), + 0, + "the compaction intent is released" + ); + assert!( + !schema.scratch_exists(op), + "the recorded scratch schema is dropped" + ); + let logical = schema.scalar("SELECT logical FROM {s}.store_totals WHERE id=1"); + let sum = schema.scalar( + "SELECT (SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.\"commit\")+(SELECT COALESCE(SUM(logical)::bigint,0) FROM {s}.layer)", + ); + assert_eq!(logical, sum, "store_totals.logical tracks the history rows"); + let canonical = schema.scalar("SELECT canonical FROM {s}.store_totals WHERE id=1"); + let canonical_sum = + schema.scalar("SELECT COALESCE(SUM(canonical_length)::bigint,0) FROM {s}.object_location"); + assert_eq!(canonical, canonical_sum, "store_totals.canonical"); + let stored = schema.scalar("SELECT stored FROM {s}.store_totals WHERE id=1"); + let stored_sum = schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack"); + assert_eq!(stored, stored_sum, "store_totals.stored"); + let metadata = schema.scalar("SELECT metadata FROM {s}.store_totals WHERE id=1"); + let metadata_sum = + schema.scalar("SELECT COALESCE(SUM(length)::bigint,0) FROM {s}.pack WHERE domain=0"); + assert_eq!(metadata, metadata_sum, "store_totals.metadata"); +} + +/// AC-43: the Store is usable at `G + 1` with its totals, the recreated +/// `pack_immutable` trigger and function, the same owner and cleared hints. +fn assert_phase_c_settled(schema: &Schema, op: u8) { + assert_settled(schema, op); + let trigger = schema.scalar( + "SELECT count(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='{s}' AND t.tgname='pack_immutable' AND NOT t.tgisinternal", + ); + assert_eq!(trigger, 1, "the swap recreates pack_immutable"); + let function = schema.scalar( + "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='{s}' AND p.proname='pack_immutable'", + ); + assert_eq!(function, 1, "the trigger is bound to the schema's function"); + let mut client = schema.client(); + let refused = client.batch_execute(&format!( + "UPDATE {schema}.pack SET length=length WHERE pack_id=(SELECT min(pack_id) FROM {schema}.pack)", + schema = schema.selection.schema + )); + assert!(refused.is_err(), "the trigger still refuses an update"); + let owner: String = client + .query_one( + &format!( + "SELECT tableowner FROM pg_tables WHERE schemaname='{}' AND tablename='pack'", + schema.selection.schema + ), + &[], + ) + .expect("owner") + .get(0); + let current: String = client + .query_one("SELECT current_user", &[]) + .expect("current user") + .get(0); + assert_eq!(owner, current, "the swapped table keeps its owner"); + assert_eq!( + schema.count("binding_hint_block"), + 0, + "Phase C clears the binding hints" + ); + assert_eq!( + schema.scalar( + "SELECT count(*) FROM information_schema.tables WHERE table_schema='{s}' AND table_name='inode'" + ), + 0, + "no inode column on PostgreSQL" + ); +} + +// ---- cases ---------------------------------------------------------------- + +/// The §6.4 vector: two live stacks, an absent head and a retirement, exactly +/// the T12.1 SQLite fixture. One literal must hold on both backends. +#[test] +fn the_live_heads_digest_vector_is_the_sqlite_literal() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "digest"); + // Out-of-order creation: the digest orders by identity, not creation. + let root_9 = tree(&f, whole_file(&f, b"digest stack nine content"), 9); + let stack_9 = initialize(&f.handles, 9, root_9); + let root_3 = tree(&f, whole_file(&f, b"digest stack three content"), 10); + let stack_3 = initialize(&f.handles, 3, root_3); + let root_7 = tree(&f, whole_file(&f, b"digest retired stack content"), 11); + let stack_7 = initialize(&f.handles, 7, root_7); + f.handles + .history + .mark_stack_deleted(stack_7.id, 7_000) + .unwrap(); + + let branch_9 = support::branch_id(9); + fork(&f.handles, stack_9.id, branch_9, 9); + let branch_3 = support::branch_id(3); + fork(&f.handles, stack_3.id, branch_3, 3); + let branch_5 = support::branch_id(5); + fork(&f.handles, stack_9.id, branch_5, 5); + let commit_root = tree(&f, whole_file(&f, b"digest commit content"), 12); + let commit_id = commit(&f.handles, branch_3, commit_root, 90); + assert_eq!( + hex(commit_id.as_slice()), + sqlite_commit_literal(), + "the same content-addressed Commit on both backends" + ); + + // branch_5 has never committed: an absent head is hashed as 33 zero bytes. + let with_empty_head = f.handles.history.mark().unwrap().live_heads_digest(); + assert_eq!( + with_empty_head, + f.handles.history.mark().unwrap().live_heads_digest(), + "the fixed fixture pins one digest" + ); + assert_eq!( + hex(&with_empty_head), + sqlite_literal("absentHeadDigest"), + "an absent head is the SQLite golden literal" + ); + // A retired Branch is absent, not zeroed. + f.handles + .history + .mark_branch_removed(branch_5, 5_000) + .unwrap(); + let retired_branch = f.handles.history.mark().unwrap().live_heads_digest(); + assert_ne!(with_empty_head, retired_branch); + assert_eq!( + hex(&retired_branch), + sqlite_literal("retiredBranchDigest"), + "a retired Branch row is absent, not zeroed" + ); + assert_doctor_clean(&f.handles); + assert_eq!(retired_stacks(&f.schema), 1); +} + +struct FixtureIds { + stacks_live: Vec, + stacks_collected: Vec, + branches_live: Vec, + branches_retained_retired: Vec, + branches_collected: Vec, + commits_retained: Vec, + commits_collected: Vec, + layers_retained: Vec, + layers_collected: Vec, + objects_live: Vec, + objects_dead: Vec, + delta_bases_sorted: Vec, + delta_chain: Vec<(ObjectId, i64)>, + delta_target: ObjectId, + live_group: ObjectId, + dead_group: ObjectId, + shared_pack: i64, + shared_pack_length: u64, + shared_dead_group: u64, + digest_collected: [u8; 32], + digest_retained: [u8; 32], +} + +/// The T12.1 rebuild fixture, backend-for-backend: retained history with a +/// deleted project, a retained retired Branch, a collected retired Branch, a +/// depth-2 delta chain held only as bases, one retained pack with an unreached +/// group, pooled inode values and the two digest roots. +fn rebuild_fixture(f: &Fixture) -> FixtureIds { + // The dead packs: unreferenced whole files, each in its own pack. + let mut objects_dead = Vec::new(); + for index in 0..12u8 { + let payload = pseudo_bytes(4096, 0xA0 + index); + objects_dead.push(whole_file(f, &payload)); + } + let retained_chunked_payload = pseudo_bytes(131_072, 0x77); + let retained_chunked = chunked_file(f, &retained_chunked_payload); + let collected_chunked_payload = pseudo_bytes(131_072, 0x88); + let collected_chunked = chunked_file(f, &collected_chunked_payload); + let digest_collected = *blake3::hash(&collected_chunked_payload).as_bytes(); + let digest_retained = *blake3::hash(&retained_chunked_payload).as_bytes(); + + // A depth-2 delta chain: base2 <- base1 <- target. Only the target is a + // retained root, so both bases are kept only as delta bases. + let base2_payload = pseudo_bytes(512, 0x11); + let base1_payload = [base2_payload.as_slice(), b" v2"].concat(); + let target_payload = [base1_payload.as_slice(), b" v3"].concat(); + let base2 = whole_file(f, &base2_payload); + let base2_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base2_payload).unwrap(), + ) + .unwrap(); + let base1_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&base1_payload).unwrap(), + ) + .unwrap(); + prefix_record(f, &base1_object, &base2_object, base2); + let base1 = base1_object.id(); + let target_object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&target_payload).unwrap(), + ) + .unwrap(); + prefix_record(f, &target_object, &base1_object, base1); + let delta_target = target_object.id(); + + // One retained pack holding one reached and one unreached record. + let live_payload = pseudo_bytes(256, 0x55); + let dead_payload = pseudo_bytes(256, 0x66); + let live_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&live_payload).unwrap(), + ) + .unwrap(); + let dead_group = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(&dead_payload).unwrap(), + ) + .unwrap(); + let (shared_pack, shared_body) = publish_groups(f, &[live_group.clone(), dead_group.clone()]); + let shared_header = layout::parse_header(&shared_body).unwrap(); + let dead_view = layout::group_view(&shared_body, shared_header, 1).unwrap(); + let shared_dead_group = (dead_view.end - dead_view.start) as u64; + let shared_pack_length = pack_length(&f.schema, shared_pack); + + // Content objects of the trees that are expected to be collected. + let collected_content_a4 = collected_chunked; + let collected_content_b0 = whole_file(f, b"deleted project genesis content"); + let collected_content_b1 = whole_file(f, b"deleted project commit content"); + let content_c3 = whole_file(f, b"live branch commit content"); + + let root_a0 = tree(f, live_group.id(), 0); + let root_a1 = tree(f, delta_target, 1); + let root_a2 = tree(f, retained_chunked, 2); + let root_a3 = tree(f, content_c3, 3); + let root_a4 = tree(f, collected_content_a4, 4); + let root_b0 = tree(f, collected_content_b0, 5); + let root_b1 = tree(f, collected_content_b1, 6); + + let stack_a = initialize(&f.handles, 1, root_a0); + let branch_published = support::branch_id(1); + fork(&f.handles, stack_a.id, branch_published, 1); + let commit_1 = commit(&f.handles, branch_published, root_a1, 120); + let layer_1 = publish(&f.handles, stack_a.id, branch_published); + let commit_2 = commit(&f.handles, branch_published, root_a2, 240); + let layer_2 = publish(&f.handles, stack_a.id, branch_published); + + let branch_live = support::branch_id(2); + fork(&f.handles, stack_a.id, branch_live, 2); + let commit_3 = commit(&f.handles, branch_live, root_a3, 360); + + let branch_retired = support::branch_id(3); + fork(&f.handles, stack_a.id, branch_retired, 3); + let commit_4 = commit(&f.handles, branch_retired, root_a4, 70); + + let stack_b = initialize(&f.handles, 2, root_b0); + let branch_b = support::branch_id(4); + fork(&f.handles, stack_b.id, branch_b, 4); + let commit_5 = commit(&f.handles, branch_b, root_b1, 40); + let layer_b = publish(&f.handles, stack_b.id, branch_b); + + // A retired Branch a retained Layer still names keeps its row and marker. + f.handles + .history + .mark_branch_removed(branch_published, 1_000) + .unwrap(); + // A retired Branch nothing retained reaches is collected whole. + f.handles + .history + .mark_branch_removed(branch_retired, 1_500) + .unwrap(); + // A deleted project goes whole, marker included. + f.handles + .history + .mark_stack_deleted(stack_b.id, 2_000) + .unwrap(); + + // AC-18: one digest root no retained root reaches, one it does. + f.handles + .history + .record_file_digests(&[ + FileDigest { + root: *collected_chunked.as_bytes(), + length: collected_chunked_payload.len() as u64, + digest: digest_collected, + }, + FileDigest { + root: *retained_chunked.as_bytes(), + length: retained_chunked_payload.len() as u64, + digest: digest_retained, + }, + ]) + .unwrap(); + + let delta_chain: Vec<(ObjectId, i64)> = [base2, base1, delta_target] + .iter() + .map(|id| (*id, pack_of(&f.schema, *id).expect("a delta-chain pack"))) + .collect(); + let mut delta_bases_sorted = vec![base1, base2]; + delta_bases_sorted.sort(); + objects_dead.push(dead_group.id()); + objects_dead.extend([ + root_a4, + root_b0, + root_b1, + collected_content_a4, + collected_content_b0, + collected_content_b1, + ]); + let objects_live = vec![ + root_a0, + root_a1, + root_a2, + root_a3, + live_group.id(), + retained_chunked, + base2, + base1, + delta_target, + content_c3, + ]; + FixtureIds { + stacks_live: vec![stack_a.id], + stacks_collected: vec![stack_b.id], + branches_live: vec![branch_live], + branches_retained_retired: vec![branch_published], + branches_collected: vec![branch_retired, branch_b], + commits_retained: vec![commit_1, commit_2, commit_3], + commits_collected: vec![commit_4, commit_5], + layers_retained: vec![stack_a.head_layer, layer_1.id, layer_2.id], + layers_collected: vec![stack_b.head_layer, layer_b.id], + objects_live, + objects_dead, + delta_bases_sorted, + delta_chain, + delta_target, + live_group: live_group.id(), + dead_group: dead_group.id(), + shared_pack, + shared_pack_length, + shared_dead_group, + digest_collected, + digest_retained, + } +} + +// ---- tests ---------------------------------------------------------------- + +/// AC-02, AC-05, AC-17, AC-18, AC-25, AC-40, AC-41, AC-43, AC-52: the whole +/// T12.1 rebuild fixture on PostgreSQL. The `live_heads_digest` is the SQLite +/// golden literal; the §11 report is non-trivial; the depth-2 chain is kept +/// and its packs keep their ids and lengths; markers and digest rows follow +/// the retained set; the settled Store passes the full doctor. +#[test] +fn the_rebuild_report_and_markers_hold_on_postgresql() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "rebuild"); + let ids = rebuild_fixture(&f); + let schema = &f.schema; + let before = totals(schema); + let generation_before = generation(&f.handles); + let serial_before = next_serial(schema); + assert_eq!(epoch_rows(schema), 2, "one project_epoch row per stack"); + let epoch_before = epoch(schema, ids.stacks_live[0]); + + let mark = f.handles.history.mark().unwrap(); + assert_eq!( + mark.live_heads_digest(), + f.handles.history.mark().unwrap().live_heads_digest(), + "the digest is stable across marks" + ); + let literals = sqlite_literals("liveHeadsDigest"); + assert_eq!(literals.len(), 3, "the SQLite golden holds three digests"); + assert_eq!( + hex(&mark.live_heads_digest()), + literals[2], + "the rebuild digest is the SQLite literal: one vector on both backends" + ); + let live_packs = mark.objects().packs().to_vec(); + let live_lengths: Vec<(i64, u64)> = live_packs + .iter() + .map(|pack| (*pack, pack_length(schema, *pack))) + .collect(); + let facts = pack_facts(schema, &live_packs); + let inode_blocks = inode_block_bytes(schema); + assert_eq!(mark.objects().waste().inode_blocks_bytes(), inode_blocks); + assert_eq!( + mark.objects().waste().dead_in_retained_packs_bytes(), + ids.shared_dead_group, + "the unreached group of the retained pack is keptWaste" + ); + assert_eq!( + mark.objects().waste().delta_base_references(), + ids.delta_bases_sorted.as_slice(), + "a depth-2 chain reports both bases, in identity order" + ); + let expected_bases: Vec<[u8; 32]> = ids + .delta_bases_sorted + .iter() + .map(|id| *id.as_bytes()) + .collect(); + assert!( + mark.objects().waste().delta_bases_bytes() > 0, + "the bases are kept only as bases" + ); + for id in &ids.objects_live { + assert!(mark.objects().contains(*id), "live object {id} is retained"); + } + for id in &ids.objects_dead { + assert!(!mark.objects().contains(*id), "object {id} is collected"); + } + assert!( + facts.dead_bytes * 10 >= facts.total, + "the fixture must take the rebuild path: {} dead of {}", + facts.dead_bytes, + facts.total + ); + + let receipt = f.handles.compact(&request(0x72)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, generation_before); + assert_eq!(receipt.generation_after, generation_before + 1); + assert_eq!(receipt.dead_bytes_bound, facts.dead_bytes); + assert_eq!(receipt.dead_bytes_refined, receipt.dead_bytes_bound); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + assert!(receipt.estimate); + assert_eq!( + receipt.kept_waste.dead_in_retained_packs_bytes, + ids.shared_dead_group + ); + assert_eq!(receipt.kept_waste.inode_blocks_bytes, inode_blocks); + assert_eq!( + receipt.kept_waste.delta_base_references, expected_bases, + "PG keeps the delta bases and reports them" + ); + assert!(receipt.kept_waste.delta_bases_bytes > 0); + assert_eq!( + receipt.residue.bytes, 0, + "a successful settle leaves no residue" + ); + assert!(receipt.residue.artifacts.is_empty()); + assert!(receipt.residue.reason.is_none()); + assert_eq!( + receipt.backend_reusable, 0, + "PostgreSQL rewrites in place: no reusable free pages" + ); + assert!(receipt + .zero_reasons + .iter() + .any(|zero| zero.field == "backendReusable" && zero.reason == "postgresql-in-place")); + + let reopened = schema.open(); + assert_eq!(generation(&reopened), generation_before + 1); + let after = totals(schema); + assert_eq!( + receipt.logical_discarded, + before.logical - after.logical, + "logicalDiscarded is the history logical delta" + ); + assert_eq!(before.logical - after.logical, 160); + assert_eq!(after.stored, sum_pack_lengths(schema)); + assert_eq!(after.canonical, sum_canonical(schema)); + assert_eq!(after.metadata, sum_metadata_packs(schema)); + assert_eq!(after.logical, sum_history_logical(schema)); + + // Live objects are readable, including the delta target through its chain. + let storage = Storage::new(reopened.storage.clone()).unwrap(); + let reader = storage.reader().unwrap(); + let canonicals = reader + .read_objects(&[ids.delta_target, ids.live_group]) + .unwrap(); + assert_eq!(canonicals.len(), 2); + assert!(!canonicals[0].is_empty()); + assert!(!canonicals[1].is_empty()); + for id in &ids.objects_live { + assert!(located(&reopened, *id), "live object {id} survives"); + } + for id in &ids.objects_dead { + assert!(!located(&reopened, *id), "dead object {id} is gone"); + } + for pack in &facts.dead { + assert_eq!(pack_row(schema, *pack), 0, "dead pack {pack} is removed"); + } + for (pack, length) in &live_lengths { + assert_eq!(pack_row(schema, *pack), 1, "live pack {pack} stays"); + assert_eq!( + pack_length(schema, *pack), + *length, + "AC-41: the swap copies the row and moves no pack bytes" + ); + } + // The retained pack keeps the unreached group: repacking is out of 2.0. + assert_eq!(pack_of(schema, ids.live_group), Some(ids.shared_pack)); + assert_eq!(object_count(schema, ids.live_group), 1); + assert_eq!(object_count(schema, ids.dead_group), 0); + assert_eq!(pack_length(schema, ids.shared_pack), ids.shared_pack_length); + // AC-17: a copied pack keeps its identity. + for (id, pack) in &ids.delta_chain { + assert_eq!( + pack_of(schema, *id), + Some(*pack), + "pack_id of {id} is preserved" + ); + } + + for id in &ids.commits_retained { + assert!( + reopened.history.commit(*id).unwrap().is_some(), + "commit {id}" + ); + } + for id in &ids.commits_collected { + assert!( + reopened.history.commit(*id).unwrap().is_none(), + "commit {id}" + ); + } + for id in &ids.layers_retained { + assert!(reopened.history.layer(*id).unwrap().is_some(), "layer {id}"); + } + for id in &ids.layers_collected { + assert!(reopened.history.layer(*id).unwrap().is_none(), "layer {id}"); + } + for id in &ids.stacks_live { + assert!(reopened.history.layer_stack(*id).unwrap().is_some()); + assert_eq!(stack_row(schema, *id), 1); + } + for id in &ids.stacks_collected { + assert!(reopened.history.layer_stack(*id).unwrap().is_none()); + assert_eq!(stack_row(schema, *id), 0); + } + for id in &ids.branches_live { + assert!(reopened.history.branch(*id).unwrap().is_some()); + } + assert_eq!(retired_branches(schema), ids.branches_retained_retired); + for id in &ids.branches_collected { + assert_eq!(branch_row(schema, *id), 0, "collected branch row {id}"); + } + assert_eq!(retired_stacks(schema), 0); + + // AC-18: a digest row survives only when a retained root reaches the root. + assert_eq!(digest_rows(schema, &ids.digest_retained), 1); + assert_eq!(digest_rows(schema, &ids.digest_collected), 0); + + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + mark.live_heads_digest(), + "the live-heads digest matches before and after" + ); + // AC-24/AC-39: the rebuild copies the epoch, never bumps it, and never + // lowers the serial counter. + assert_eq!(epoch(schema, ids.stacks_live[0]), epoch_before); + assert_eq!( + epoch_rows(schema), + 1, + "the collected project takes its epoch" + ); + assert!(next_serial(schema) >= serial_before); + // AC-43/AC-02: the settled PostgreSQL Store is usable at G + 1. + assert_phase_c_settled(schema, 0x72); +} + +/// AC-28, AC-43, AC-49 (Phase C arm): the swap is durably at `G + 1` when the +/// reply is lost. A save made after the swap is not a member of the persisted +/// dead set, so the resumed cleanup keeps it, and the settled Store passes the +/// full doctor. +#[test] +fn a_save_after_the_swap_survives_the_resumed_cleanup() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "late"); + let live = tree(&f, whole_file(&f, b"late live content"), 1); + let stack = initialize(&f.handles, 1, live); + let dead = saved_whole_file(&f.handles, b"a pack Phase C collects"); + let before = generation(&f.handles); + let error = f + .handles + .compact(&request_at(0x31, CompactFailpoint::PostgresAfterSwap)) + .unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome)); + let schema = &f.schema; + assert_eq!(db_generation(schema), before + 1); + assert_eq!(schema.count("gc_op"), 1, "the op still names Phase C"); + let phase: String = schema + .client() + .query_one( + &format!( + "SELECT phase FROM {}.gc_op WHERE id=1", + schema.selection.schema + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(phase, "swap", "AC-51: the op records its §8 phase"); + assert!( + schema.scalar("SELECT count(*) FROM {s}.gc_dead") > 0, + "the persisted dead set waits for Phase C" + ); + assert!( + !located(&f.handles, dead), + "the swap dropped the dead pack already" + ); + assert!( + schema.scalar("SELECT pid FROM {s}.gc_op WHERE id=1") > 0, + "the op records its owner backend" + ); + + // AC-28: a publication after the swap is not in `gc_dead`. + let late = saved_whole_file(&f.handles, b"written between swap and cleanup"); + assert!(located(&f.handles, late)); + let late_pack = pack_of(schema, late).expect("a late pack"); + let receipt = f.handles.compact(&request(0x32)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + assert!( + located(&f.handles, late), + "the late write survives the high-water bound" + ); + assert_eq!(pack_row(schema, late_pack), 1); + assert!(!located(&f.handles, dead)); + assert_phase_c_settled(schema, 0x31); + + let reopened = schema.open(); + assert_doctor_clean(&reopened); + assert!(located(&reopened, late)); + assert!(!located(&reopened, dead)); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); +} + +/// AC-21, AC-24, AC-39: the PostgreSQL `project_epoch` fence. Commits and +/// Layers leave it alone; a marker write bumps it once; a repeated marker is +/// `UpToDate` and does not bump it; compact copies it; a missing row refuses +/// every writer with `Conflict` and writes nothing. +#[test] +fn a_marker_write_bumps_the_epoch_and_the_fence_refuses_a_racing_write() { + let Some(url) = url() else { return }; + let f = Fixture::new(&url, "epoch"); + let root = tree(&f, whole_file(&f, b"epoch genesis content"), 1); + let stack = initialize(&f.handles, 1, root); + let branch = support::branch_id(1); + fork(&f.handles, stack.id, branch, 1); + let commit_root = tree(&f, whole_file(&f, b"epoch commit content"), 2); + let _first = commit(&f.handles, branch, commit_root, 50); + let _layer = publish(&f.handles, stack.id, branch); + assert_eq!( + epoch(&f.schema, stack.id), + 0, + "a Commit or a published Layer does not bump the epoch" + ); + assert_eq!(epoch_rows(&f.schema), 1); + + // Capture the request while the Branch is still visible, then let the + // removal win: the fence refuses and leaves no Commit behind. + let candidate = whole_file(&f, b"epoch racing commit content"); + let stale = commit_request(&f.handles, branch, candidate); + let commits_before = f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""); + f.handles + .history + .mark_branch_removed(branch, 1_000) + .unwrap(); + assert_eq!( + epoch(&f.schema, stack.id), + 1, + "the marker bumped the epoch once" + ); + let refused = f.handles.history.commit_branch(&stale).unwrap_err(); + assert!( + matches!( + refused, + HistoryError::Missing(Missing::Branch) | HistoryError::Conflict("project epoch") + ), + "{refused:?}" + ); + assert_eq!( + f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""), + commits_before, + "the refused write left no Commit" + ); + + // A live project's marker write bumps it again; a repeat is not a write. + let removal = || PathRemovalRequest { + stack: stack.id, + path: NormalizedPath::new("file-1").unwrap().as_bytes().to_vec(), + removed_ms: 2_000, + }; + let outcome = f.handles.history.remove_path(&removal()).unwrap(); + assert_eq!(outcome.kind, PathRemovalKind::Removed); + assert_eq!(epoch(&f.schema, stack.id), 2); + let repeat = f.handles.history.remove_path(&removal()).unwrap(); + assert_eq!(repeat.kind, PathRemovalKind::UpToDate); + assert_eq!( + epoch(&f.schema, stack.id), + 2, + "UpToDate is not a marker write" + ); + + // AC-24/AC-39: compaction copies the epoch and never lowers the serial. + let serial = next_serial(&f.schema); + let receipt = f.handles.compact(&request(0x51)).unwrap(); + assert_eq!( + receipt.path, + CompactPath::None, + "a live Store takes the none path" + ); + assert_eq!(epoch(&f.schema, stack.id), 2); + assert_eq!(epoch_rows(&f.schema), 1); + assert!(next_serial(&f.schema) >= serial); + assert_settled(&f.schema, 0x51); + + // No epoch row: every writer refuses `Conflict`, never "unchanged". + let branch_2 = support::branch_id(2); + fork(&f.handles, stack.id, branch_2, 2); + f.schema + .client() + .batch_execute(&format!( + "DELETE FROM {}.project_epoch WHERE layer_stack_id=decode('{}','hex')", + f.schema.selection.schema, + hex(stack.id.as_slice()) + )) + .unwrap(); + let candidate = whole_file(&f, b"epoch missing row content"); + let stale = commit_request(&f.handles, branch_2, candidate); + let before = f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""); + let refused = f.handles.history.commit_branch(&stale).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!( + f.schema.count_where("SELECT count(*) FROM {s}.\"commit\""), + before, + "the missing fence row wrote nothing" + ); +} + +/// AC-01, AC-03, AC-28, AC-43: the PostgreSQL failpoint matrix. Each named +/// §8 point aborts the round, then the next `store.compact` resumes it: `mark` +/// and `build` leave the old Store at `G` and re-mark, `swap` leaves the new +/// Store at `G + 1` and finishes Phase C, `cleanup` leaves nothing to resume. +/// Every arm settles with the same heads digest and a clean doctor. +#[test] +fn every_postgresql_failpoint_leaves_a_resumable_leftover() { + let Some(url) = url() else { return }; + for (tag, point, op, phase, scratch) in [ + ( + "cmark", + CompactFailpoint::PostgresAfterMark, + 0x61u8, + Some("mark"), + false, + ), + ( + "cbuild", + CompactFailpoint::PostgresAfterBuild, + 0x62, + Some("build"), + true, + ), + ( + "cswap", + CompactFailpoint::PostgresAfterSwap, + 0x63, + Some("swap"), + false, + ), + ( + "ccleanup", + CompactFailpoint::PostgresAfterCleanup, + 0x64, + None, + false, + ), + ] { + let f = Fixture::new(&url, tag); + let live = tree(&f, whole_file(&f, b"crash matrix live content"), 1); + let stack = initialize(&f.handles, 1, live); + let dead = saved_whole_file(&f.handles, b"crash matrix dead content"); + let before = generation(&f.handles); + let digest = f.handles.history.mark().unwrap().live_heads_digest(); + let error = f.handles.compact(&request_at(op, point)).unwrap_err(); + assert!( + matches!(error, HistoryError::UnknownOutcome), + "{tag}: {error:?}" + ); + let swapped = matches!( + point, + CompactFailpoint::PostgresAfterSwap | CompactFailpoint::PostgresAfterCleanup + ); + let expected = if swapped { before + 1 } else { before }; + assert_eq!(db_generation(&f.schema), expected, "{tag}: generation"); + assert_eq!( + located(&f.handles, dead), + !swapped, + "{tag}: the dead pack moves only at the swap" + ); + assert!(located(&f.handles, live), "{tag}: the live object stays"); + assert_eq!( + f.schema.scratch_exists(op), + scratch, + "{tag}: the scratch schema state" + ); + if let Some(phase) = phase { + let recorded: String = f + .schema + .client() + .query_one( + &format!( + "SELECT phase FROM {}.gc_op WHERE id=1", + f.schema.selection.schema + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(recorded, phase, "{tag}: the op records its phase"); + } + // The next call either adopts the recorded round or starts clean. + let receipt = f.handles.compact(&request(op + 1)).unwrap(); + assert_eq!(receipt.generation_after, before + 1, "{tag}: installed"); + if point == CompactFailpoint::PostgresAfterCleanup { + assert_eq!( + receipt.path, + CompactPath::None, + "{tag}: the lost cleanup reply left a clean Store" + ); + } else { + assert_eq!( + receipt.path, + CompactPath::Rebuild, + "{tag}: the adopted round rebuilds" + ); + } + assert!(!located(&f.handles, dead), "{tag}: collected"); + assert!(located(&f.handles, live), "{tag}: retained"); + assert_settled(&f.schema, op); + let reopened = f.schema.open(); + assert_doctor_clean(&reopened); + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + digest, + "{tag}: the heads digest is unchanged" + ); + assert!(reopened.history.layer_stack(stack.id).unwrap().is_some()); + } +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs b/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs index c0fae1e0fe..49cdb04151 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_diagnostic.rs @@ -13,6 +13,9 @@ use std::{ time::{Duration, Instant}, }; +#[path = "pg_doctor/conninfo.rs"] +mod conninfo; + #[test] fn tcp_peer_that_stalls_authentication_is_bounded_and_closed() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); @@ -152,10 +155,17 @@ fn deadline_driver_reads_one_view_and_ordinary_sync_writes_still_work() { "{connection:?}" ); drop(writer); - let fallback = PostgresSelection::new( - url.replace("127.0.0.1:", "missing.invalid,127.0.0.1:"), - &schema, + let fallback_url = conninfo::with_failing_first_host(&url); + assert_ne!( + fallback_url, url, + "failover fixture must actually rewrite the conninfo host" ); + let fallback = PostgresSelection::new(&fallback_url, &schema); + let mut attempted = Vec::new(); + let mut recording = |host: &str, port: u16, until: Instant| { + attempted.push((host.to_owned(), port)); + fixture_resolver(host, port, until) + }; let mut fallback_observations = Vec::new(); diagnostic::postgresql_connection( &fallback, @@ -164,10 +174,14 @@ fn deadline_driver_reads_one_view_and_ordinary_sync_writes_still_work() { deadline: Some(Instant::now() + Duration::from_secs(3)), ..Options::default() }, - &mut fixture_resolver, + &mut recording, &mut |o| fallback_observations.push(o), ) .unwrap(); + assert!( + attempted.iter().any(|(host, _)| host == "missing.invalid"), + "failover case must first attempt the unresolvable host: {attempted:?}" + ); assert!( fallback_observations .iter() @@ -188,12 +202,9 @@ fn lock_wait_cancellation_releases_backend_work_and_snapshot() { }; let schema = format!("doctor_cancel_{}", std::process::id()); let application = format!("doctor_cancel_{}", std::process::id()); - // URL syntax is not reserialized or logged; an explicit startup option - // marks the diagnostic session for independent backend observation. - let diagnostic_url = format!( - "{url}{}application_name={application}", - if url.contains('?') { "&" } else { "?" } - ); + // The connection string is not reserialized or logged; an explicit startup + // option marks the diagnostic session for independent backend observation. + let diagnostic_url = conninfo::with_option(&url, "application_name", &application); let selection = PostgresSelection::new(&diagnostic_url, &schema); let authority = HistoryCatalogConfig { binding_key: b"doctor-pg-cancel".to_vec(), @@ -250,12 +261,9 @@ fn cancellation_signal_without_deadline_releases_backend_work_and_snapshot() { }; let schema = format!("doctor_signal_{}", std::process::id()); let application = format!("doctor_signal_{}", std::process::id()); - // URL syntax is not reserialized or logged; an explicit startup option - // marks the diagnostic session for independent backend observation. - let diagnostic_url = format!( - "{url}{}application_name={application}", - if url.contains('?') { "&" } else { "?" } - ); + // The connection string is not reserialized or logged; an explicit startup + // option marks the diagnostic session for independent backend observation. + let diagnostic_url = conninfo::with_option(&url, "application_name", &application); let selection = PostgresSelection::new(&diagnostic_url, &schema); let authority = HistoryCatalogConfig { binding_key: b"doctor-pg-cancel".to_vec(), @@ -343,10 +351,7 @@ fn denied_signature_read_is_unknown_instead_of_corruption() { PostgresHandles::create(&selection, StoragePolicy::frozen_default(), &authority).unwrap(); let mut admin = postgres::Client::connect(&url, postgres::NoTls).unwrap(); admin.batch_execute(&format!("CREATE ROLE {role}; GRANT USAGE ON SCHEMA {schema} TO {role}; GRANT SELECT ON ALL TABLES IN SCHEMA {schema} TO {role}")).unwrap(); - let restricted_url = format!( - "{url}{}options=-c%20role%3D{role}", - if url.contains('?') { "&" } else { "?" } - ); + let restricted_url = conninfo::with_option(&url, "options", &format!("-c role={role}")); let restricted = PostgresSelection::new(&restricted_url, &schema); let reader = PostgresHandles::open_diagnostic( &restricted, diff --git a/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs b/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs index c7bdcbdf6b..70bca616a0 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_doctor_access.rs @@ -91,9 +91,7 @@ fn namespace_phase_transport_loss_is_unknown_and_releases_its_snapshot() { let before = f.fingerprint(); let mut selection = f.selection.clone(); let application = f.selection.schema.clone(); - selection - .url - .push_str(&format!("&application_name={application}")); + selection.url = conninfo::with_option(&selection.url, "application_name", &application); let reader = PostgresHandles::open_diagnostic( &selection, BINDING, @@ -230,9 +228,8 @@ fn configured_unix_then_tcp_fallback_preserves_saved_options() { "{out:?}" ); let mut random = f.selection.clone(); - random - .url - .push_str("&load_balance_hosts=random&host=127.0.0.1,127.0.0.1"); + random.url = conninfo::with_option(&random.url, "load_balance_hosts", "random"); + random.url = conninfo::with_option(&random.url, "host", "127.0.0.1,127.0.0.1"); let out = connection(&random, BINDING); assert!( has(&out, "store.connection.configuration", Outcome::Unknown), diff --git a/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs b/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs index 4b923ef872..4c2d3ac398 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_doctor_physical.rs @@ -32,7 +32,9 @@ fn supported_amcheck_reports_each_relation_and_missing_extension_is_unknown() { assert_eq!(actual, expected); drop(reader); assert_eq!(before, f.fingerprint()); - let missing = std::env::var("LAYERFS_TEST_POSTGRES_MISSING_URL").unwrap(); + let Some(missing) = auxiliary_url("LAYERFS_TEST_POSTGRES_MISSING_URL") else { + return; + }; let mut f = Fixture::at("no_extension", missing).unwrap(); let reader = f.reader(); let (result, out) = audit(&reader, false); @@ -107,10 +109,12 @@ fn denied_amcheck_and_non_btree_indexes_leave_explicit_incomplete_coverage() { } #[test] fn physical_standby_uses_supported_online_checks_without_primary_writes() { + let Some(url) = auxiliary_url("LAYERFS_TEST_POSTGRES_REPLICA_URL") else { + return; + }; let Some(f) = Fixture::new("replica") else { return; }; - let url = std::env::var("LAYERFS_TEST_POSTGRES_REPLICA_URL").unwrap(); let mut monitor = postgres::Client::connect(&url, postgres::NoTls).unwrap(); assert!(monitor .query_one("SELECT pg_is_in_recovery()", &[]) @@ -173,9 +177,7 @@ fn oversized_physical_payloads_are_rejected_before_client_materialization() { let port = config.get_ports()[0]; let proxy = proxy::Proxy::new(port); let selected = layerfs_persistence::PostgresSelection::new( - f.selection - .url - .replace(&format!(":{port}/"), &format!(":{}/", proxy.port)), + conninfo::with_local_proxy(&f.selection.url, proxy.port), &f.selection.schema, ); let reader = PostgresHandles::open_diagnostic( @@ -189,6 +191,10 @@ fn oversized_physical_payloads_are_rejected_before_client_materialization() { let (result, out) = audit(&reader, false); result.unwrap(); assert!(has(&out, "store.packs", Outcome::Fail), "{out:?}"); + assert!( + proxy.bytes() > 0, + "the diagnostic bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "one-megabyte corrupted body crossed wire: {} bytes", @@ -377,9 +383,9 @@ fn pinned_catalog_replacement_is_rejected_as_authority_change() { fn oversized_authority_and_saved_ddl_stay_on_server_during_admission() { use std::str::FromStr; for (tag,mutation) in [("binding","ALTER TABLE {s}.history_meta DROP CONSTRAINT history_meta_binding_key_check; UPDATE {s}.history_meta SET binding_key=decode(repeat('00',1048576),'hex')"),("source","UPDATE {s}.history_meta SET schema_source=repeat('x',1048576)"),("definition","UPDATE {s}.history_meta SET schema_definition=repeat('x',1048576)")]{ - let Some(mut f)=Fixture::new(tag)else{return};f.admin.batch_execute(&mutation.replace("{s}",&f.selection.schema)).unwrap();let before=f.fingerprint();let config=postgres::Config::from_str(&f.selection.url).unwrap();let port=config.get_ports()[0];let proxy=proxy::Proxy::new(port);let selection=layerfs_persistence::PostgresSelection::new(f.selection.url.replace(&format!(":{port}/"),&format!(":{}/",proxy.port)),&f.selection.schema); + let Some(mut f)=Fixture::new(tag)else{return};f.admin.batch_execute(&mutation.replace("{s}",&f.selection.schema)).unwrap();let before=f.fingerprint();let config=postgres::Config::from_str(&f.selection.url).unwrap();let port=config.get_ports()[0];let proxy=proxy::Proxy::new(port);let selection=layerfs_persistence::PostgresSelection::new(conninfo::with_local_proxy(&f.selection.url,proxy.port),&f.selection.schema); if tag=="binding"{let out=connection(&selection,BINDING);assert!(has(&out,"store.connection.identity",Outcome::Fail),"{out:?}");}else{assert!(PostgresHandles::open_diagnostic(&selection,BINDING,[71;32],Some(Instant::now()+Duration::from_secs(5)),&mut resolver).is_err());} - assert!(proxy.bytes()<512*1024,"oversized metadata crossed wire: {} bytes",proxy.bytes());drop(proxy);assert_eq!(before,f.fingerprint()); + assert!(proxy.bytes()>0,"oversized metadata bypassed the byte-counting proxy (direct connection?)");assert!(proxy.bytes()<512*1024,"oversized metadata crossed wire: {} bytes",proxy.bytes());drop(proxy);assert_eq!(before,f.fingerprint()); } } @@ -412,9 +418,7 @@ fn oversized_pack_descriptor_digest_is_rejected_before_wire_extraction() { .get_ports()[0]; let proxy = proxy::Proxy::new(port); let selected = layerfs_persistence::PostgresSelection::new( - f.selection - .url - .replace(&format!(":{port}/"), &format!(":{}/", proxy.port)), + conninfo::with_local_proxy(&f.selection.url, proxy.port), &f.selection.schema, ); let reader = PostgresHandles::open_diagnostic( @@ -432,6 +436,10 @@ fn oversized_pack_descriptor_digest_is_rejected_before_wire_extraction() { has(&out, "store.objects", Outcome::Fail), "locator must independently reject invalid pack descriptor: {out:?}" ); + assert!( + proxy.bytes() > 0, + "the pack descriptor bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "oversized digest crossed wire: {} bytes", @@ -467,15 +475,7 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { .get_ports()[0]; let proxied = |port: u16| { layerfs_persistence::PostgresSelection::new( - f.selection.url.replace( - &format!( - ":{}/", - postgres::Config::from_str(&f.selection.url) - .unwrap() - .get_ports()[0] - ), - &format!(":{port}/"), - ), + conninfo::with_local_proxy(&f.selection.url, port), &f.selection.schema, ) }; @@ -486,6 +486,10 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { has(&out, "store.connection.identity", Outcome::Fail), "{out:?}" ); + assert!( + proxy.bytes() > 0, + "the preflight scalar bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "corrupt scalar crossed preflight wire" @@ -501,6 +505,10 @@ fn altered_numeric_authority_types_do_not_materialize_large_text_values() { &mut resolver ) .is_err()); + assert!( + proxy.bytes() > 0, + "the admitted-open scalar bypassed the byte-counting proxy (direct connection?)" + ); assert!( proxy.bytes() < 512 * 1024, "corrupt scalar crossed admitted-open wire" diff --git a/engine/crates/layerfs-persistence/tests/postgresql_gc_fence.rs b/engine/crates/layerfs-persistence/tests/postgresql_gc_fence.rs new file mode 100644 index 0000000000..6595664205 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_gc_fence.rs @@ -0,0 +1,532 @@ +//! PostgreSQL runtime evidence for the GC admission and epoch paths that +//! shipped compile-only: T6's `gc_intent` admission and T6.5's writer-side +//! `project_epoch` fence, including the `SELECT ... FOR SHARE` re-check. +//! +//! The tests drive `PostgresHandles` through the public `HistoryCatalog` +//! methods. A trigger stands in for a removal marker that bumps the epoch +//! inside the writer's own transaction, which is exactly where the fence's +//! re-read runs; a deleted epoch row stands in for a compacted-away project. +//! +//! Needs `LAYERFS_TEST_POSTGRES_URL`; NOT_RUN without it. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_history::error::{HistoryError, Missing}; +use layerfs_history::records::{ + GcDeletion, GcIntent, GcIntentKind, GcScopeKind, PathRemovalKind, PathRemovalRequest, +}; +use layerfs_history::{CatalogId, HistoryCatalog}; +use support::pg::{ + commit, commit_request, counts, epoch, layer_request, merge_request, project, publish, url, + Schema, +}; +use support::root; + +const CURSOR: [u8; 32] = [78; 32]; + +/// Every test opens its own catalog identity, as a real Store does: the +/// PostgreSQL advisory keys are server-global, so two tests that shared a +/// binding could see each other's intents while running in parallel. +fn binding(tag: &str) -> Vec { + format!("postgresql-gc-fence-{tag}").into_bytes() +} + +fn op(tag: u8) -> [u8; 16] { + [tag; 16] +} + +fn mount( + stack: layerfs_history::LayerStackId, + branch: layerfs_history::BranchId, + tag: u8, +) -> GcIntent { + GcIntent::shared( + op(tag), + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("proj"), Some("work")) + .placed( + Some([tag; 32]), + Some(&format!("sandbox-{tag}")), + Some(&format!("machine-{tag}")), + ) + .owned(Some("runtime-a"), Some(&format!("boot-{tag}"))) +} + +fn init(stack: layerfs_history::LayerStackId, tag: u8) -> GcIntent { + GcIntent::shared( + op(tag), + GcIntentKind::Init, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("proj"), None) + .owned(Some("runtime-a"), Some(&format!("boot-{tag}"))) +} + +fn merge(stack: layerfs_history::LayerStackId, tag: u8) -> GcIntent { + GcIntent::shared( + op(tag), + GcIntentKind::Merge, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("proj"), Some("merged")) + .owned(Some("runtime-a"), Some(&format!("boot-{tag}"))) +} + +fn removal(stack: layerfs_history::LayerStackId, body: u8) -> GcIntent { + GcIntent::operation( + op(body), + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("proj"), None) +} + +fn path_request(stack: layerfs_history::LayerStackId) -> PathRemovalRequest { + PathRemovalRequest { + stack, + path: b"a.txt".to_vec(), + removed_ms: 1_000, + } +} + +fn hex16(bytes: &[u8; 16]) -> String { + let mut out = String::with_capacity(32); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +/// A trigger that bumps the project epoch before each inserted row, standing +/// in for a removal marker that wins the race inside the writer's transaction. +fn bump_trigger(admin: &mut postgres::Client, table: &str) { + admin + .batch_execute(&format!( + "CREATE OR REPLACE FUNCTION g6a_epoch_race() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN UPDATE project_epoch SET epoch=epoch+1 WHERE layer_stack_id=NEW.layer_stack_id; RETURN NEW; END $$; \ + CREATE TRIGGER g6a_epoch_race BEFORE INSERT ON \"{table}\" FOR EACH ROW \ + EXECUTE FUNCTION g6a_epoch_race();" + )) + .unwrap(); +} + +fn drop_trigger(admin: &mut postgres::Client) { + admin + .batch_execute( + "DROP TRIGGER IF EXISTS g6a_epoch_race ON \"commit\"; \ + DROP TRIGGER IF EXISTS g6a_epoch_race ON layer; \ + DROP FUNCTION IF EXISTS g6a_epoch_race();", + ) + .unwrap(); +} + +#[test] +fn a_normal_commit_publish_and_merge_leave_the_epoch_alone() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_normal"); + let binding = binding("fence_normal"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + + let first = commit(&handles.history, work, 5); + assert_eq!( + epoch(&mut schema.admin(), shop), + 0, + "a Commit does not bump the epoch" + ); + let layer = publish(&handles.history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!( + epoch(&mut schema.admin(), shop), + 0, + "a published Layer does not bump it" + ); + let merged = handles + .history + .merge_branch(&merge_request( + &handles.history, + shop, + work, + first.id, + genesis, + 60, + )) + .unwrap(); + assert_eq!(merged.branch.name.as_str(), "merged"); + assert_eq!( + epoch(&mut schema.admin(), shop), + 0, + "a merge does not bump it" + ); + assert_eq!(counts(&mut schema.admin()), (2, 2)); +} + +#[test] +fn a_missing_epoch_row_refuses_every_writer_with_conflict() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_missing"); + let binding = binding("fence_missing"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + let first = commit(&handles.history, work, 5); + let side_request = merge_request(&handles.history, shop, work, first.id, genesis, 61); + let before = counts(&mut schema.admin()); + + schema + .admin() + .execute( + "DELETE FROM project_epoch WHERE layer_stack_id=$1", + &[&shop.as_slice()], + ) + .unwrap(); + + let refused = handles + .history + .commit_branch(&commit_request(&handles.history, work, root(6))) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = handles + .history + .add_layer(&layer_request(&handles.history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + let refused = handles.history.merge_branch(&side_request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + + assert_eq!(counts(&mut schema.admin()), before, "no partial write"); + let snapshot = handles.history.branch_snapshot(work).unwrap().unwrap(); + assert_eq!(snapshot.branch.head_commit, Some(first.id)); + assert_eq!( + handles + .history + .layer_stack(shop) + .unwrap() + .unwrap() + .head_layer, + genesis + ); + assert!(handles.history.branch(merged_branch()).unwrap().is_none()); +} + +/// The Branch id `merge_request` creates for the merge. +fn merged_branch() -> layerfs_history::BranchId { + layerfs_history::BranchId::from_authority([9; 16]) +} + +#[test] +fn an_epoch_change_during_a_commit_is_detected_and_rolled_back() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_trigger_commit"); + let binding = binding("fence_trigger_commit"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, _genesis) = project(&handles.history, 1); + + bump_trigger(&mut schema.admin(), "commit"); + let refused = handles + .history + .commit_branch(&commit_request(&handles.history, work, root(5))) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&mut schema.admin()).0, 0, "the Commit rolled back"); + assert_eq!( + epoch(&mut schema.admin(), shop), + 0, + "the fake bump rolled back too" + ); + assert!(handles + .history + .branch(work) + .unwrap() + .unwrap() + .head_commit + .is_none()); + + drop_trigger(&mut schema.admin()); + let record = commit(&handles.history, work, 5); + assert_eq!(record.stack, shop); + assert_eq!(epoch(&mut schema.admin(), shop), 0); + assert_eq!( + handles.history.branch(work).unwrap().unwrap().head_commit, + Some(record.id) + ); +} + +#[test] +fn an_epoch_change_during_a_publish_is_detected_and_rolled_back() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_trigger_layer"); + let binding = binding("fence_trigger_layer"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + let _first = commit(&handles.history, work, 5); + let before = counts(&mut schema.admin()); + + bump_trigger(&mut schema.admin(), "layer"); + let refused = handles + .history + .add_layer(&layer_request(&handles.history, shop, work)) + .unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&mut schema.admin()), before, "the Layer rolled back"); + assert_eq!(epoch(&mut schema.admin(), shop), 0); + assert_eq!( + handles + .history + .layer_stack(shop) + .unwrap() + .unwrap() + .head_layer, + genesis + ); + + drop_trigger(&mut schema.admin()); + let layer = publish(&handles.history, shop, work); + assert_eq!(layer.parent, Some(genesis)); + assert_eq!(counts(&mut schema.admin()).1, before.1 + 1); +} + +#[test] +fn an_epoch_change_during_a_merge_is_detected_and_rolled_back() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_trigger_merge"); + let binding = binding("fence_trigger_merge"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + let first = commit(&handles.history, work, 5); + let request = merge_request(&handles.history, shop, work, first.id, genesis, 62); + let before = counts(&mut schema.admin()); + + bump_trigger(&mut schema.admin(), "commit"); + let refused = handles.history.merge_branch(&request).unwrap_err(); + assert!( + matches!(refused, HistoryError::Conflict("project epoch")), + "{refused:?}" + ); + assert_eq!(counts(&mut schema.admin()), before, "the merge rolled back"); + assert_eq!(epoch(&mut schema.admin(), shop), 0); + assert!(handles.history.branch(merged_branch()).unwrap().is_none()); + + drop_trigger(&mut schema.admin()); + let merged = handles.history.merge_branch(&request).unwrap(); + assert_eq!(merged.branch.name.as_str(), "merged"); + assert_eq!(counts(&mut schema.admin()).0, before.0 + 1); +} + +#[test] +fn a_mount_blocks_all_three_removals_until_released() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "admission_mount"); + let binding = binding("admission_mount"); + let handles = schema.create(&binding, CURSOR); + let (project_id, branch, _genesis) = project(&handles.history, 1); + let mount = mount(project_id, branch, 1); + handles.history.gc_register(&mount).unwrap(); + + let refusal = handles + .history + .gc_remove_branch(branch, 1_000, &removal(project_id, 2)) + .unwrap_err(); + let HistoryError::MountsActive(state) = &refusal else { + panic!("{refusal:?}"); + }; + assert_eq!(state.holders.len(), 1); + assert_eq!(state.holders[0].kind, GcIntentKind::Mount); + assert_eq!(state.holders[0].sandbox.as_deref(), Some("sandbox-1")); + assert_eq!(state.holders[0].machine.as_deref(), Some("machine-1")); + assert_eq!( + state.holders[0].workspace.as_deref(), + Some("01".repeat(32).as_str()) + ); + assert_eq!(state.holders[0].op, hex16(&op(1))); + + assert!(matches!( + handles + .history + .gc_delete_stack(project_id, 1_000, &removal(project_id, 3)) + .unwrap_err(), + HistoryError::MountsActive(_) + )); + assert!(matches!( + handles + .history + .gc_remove_path(&path_request(project_id), &removal(project_id, 4)) + .unwrap_err(), + HistoryError::MountsActive(_) + )); + + let listed = handles + .history + .gc_intents(GcScopeKind::Branch, branch.as_slice()) + .unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].kind, GcIntentKind::Mount); + handles.history.gc_release(&op(1)).unwrap(); + assert!(handles + .history + .gc_intents(GcScopeKind::Branch, branch.as_slice()) + .unwrap() + .is_empty()); + handles.history.gc_release(&op(1)).unwrap(); + + let outcome = handles + .history + .gc_remove_path(&path_request(project_id), &removal(project_id, 4)) + .unwrap(); + assert_eq!(outcome.kind, PathRemovalKind::Removed); + assert_eq!(epoch(&mut schema.admin(), project_id), 1); + handles + .history + .gc_remove_branch(branch, 2_000, &removal(project_id, 2)) + .unwrap(); + assert_eq!(epoch(&mut schema.admin(), project_id), 2); + assert!(matches!( + handles + .history + .gc_delete_stack(project_id, 3_000, &removal(project_id, 3)) + .unwrap(), + GcDeletion::Marked { deleted_ms: 3_000 } + )); + assert_eq!(epoch(&mut schema.admin(), project_id), 3); +} + +#[test] +fn init_and_merge_are_project_holders() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "admission_shared"); + let binding = binding("admission_shared"); + let handles = schema.create(&binding, CURSOR); + let (project_id, _branch, _genesis) = project(&handles.history, 1); + + handles.history.gc_register(&init(project_id, 5)).unwrap(); + let refusal = handles + .history + .gc_delete_stack(project_id, 1_000, &removal(project_id, 6)) + .unwrap_err(); + let HistoryError::MountsActive(state) = &refusal else { + panic!("{refusal:?}"); + }; + assert_eq!(state.holders[0].kind, GcIntentKind::Init); + handles.history.gc_release(&op(5)).unwrap(); + + handles.history.gc_register(&merge(project_id, 7)).unwrap(); + let refusal = handles + .history + .gc_remove_path(&path_request(project_id), &removal(project_id, 8)) + .unwrap_err(); + let HistoryError::MountsActive(state) = &refusal else { + panic!("{refusal:?}"); + }; + assert_eq!(state.holders[0].kind, GcIntentKind::Merge); + handles.history.gc_release(&op(7)).unwrap(); + + handles + .history + .gc_remove_path(&path_request(project_id), &removal(project_id, 8)) + .unwrap(); +} + +#[test] +fn a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "admission_op"); + let binding = binding("admission_op"); + let handles = schema.create(&binding, CURSOR); + let (project_id, branch, _genesis) = project(&handles.history, 1); + let catalog = CatalogId::derive(&binding).unwrap(); + let compaction = GcIntent::operation( + op(9), + GcIntentKind::Compaction, + GcScopeKind::Store, + catalog.as_slice(), + ) + .owned(Some("runtime-a"), Some("boot-9")) + .with_phase(Some("mark")); + handles.history.gc_register(&compaction).unwrap(); + + let refusal = handles + .history + .gc_remove_branch(branch, 1_000, &removal(project_id, 10)) + .unwrap_err(); + let HistoryError::BusyOp(state) = &refusal else { + panic!("{refusal:?}"); + }; + assert_eq!(state.kind, GcIntentKind::Compaction); + assert_eq!(state.entity, "store"); + assert_eq!(state.phase.as_deref(), Some("mark")); + assert_eq!(state.op, hex16(&op(9))); + + assert!(matches!( + handles + .history + .gc_register(&mount(project_id, branch, 11)) + .unwrap_err(), + HistoryError::BusyOp(_) + )); + + handles.history.gc_mark_uncertain(&op(9)).unwrap(); + let refusal = handles + .history + .gc_remove_branch(branch, 1_000, &removal(project_id, 10)) + .unwrap_err(); + assert!(matches!(refusal, HistoryError::Unresolved(_))); + + handles.history.gc_release(&op(9)).unwrap(); + handles + .history + .gc_remove_branch(branch, 2_000, &removal(project_id, 10)) + .unwrap(); +} + +#[test] +fn a_removal_first_refuses_the_later_commit_and_writes_nothing() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "fence_removal_first"); + let binding = binding("fence_removal_first"); + let handles = schema.create(&binding, CURSOR); + let (shop, work, _genesis) = project(&handles.history, 1); + let _first = commit(&handles.history, work, 5); + let before = counts(&mut schema.admin()); + + // Capture the request while the Branch is still visible; the removal then + // wins, and the filtered Branch resolution hides the retired Branch before + // the fence opens, so the write is refused and leaves no Commit behind. + let request = commit_request(&handles.history, work, root(6)); + handles.history.mark_branch_removed(work, 1_000).unwrap(); + assert_eq!( + epoch(&mut schema.admin(), shop), + 1, + "the marker bumped the epoch once" + ); + + let refused = handles.history.commit_branch(&request).unwrap_err(); + assert!( + matches!( + refused, + HistoryError::Missing(Missing::Branch) | HistoryError::Conflict("project epoch") + ), + "{refused:?}" + ); + assert_eq!(counts(&mut schema.admin()), before); +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_gc_mark.rs b/engine/crates/layerfs-persistence/tests/postgresql_gc_mark.rs new file mode 100644 index 0000000000..5bb4af3df1 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_gc_mark.rs @@ -0,0 +1,387 @@ +//! PostgreSQL runtime evidence for the GC additions that shipped without it: +//! T1.3b's additive GC tables, T7.1's `kept_node` table and history closure, +//! and T7.2's object closure and live-heads digest. +//! +//! Each test creates one disposable schema and drives it through +//! `PostgresHandles`; a test that simulates an older Store rewrites the live +//! schema, its recorded source and its recorded definition exactly as the +//! release before the feature wrote them, then opens it writable. +//! +//! Needs `LAYERFS_TEST_POSTGRES_URL`; NOT_RUN without it. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{construct_bytes, read_range, ObjectId}; +use layerfs_history::HistoryCatalog; +use layerfs_persistence::{KeptNodeId, PostgresHandles}; +use layerfs_storage::{port::PackPersistence, Storage}; +use layerfs_telemetry::timer::Timing; +use std::sync::Arc; +use support::pg::{ + commit, commit_with, counts, project, project_with_genesis, publish, url, Schema, DEFINITION, +}; + +const BINDING: &[u8] = b"postgresql-gc-mark"; +const CURSOR: [u8; 32] = [76; 32]; + +/// The eight tables T1.3b appended; the pre-GC Store has none of them. +const GC_TABLES: [&str; 8] = [ + "branch_retired", + "gc_dead", + "gc_intent", + "gc_op", + "kept_node", + "layer_stack_retired", + "path_removed", + "project_epoch", +]; + +/// Rows of `kept_node`, in the statement's order. +fn kept_rows(admin: &mut postgres::Client) -> Vec<(Vec, String, Vec, i64)> { + admin + .query( + "SELECT layer_stack_id,kind,node_id,serial FROM kept_node \ + ORDER BY layer_stack_id,kind,node_id", + &[], + ) + .unwrap() + .into_iter() + .map(|row| (row.get(0), row.get(1), row.get(2), row.get(3))) + .collect() +} + +/// Rewrites a fresh Store into the format a release before T1.3b wrote: no GC +/// tables and no position columns/state. Returns the source it replaced. +fn make_pre_gc(schema: &Schema) -> String { + let fresh: String = schema + .admin() + .query_one("SELECT schema_source FROM history_meta", &[]) + .unwrap() + .get(0); + let start = fresh + .find("\nCREATE TABLE branch_retired (") + .expect("the GC schema starts with branch_retired"); + let acquisition = fresh + .find("CREATE TABLE init_operation (") + .expect("the acquisition schema follows the GC tables"); + assert!(start < acquisition); + let old = format!("{}{}", &fresh[..start], &fresh[acquisition..]); + assert_ne!(old, fresh); + schema + .admin() + .batch_execute( + "DROP TABLE gc_dead; DROP TABLE gc_intent; DROP TABLE gc_op; \ + DROP TABLE project_epoch; DROP TABLE path_removed; DROP TABLE kept_node; \ + DROP TABLE branch_retired; DROP TABLE layer_stack_retired; \ + ALTER TABLE history_meta DROP COLUMN next_history_serial; \ + ALTER TABLE \"commit\" DROP COLUMN history_serial; \ + ALTER TABLE layer DROP COLUMN history_serial; \ + ALTER TABLE store_policy DROP COLUMN trace_salt;", + ) + .unwrap(); + let definition: String = schema.admin().query_one(DEFINITION, &[]).unwrap().get(0); + schema + .admin() + .execute( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2", + &[&old, &definition], + ) + .unwrap(); + fresh +} + +/// Rewrites a fresh Store into the format a release before T7.1 wrote: the +/// `kept_node` table is gone from the schema and from the recorded source. +fn make_pre_kept_node(schema: &Schema) -> String { + let fresh: String = schema + .admin() + .query_one("SELECT schema_source FROM history_meta", &[]) + .unwrap() + .get(0); + let start = fresh + .find("CREATE TABLE kept_node (") + .expect("a fresh Store holds kept_node"); + let end = start + fresh[start..].find(";\n").expect("the table ends") + 2; + let old = format!("{}{}", &fresh[..start], &fresh[end..]); + assert_ne!(old, fresh); + schema + .admin() + .batch_execute("DROP TABLE kept_node;") + .unwrap(); + let definition: String = schema.admin().query_one(DEFINITION, &[]).unwrap().get(0); + schema + .admin() + .execute( + "UPDATE history_meta SET schema_source=$1,schema_definition=$2", + &[&old, &definition], + ) + .unwrap(); + fresh +} + +/// One Save of `bytes`; returns its root. +fn save(storage: &Storage, bytes: &[u8]) -> ObjectId { + let save = storage.begin_save().unwrap(); + let root = Timing::disabled("construct", |scope| { + construct_bytes( + storage.policy().construction(), + &storage.policy().construction().capacities(), + bytes, + &mut save.sink(), + scope.child("content"), + ) + }) + .0 + .unwrap() + .root; + save.finish().unwrap(); + root +} + +fn bytes(seed: u32, len: usize) -> Vec { + let mut state = seed.wrapping_mul(2654435761) | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 24) as u8 + }) + .collect() +} + +#[test] +fn a_pre_gc_store_gains_the_gc_tables_and_its_project_epoch_rows() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "pre_gc"); + let handles = schema.create(BINDING, CURSOR); + let (shop, work, _genesis) = project(&handles.history, 1); + drop(handles); + let fresh = make_pre_gc(&schema); + + let mut before = schema.admin(); + assert_eq!( + before + .query_one("SELECT count(*) FROM layer_stack", &[]) + .unwrap() + .get::<_, i64>(0), + 1 + ); + assert!(before + .query_one( + "SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='gc_intent'", + &[], + ) + .unwrap() + .get::<_, i64>(0) + == 0); + drop(before); + + let _upgraded = schema + .open(BINDING, CURSOR) + .expect("a pre-T1.3b Store opens writable"); + let mut admin = schema.admin(); + for table in GC_TABLES { + assert_eq!( + admin + .query_one( + "SELECT count(*) FROM information_schema.tables \ + WHERE table_schema=current_schema() AND table_name=$1 AND table_type='BASE TABLE'", + &[&table], + ) + .unwrap() + .get::<_, i64>(0), + 1, + "{table} exists after the additive migration" + ); + } + assert_eq!( + admin + .query_one("SELECT count(*) FROM project_epoch", &[]) + .unwrap() + .get::<_, i64>(0), + 1, + "the existing project gains one epoch row" + ); + assert_eq!( + admin + .query_one( + "SELECT epoch FROM project_epoch WHERE layer_stack_id=$1", + &[&shop.as_slice()], + ) + .unwrap() + .get::<_, i64>(0), + 0 + ); + let (source, recorded_definition): (String, String) = { + let row = admin + .query_one( + "SELECT schema_source,schema_definition FROM history_meta", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + assert_eq!(source, fresh, "the migration records the current source"); + assert_eq!(recorded_definition, schema.definition()); + drop(admin); + + // A second writable open changes nothing. + let again = schema + .open(BINDING, CURSOR) + .expect("the migrated Store reopens"); + assert_eq!( + again + .history + .branch(work) + .unwrap() + .expect("the Branch survived") + .head_commit, + None + ); +} + +#[test] +fn a_pre_kept_node_store_gains_the_table_and_a_mark_fills_it() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "pre_kept"); + let handles = schema.create(BINDING, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + let first = commit(&handles.history, work, 5); + let layer = publish(&handles.history, shop, work); + drop(handles); + let fresh = make_pre_kept_node(&schema); + + let handles = schema + .open(BINDING, CURSOR) + .expect("a pre-T7.1 Store opens writable"); + let mut admin = schema.admin(); + assert_eq!( + admin + .query_one("SELECT count(*) FROM kept_node", &[]) + .unwrap() + .get::<_, i64>(0), + 0, + "the table is empty until a mark writes it" + ); + let source: String = admin + .query_one("SELECT schema_source FROM history_meta", &[]) + .unwrap() + .get(0); + assert_eq!(source, fresh); + drop(admin); + + let kept = handles + .history + .mark_history_closure() + .expect("a mark writes the history closure"); + assert!(kept.contains(KeptNodeId::Stack(shop))); + assert!(kept.contains(KeptNodeId::Branch(work))); + assert!(kept.contains(KeptNodeId::Commit(first.id))); + assert!(kept.contains(KeptNodeId::Layer(layer.id))); + assert!( + kept.contains(KeptNodeId::Layer(genesis)), + "the genesis Layer is reached" + ); + assert_eq!(kept_rows(&mut schema.admin()).len(), kept.len()); +} + +#[test] +fn a_mark_records_the_history_closure_on_postgresql() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "mark_history"); + let handles = schema.create(BINDING, CURSOR); + let (shop, work, genesis) = project(&handles.history, 1); + let first = commit(&handles.history, work, 5); + let layer = publish(&handles.history, shop, work); + + let kept = handles + .history + .mark_history_closure() + .expect("a mark writes kept_node"); + for id in [ + KeptNodeId::Stack(shop), + KeptNodeId::Branch(work), + KeptNodeId::Commit(first.id), + KeptNodeId::Layer(genesis), + KeptNodeId::Layer(layer.id), + ] { + assert!(kept.contains(id), "{id:?} is retained"); + } + let rows = kept_rows(&mut schema.admin()); + assert_eq!(rows.len(), kept.len()); + let mut expected: Vec<(Vec, String, Vec)> = kept + .nodes() + .iter() + .map(|node| { + ( + node.stack().as_slice().to_vec(), + node.id().kind().to_owned(), + node.id().as_slice().to_vec(), + ) + }) + .collect(); + expected.sort(); + let mut actual: Vec<(Vec, String, Vec)> = rows + .iter() + .map(|(stack, kind, id, _)| (stack.clone(), kind.clone(), id.clone())) + .collect(); + actual.sort(); + assert_eq!(actual, expected); + + // A second mark over unchanged history writes the same rows. + let again = handles.history.mark_history_closure().unwrap(); + assert_eq!(again, kept); + assert_eq!(kept_rows(&mut schema.admin()), rows); +} + +#[test] +fn a_mark_keeps_the_saved_objects_and_reports_the_live_heads_digest() { + let Some(url) = url() else { return }; + let schema = Schema::new(url, "mark_objects"); + let handles: PostgresHandles = schema.create(BINDING, CURSOR); + let storage = Storage::new(handles.storage.clone() as Arc).unwrap(); + let genesis_root = save(&storage, b"genesis tree"); + let (shop, work, genesis) = project_with_genesis(&handles.history, 1, genesis_root); + let root_id = save(&storage, &bytes(5, 5000)); + let record = commit_with(&handles.history, work, root_id); + let layer = publish(&handles.history, shop, work); + + let mark = handles.history.mark().expect("a mark writes both closures"); + assert!( + mark.objects().contains(root_id), + "the committed root's object is retained: {:?}", + mark.objects().objects() + ); + assert!(!mark.objects().packs().is_empty(), "its pack is retained"); + assert!(mark.history().contains(KeptNodeId::Commit(record.id))); + assert!(mark.history().contains(KeptNodeId::Layer(layer.id))); + assert!(mark.history().contains(KeptNodeId::Layer(genesis))); + assert_eq!(kept_rows(&mut schema.admin()).len(), mark.history().len()); + + let digest = mark.live_heads_digest(); + assert_ne!( + digest, [0; 32], + "the live-heads digest is not a placeholder" + ); + let again = handles.history.mark().expect("a second mark"); + assert_eq!(again.objects(), mark.objects()); + assert_eq!(again.live_heads_digest(), digest); + + // Nothing was compacted: the object and its pack are still readable. + let mut bytes_out = Vec::new(); + Timing::disabled("read", |scope| { + read_range( + &storage.reader().unwrap(), + root_id, + 0..5000, + &mut bytes_out, + scope.child("read"), + ) + }) + .0 + .unwrap(); + assert_eq!(bytes_out, bytes(5, 5000)); + let (commits, _layers) = counts(&mut schema.admin()); + assert_eq!(commits, 1); +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs b/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs index c1e8694d9c..07305ec894 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs @@ -236,8 +236,27 @@ fn only_tables_with_unbounded_values_have_a_toast_relation() { .iter() .map(|row| row.get(0)) .collect(); + // The additive T1.2c/T1.3b/T7.1 tables keep PostgreSQL's default column + // storage, so they gain a TOAST relation even where every column is + // bounded (bytea ids, bigint counters, short text); the base schema marks + // exactly its bounded columns STORAGE PLAIN. assert_eq!( toasted, - ["commit", "history_meta", "layer", "pack", "reach"] + [ + "branch_retired", + "commit", + "gc_dead", + "gc_intent", + "gc_op", + "history_meta", + "kept_node", + "layer", + "layer_stack_retired", + "pack", + "path_removed", + "project_epoch", + "reach", + "store_policy", + ] ); } diff --git a/engine/crates/layerfs-persistence/tests/postgresql_status.rs b/engine/crates/layerfs-persistence/tests/postgresql_status.rs new file mode 100644 index 0000000000..6f4f54a1be --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_status.rs @@ -0,0 +1,272 @@ +//! #41 T10.5: the PostgreSQL `store.status` compaction projection. +//! +//! `store.status` reads the Store-side `gc_op` row with its compaction +//! `gc_intent` in one status query — no Store open, schema mutation or +//! artifact read — and projects the same shape as the SQLite sidecar: the op +//! `{id, name, phase, started, uncertain}` from the row, `state` from the +//! owner's liveness, and PostgreSQL's `mark`/`build`/`swap`/`cleanup` phases. +//! An idle Store (no row, or a run that settled and removed it) projects +//! `None`. The record's own fact `uncertain` stays true for every remaining +//! row; `state` is what separates a busy run from a leftover. +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` (for +//! example `host=127.0.0.1 port=55432 user=postgres dbname=lfs_t105 +//! sslmode=disable`). Without it each test reports NOT_RUN. +#![cfg(feature = "postgresql")] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::HistoryCatalog as _; +use layerfs_persistence::{ + postgresql_compact_status, CompactFailpoint, CompactRequest, CompactStatus, PostgresHandles, +}; +use layerfs_storage::Storage; +use support::pg::Schema; + +fn binding(tag: &str) -> Vec { + format!("postgresql-status-{tag}").into_bytes() +} + +fn whole_file(handles: &PostgresHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &PostgresHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn request(op: u8, failpoint: Option) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + failpoint, + ..CompactRequest::default() + } +} + +/// The live §8 phases, in order, as `store.status` names them. +const PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + +/// Waits until the recorded owner backend is gone from `pg_stat_activity`. +fn wait_gone(admin: &mut postgres::Client, pid: i32, start: &str) { + for _ in 0..400 { + let live: bool = admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2)", + &[&pid, &start], + ) + .unwrap() + .get(0); + if !live { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend never ended"); +} + +#[test] +fn the_projection_reads_gc_op_gc_intent_and_owner_liveness() { + let Some(url) = support::pg::url() else { + return; + }; + let schema = Schema::new(url.clone(), "row"); + let _handles = schema.create(&binding("row"), [71; 32]); + let name = schema.name.clone(); + let s = &schema.name; + + // Idle: no compaction row, so no op and no phases. + let idle = postgresql_compact_status(&url, &name).unwrap(); + assert!(idle.is_none(), "an idle Store projects no op"); + + // A live owner: an open, non-idle backend is exactly what the §9 + // dead-owner rule treats as "still running the op". + let mut owner = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + owner.batch_execute("BEGIN; SELECT 1").unwrap(); + let (pid, start): (i32, String) = { + let row = owner + .query_one( + "SELECT pg_backend_pid(),backend_start::text FROM pg_stat_activity \ + WHERE pid=pg_backend_pid()", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + let op_id = [0xabu8; 16]; + let artifact = format!("lfs_gc_{}", "ab".repeat(16)); + let digest = [9u8; 32]; + let mut admin = schema.admin(); + admin + .execute( + &format!( + "INSERT INTO {s}.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,\ + started_ms,artifact,pid,backend_start) \ + VALUES (1,$1,'build',$2,$3,$4,$5,$6,$2,$7,$8)" + ), + &[ + &7i64, + &artifact, + &11i64, + &12i64, + &digest.to_vec(), + &1_700_000_000_000i64, + &i64::from(pid), + &start, + ], + ) + .unwrap(); + admin + .execute( + &format!( + "INSERT INTO {s}.gc_intent \ + (op_id,kind,scope_kind,owner_runtime,owner_boot,state,uncertain,phase,\ + generation,artifact,updated_ms) \ + VALUES ($1,'compaction','store',$2,$3,'live',FALSE,'build',7,$4,1700000000000)" + ), + &[&op_id.to_vec(), &"runtime-t105", &"boot-t105", &artifact], + ) + .unwrap(); + + let live = postgresql_compact_status(&url, &name) + .unwrap() + .expect("a live op is projected"); + assert_eq!(live.op_id, op_id, "the intent carries the op identity"); + assert_eq!(live.generation, 7); + assert_eq!(live.phase, "build"); + assert_eq!(live.started_ms, 1_700_000_000_000); + assert_eq!(live.owner_pid, Some(i64::from(pid))); + assert_eq!(live.owner_backend_start.as_deref(), Some(start.as_str())); + assert_eq!(live.owner_runtime.as_deref(), Some("runtime-t105")); + assert_eq!(live.owner_boot.as_deref(), Some("boot-t105")); + assert_eq!(live.intent_state.as_deref(), Some("live")); + assert_eq!(live.mark_serial, Some(11)); + assert_eq!(live.mark_pack_id, Some(12)); + assert_eq!(live.mark_digest.as_deref(), Some(&digest[..])); + assert!(live.alive, "a non-idle backend is a live owner"); + assert!(live.uncertain, "the remaining row is uncertain"); + let busy: CompactStatus = live.status(); + assert_eq!(busy.id, format!("op-{}", "ab".repeat(16))); + assert_eq!(busy.name, "store.compact"); + assert_eq!(busy.phase, "build"); + assert_eq!(busy.started, 1_700_000_000_000); + assert!(busy.uncertain); + assert_eq!(busy.state, "busy"); + assert_eq!(busy.phases, &PHASES[..]); + + // A gone owner leaves the same record as an uncertain leftover. + owner.close().unwrap(); + wait_gone(&mut admin, pid, &start); + let leftover = postgresql_compact_status(&url, &name) + .unwrap() + .expect("the row remains"); + assert!(!leftover.alive); + assert!(leftover.uncertain); + assert_eq!(leftover.op_id, op_id); + let gone = leftover.status(); + assert_eq!(gone.state, "unresolved"); + assert_eq!(gone.phase, "build"); + assert_eq!(gone.id, build_op(0xab)); + assert!(gone.uncertain); + + // A settled run removes the row: the idle projection is back. + admin + .batch_execute(&format!( + "DELETE FROM {s}.gc_op; DELETE FROM {s}.gc_intent WHERE kind='compaction'" + )) + .unwrap(); + assert!(postgresql_compact_status(&url, &name).unwrap().is_none()); +} + +fn build_op(byte: u8) -> String { + format!("op-{}", format!("{byte:02x}").repeat(16)) +} + +#[test] +fn every_postgresql_failpoint_projects_its_phase() { + let Some(url) = support::pg::url() else { + return; + }; + for (tag, point, phase, op) in [ + ("cmark", CompactFailpoint::PostgresAfterMark, "mark", 0x61u8), + ("cbuild", CompactFailpoint::PostgresAfterBuild, "build", 0x62), + ("cswap", CompactFailpoint::PostgresAfterSwap, "swap", 0x63), + ] { + let schema = Schema::new(url.clone(), tag); + let handles = schema.create(&binding(tag), [71; 32]); + let _ = whole_file(&handles, b"t10.5 failpoint dead content"); + let error = handles.compact(&request(op, Some(point))).unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{tag}: {error:?}" + ); + let status = postgresql_compact_status(&url, &schema.name) + .unwrap() + .unwrap_or_else(|| panic!("{tag}: the op row remains")); + assert_eq!(status.phase, phase, "{tag}: the row records the phase"); + // The released `gc_intent` is gone, so the scratch name carries the + // op identity the projection shows. + assert_eq!(status.op_id, [op; 16], "{tag}: the artifact names the op"); + assert!(!status.alive, "{tag}: the failed owner is not running the op"); + assert!(status.uncertain, "{tag}: a failed run stays uncertain"); + let projected = status.status(); + assert_eq!(projected.name, "store.compact"); + assert_eq!(projected.id, build_op(op)); + assert_eq!(projected.state, "unresolved", "{tag}: the leftover"); + assert_eq!(projected.phase, phase); + assert_eq!(projected.phases, &PHASES[..]); + // The next call settles the leftover and removes the row. + let receipt = handles + .compact(&request(op.wrapping_add(1), None)) + .unwrap(); + assert!( + receipt.generation_after >= receipt.generation_before, + "{tag}: settled" + ); + assert!( + postgresql_compact_status(&url, &schema.name).unwrap().is_none(), + "{tag}: a settled run projects no op" + ); + } +} + +#[test] +fn a_completed_run_leaves_no_projection() { + let Some(url) = support::pg::url() else { + return; + }; + let schema = Schema::new(url.clone(), "done"); + let handles = schema.create(&binding("done"), [71; 32]); + let _ = whole_file(&handles, b"t10.5 completed dead content"); + let before = handles.history.incarnation(); + let receipt = handles.compact(&request(0x71, None)).unwrap(); + assert_eq!(receipt.generation_after, before + 1, "the rebuild installed"); + // `store.status` after a completed run: idle, with the intent released. + assert!(postgresql_compact_status(&url, &schema.name).unwrap().is_none()); + let intents: i64 = schema + .admin() + .query_one( + &format!( + "SELECT count(*) FROM {}.gc_intent WHERE kind='compaction'", + schema.name + ), + &[], + ) + .unwrap() + .get(0); + assert_eq!(intents, 0, "the compaction intent was released"); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs index 8445f4f917..c2c4ff7f23 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -438,7 +438,7 @@ fn an_installed_but_unsettled_run_is_judged_at_g_plus_one() { #[test] fn the_report_projects_every_section_11_field() { - let (_temp, handles, path) = create_store(); + let (_temp, handles, _path) = create_store(); let live = filesystem_root(&handles, 1); initialize(&handles, 1, live); let _ = whole_file(&handles, b"dead"); diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs new file mode 100644 index 0000000000..0510d5790e --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs @@ -0,0 +1,902 @@ +//! AC-57 section 13 capacity arm: a real, large SQLite Store compaction measurement. +//! +//! The acceptance matrix's 50 GB arms were extrapolated from the 28.2 ms tiny +//! fixture. This opt-in test builds a Store of the requested size through the +//! public persistence/storage APIs, measures the `none` path at ~9% wholly dead +//! bytes and the `rebuild` path at ~10%, and records wall time, the section 11 +//! receipt values and the T10.3 phase durations. +//! +//! It is ignored by default: `cargo test` never allocates more than a few MiB. +//! Run it with a TMPDIR that has at least 3x the target bytes free: +//! +//! ```text +//! CAPACITY_MEASURE=1 TMPDIR=/var/tmp/g61 UPDATE_CAPACITY=1 \ +//! cargo test -p layerfs-persistence --release \ +//! --test sqlite_compact_capacity -- --ignored --nocapture +//! ``` +//! +//! Knobs: `CAPACITY_GROUPS` (live inode leaves, default 701), +//! `CAPACITY_GROUP` (files per inode leaf, default 99; the page cap is 100 rows including the root directory), +//! `CAPACITY_DEAD_OBJECTS` (initial dead objects, default 6770), +//! `CAPACITY_OBJECT_BYTES` (default 131071, the largest whole-file +//! payload), `CAPACITY_RUNS` (default 3 per arm), `CAPACITY_DURABLE=1` (adds +//! one WAL/FULL rebuild after the WAL/OFF runs), `CAPACITY_KEEP=1` (keeps the +//! temporary Store) and `CAPACITY_OUT` (fixture path override). +//! +//! Fixture shape: every live file is the largest whole-file object (131071 B) +//! reached through one `InodeLeaf` of `CAPACITY_GROUP` file entries and one +//! `FilesystemRoot`, whose `LayerStack` genesis publishes it. The dead payload +//! is whole-file objects no retained root reaches. Every pack holds one object, +//! so `SELECT count(*), sum(length) FROM pack` gives the exact pack byte total +//! and the section 11 `deadBytesBound` can be checked against it. Production +//! saves would add a directory namespace over these inodes; that changes the +//! object count a little, not the bytes compaction copies. +#![cfg(target_os = "linux")] + +mod support; + +use layerfs_content::filesystem::attributes::PortableMetadata; +use layerfs_content::filesystem::directory::{encode_directory_page, DirectoryPage}; +use layerfs_content::filesystem::inode::{encode_inode_page, InodePage}; +use layerfs_content::filesystem::root::profile_id; +use layerfs_content::filesystem::{scope_for_seed, FilesystemRoot}; +use layerfs_content::inode_leaf::{InodeKind, InodeMetadata, InodeValue}; +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId, StackInitialization, +}; +use layerfs_persistence::compact_metrics::{CompactMetrics, CompactMetricsCapture}; +use layerfs_persistence::{ + CompactPath, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{Storage, StoragePolicy}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-capacity-measure"; +const DEFAULT_GROUPS: u64 = 701; +const DEFAULT_GROUP: u64 = 99; +const DEFAULT_DEAD_OBJECTS: u64 = 6_770; +const DEFAULT_OBJECT_BYTES: usize = 131_071; +const BATCH: usize = 100; + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +fn profile_label(profile: SqlitePersistenceProfile) -> &'static str { + match profile { + SqlitePersistenceProfile::Durable => "SqlitePersistenceProfile::Durable (WAL/FULL)", + SqlitePersistenceProfile::Disposable => "SqlitePersistenceProfile::Disposable (WAL/OFF)", + } +} + +fn create_store(path: &Path, profile: SqlitePersistenceProfile) -> SqliteHandles { + SqliteHandles::create( + PersistenceConfig::sqlite(path.to_path_buf()).with_sqlite_profile(profile), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .expect("Store creation") +} + +fn reopen(path: &Path, profile: SqlitePersistenceProfile) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()).with_sqlite_profile(profile), + BINDING, + [71; 32], + ) + .expect("Store reopen") +} + +/// Deterministic payload; the seed makes every object distinct. The first eight +/// bytes carry the object index so two seeds cannot collide by accident. +fn payload(seed: u64, len: usize) -> Vec { + let mut out = vec![0u8; len]; + out[..8].copy_from_slice(&seed.to_le_bytes()); + let mut state = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + for chunk in out[8..].chunks_exact_mut(8) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + chunk.copy_from_slice(&state.to_le_bytes()); + } + out +} + +fn whole_file(bytes: &[u8]) -> FinalizedObject { + FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).expect("whole-file payload"), + ) + .expect("finalized object") +} + +/// Saves a batch of already-finalized objects in one Save. +fn save_batch(handles: &SqliteHandles, objects: Vec) -> Vec { + let storage = Storage::new(handles.storage.clone()).expect("storage"); + let save = storage.begin_save().expect("save"); + let mut ids = Vec::with_capacity(objects.len()); + for object in objects { + ids.push(object.id()); + save.accept(object).expect("accept"); + } + save.finish().expect("save finish"); + ids +} + +fn save_whole_files(handles: &SqliteHandles, payloads: &[Vec]) -> Vec { + let objects = payloads.iter().map(|bytes| whole_file(bytes)).collect(); + save_batch(handles, objects) +} + +/// Publishes one `InodeLeaf` of whole-file content roots and its +/// `FilesystemRoot` as the genesis of its own LayerStack. +fn publish_group(handles: &SqliteHandles, index: u64, roots: &[ObjectId]) { + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).expect("directory"), + ) + .expect("directory leaf"); + let directory_id = directory.id(); + let mut entries = Vec::with_capacity(roots.len() + 1); + entries.push(( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::try_from(index).unwrap_or(0), + mtime_nanoseconds: 0, + }), + }, + )); + for (offset, root) in roots.iter().enumerate() { + entries.push(( + u64::try_from(offset + 2).expect("inode serial"), + InodeValue { + kind: InodeKind::RegularFile, + namespace_ref_count: 1, + content_root: *root, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o644, + mtime_seconds: i64::try_from(index).unwrap_or(0), + mtime_nanoseconds: 0, + }), + }, + )); + } + let mut references = Vec::with_capacity(roots.len() + 1); + references.push(directory_id); + references.extend_from_slice(roots); + let leaf = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { entries }).expect("inode page"), + ) + .expect("inode leaf") + .with_references(references); + let leaf_id = leaf.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, leaf_id) + .expect("filesystem root"); + let root_object = + FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().expect("root encode")) + .expect("filesystem root object") + .with_references(vec![leaf_id]); + let root_id = root_object.id(); + save_batch(handles, vec![directory, leaf, root_object]); + + let mut stack = [0u8; 16]; + stack[..8].copy_from_slice(&index.to_le_bytes()); + stack[8..].copy_from_slice(b"g61cap00"); + handles + .history + .initialize_layerstack(&StackInitialization { + stack: LayerStackId::from_authority(stack), + name: HistoryName::new(&format!("capacity-{index:07}")).expect("name"), + scope: scope_for_seed([1; 32]).object(), + profile: profile_id(), + genesis_root: root_id, + logical: 0, + }) + .expect("layerstack initialization"); +} + +#[derive(Clone, Copy, Debug, Default)] +struct PackStats { + count: u64, + total: u64, + min: u64, + max: u64, +} + +/// Reads the exact pack population straight from the Store file. The read-only +/// connection is closed before any compaction starts. +fn pack_stats(path: &Path) -> PackStats { + let open = |flags| rusqlite::Connection::open_with_flags(path, flags); + let connection = open(rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .or_else(|_| open(rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE)) + .expect("Store probe"); + let row: (i64, i64, i64, i64) = connection + .query_row( + "SELECT count(*), COALESCE(sum(length),0), COALESCE(min(length),0), \ + COALESCE(max(length),0) FROM pack", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("pack statistics"); + PackStats { + count: u64::try_from(row.0).unwrap_or(0), + total: u64::try_from(row.1).unwrap_or(0), + min: u64::try_from(row.2).unwrap_or(0), + max: u64::try_from(row.3).unwrap_or(0), + } +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +#[derive(Clone, Debug)] +struct Measured { + micros: u64, + path: &'static str, + generation_before: u64, + generation_after: u64, + dead_bytes_bound: u64, + dead_bytes_refined: u64, + physical_shrink: u64, + peak_temp_bytes: u64, + old_file_bytes: u64, + new_file_bytes: u64, + added_objects: u64, + pack_before: PackStats, + pack_after: PackStats, + phases_nanos: Vec<(&'static str, u64)>, +} + +fn measured(path: &Path, profile: SqlitePersistenceProfile, op: u8, added: u64) -> Measured { + let handles = reopen(path, profile); + let pack_before = pack_stats(path); + let old_file_bytes = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let capture = CompactMetricsCapture::start(); + let started = Instant::now(); + let receipt = handles.compact(&request(op)).expect("compaction"); + let micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + let metrics: CompactMetrics = capture.finish(); + let new_file_bytes = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let pack_after = pack_stats(path); + Measured { + micros, + path: match receipt.path { + CompactPath::None => "none", + CompactPath::Rebuild => "rebuild", + }, + generation_before: receipt.generation_before, + generation_after: receipt.generation_after, + dead_bytes_bound: receipt.dead_bytes_bound, + dead_bytes_refined: receipt.dead_bytes_refined, + physical_shrink: receipt.physical_shrink, + peak_temp_bytes: receipt.peak_temp_bytes, + old_file_bytes, + new_file_bytes, + added_objects: added, + pack_before, + pack_after, + phases_nanos: phase_nanos(&metrics), + } +} + +fn phase_nanos(metrics: &CompactMetrics) -> Vec<(&'static str, u64)> { + let mut phases: Vec<(&'static str, u64)> = metrics + .samples() + .into_iter() + .filter(|sample| sample.name == "compact.phase.duration_ns") + .filter(|sample| sample.label("statistic") == Some("sum")) + .filter_map(|sample| Some((sample.label("phase")?, sample.value))) + .collect(); + phases.sort_by_key(|(phase, _)| *phase); + phases +} + +fn median(values: &[u64]) -> u64 { + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] +} + +fn json_u64_array(values: &[u64]) -> String { + values + .iter() + .map(u64::to_string) + .collect::>() + .join(", ") +} + +fn phase_value(run: &Measured, phase: &str) -> u64 { + run.phases_nanos + .iter() + .find(|(name, _)| *name == phase) + .map(|(_, value)| *value) + .unwrap_or(0) +} + +/// One arm as a JSON object, with the per-run values and the median/range. +fn json_arm(name: &str, runs: &[Measured]) -> String { + let micros: Vec = runs.iter().map(|run| run.micros).collect(); + let old: Vec = runs.iter().map(|run| run.old_file_bytes).collect(); + let new: Vec = runs.iter().map(|run| run.new_file_bytes).collect(); + let dead: Vec = runs.iter().map(|run| run.dead_bytes_bound).collect(); + let refined: Vec = runs.iter().map(|run| run.dead_bytes_refined).collect(); + let shrink: Vec = runs.iter().map(|run| run.physical_shrink).collect(); + let peak: Vec = runs.iter().map(|run| run.peak_temp_bytes).collect(); + let generation_before: Vec = runs.iter().map(|run| run.generation_before).collect(); + let generation_after: Vec = runs.iter().map(|run| run.generation_after).collect(); + let packs_before: Vec = runs.iter().map(|run| run.pack_before.count).collect(); + let packs_after: Vec = runs.iter().map(|run| run.pack_after.count).collect(); + let added: Vec = runs.iter().map(|run| run.added_objects).collect(); + let phases = ["mark", "build", "verify", "install"] + .iter() + .map(|phase| { + let values: Vec = runs.iter().map(|run| phase_value(run, phase)).collect(); + format!("\n \"{phase}\": [{}]", json_u64_array(&values)) + }) + .collect::>() + .join(","); + format!( + "\n \"{name}\": {{\n \"path\": \"{}\",\n \"runs\": {},\n \"micros\": [{}],\n \ + \"medianMicros\": {},\n \"minMicros\": {},\n \"maxMicros\": {},\n \ + \"deadBytesBound\": [{}],\n \"deadBytesRefined\": [{}],\n \ + \"physicalShrink\": [{}],\n \"peakTempBytes\": [{}],\n \ + \"oldFileBytes\": [{}],\n \"newFileBytes\": [{}],\n \ + \"generationBefore\": [{}],\n \"generationAfter\": [{}],\n \ + \"packCountBefore\": [{}],\n \"packCountAfter\": [{}],\n \ + \"addedDeadObjects\": [{}],\n \"phasesNanos\": {{{}\n }}\n }}", + runs[0].path, + runs.len(), + json_u64_array(µs), + median(µs), + micros.iter().min().copied().unwrap_or(0), + micros.iter().max().copied().unwrap_or(0), + json_u64_array(&dead), + json_u64_array(&refined), + json_u64_array(&shrink), + json_u64_array(&peak), + json_u64_array(&old), + json_u64_array(&new), + json_u64_array(&generation_before), + json_u64_array(&generation_after), + json_u64_array(&packs_before), + json_u64_array(&packs_after), + json_u64_array(&added), + phases, + ) +} + +fn rustc_version() -> String { + std::process::Command::new("rustc") + .arg("--version") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|text| text.trim().to_owned()) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +fn cpus() -> u64 { + std::thread::available_parallelism() + .map(|value| u64::try_from(value.get()).unwrap_or(0)) + .unwrap_or(0) +} + +fn mem_bytes() -> u64 { + std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|text| { + text.lines().find_map(|line| { + let rest = line.strip_prefix("MemTotal:")?; + let kib: u64 = rest.trim().trim_end_matches(" kB").trim().parse().ok()?; + Some(kib.saturating_mul(1024)) + }) + }) + .unwrap_or(0) +} + +fn update_fixture(path: &Path, json: &str) { + std::fs::write(path, json.as_bytes()).expect("fixture write"); +} + +/// Saves `count` dead whole-file objects in batches and returns how many it added. +fn add_dead(handles: &SqliteHandles, count: u64, seed_base: u64, object_bytes: usize) { + let mut left = count; + let mut seed = seed_base; + while left > 0 { + let take = left.min(BATCH as u64); + let payloads: Vec> = (0..take) + .map(|offset| payload(seed + offset, object_bytes)) + .collect(); + let _ = save_whole_files(handles, &payloads); + seed += take; + left -= take; + } +} + +/// Adds the smallest number of whole-file dead objects that makes the pack-byte +/// dead share reach 10%. Returns `(dead_bytes, objects_added, next_seed)`. +fn cross_ten( + store: &Path, + profile: SqlitePersistenceProfile, + dead_bytes: u64, + mut seed: u64, + object_bytes: usize, +) -> (u64, u64, u64) { + // One object is one singleton pack: measure its width instead of assuming + // the pack framing. + let before = pack_stats(store); + { + let handles = reopen(store, profile); + add_dead(&handles, 1, seed, object_bytes); + drop(handles); + } + seed += 1; + let after = pack_stats(store); + let unit = after.total.saturating_sub(before.total); + let mut dead = dead_bytes.saturating_add(unit); + let mut added = 1u64; + loop { + let before = pack_stats(store); + if before.total > 0 && dead.saturating_mul(10) >= before.total { + break; + } + let need = before + .total + .saturating_sub(dead.saturating_mul(10)) + .div_ceil(unit.saturating_mul(9)) + .max(1); + let handles = reopen(store, profile); + add_dead(&handles, need, seed, object_bytes); + drop(handles); + seed += need; + added += need; + let after = pack_stats(store); + dead = dead.saturating_add(after.total.saturating_sub(before.total)); + if added > 10_000_000 { + panic!("could not cross the 10% threshold"); + } + } + (dead, added, seed) +} + +/// The opt-in section 13 capacity measurement. See the module docs. +#[test] +#[ignore = "opt-in 50 GB capacity measurement: set CAPACITY_MEASURE=1 and a large TMPDIR"] +fn capacity_measure() { + if std::env::var_os("CAPACITY_MEASURE").is_none() { + eprintln!("capacity_measure: skipped; set CAPACITY_MEASURE=1 to run the section 13 arm"); + return; + } + let groups = env_u64("CAPACITY_GROUPS", DEFAULT_GROUPS); + let group_size = env_u64("CAPACITY_GROUP", DEFAULT_GROUP); + let initial_dead = env_u64("CAPACITY_DEAD_OBJECTS", DEFAULT_DEAD_OBJECTS); + let runs_per_arm = env_u64("CAPACITY_RUNS", 3).max(1); + let object_bytes = + usize::try_from(env_u64("CAPACITY_OBJECT_BYTES", DEFAULT_OBJECT_BYTES as u64)) + .expect("object bytes"); + let live_objects = groups.saturating_mul(group_size); + let durable = std::env::var_os("CAPACITY_DURABLE").is_some(); + let keep = std::env::var_os("CAPACITY_KEEP").is_some(); + + let temp = Temp::new("sqlite-compact-capacity"); + let store = temp.join("store.sqlite"); + eprintln!( + "capacity_measure: {} groups x {} whole-file objects of {} B + {} dead objects under {}", + groups, + group_size, + object_bytes, + initial_dead, + temp.path().display() + ); + + // Build the live payload through the public APIs: whole-file objects reached + // through an inode leaf and a filesystem root, published as a stack genesis. + let handles = create_store(&store, SqlitePersistenceProfile::Disposable); + let build_started = Instant::now(); + let mut seed = 0u64; + for group in 0..groups { + let payloads: Vec> = (0..group_size) + .map(|offset| payload(seed + offset, object_bytes)) + .collect(); + seed += group_size; + let roots = save_whole_files(&handles, &payloads); + publish_group(&handles, group, &roots); + if group % 50 == 0 || group + 1 == groups { + let elapsed = build_started.elapsed().as_secs_f64(); + let written = (group + 1) as f64 * group_size as f64 * object_bytes as f64; + eprintln!( + "capacity_measure: build live groups {}/{} ({:.2} GB, {:.2} GB/s)", + group + 1, + groups, + written / 1e9, + written / 1e9 / elapsed.max(1e-9) + ); + let _ = std::io::stderr().flush(); + } + } + add_dead(&handles, initial_dead, 1_000_000_000, object_bytes); + let fixture_build_seconds = build_started.elapsed().as_secs_f64(); + let built = pack_stats(&store); + eprintln!( + "capacity_measure: fixture built in {:.1} s; {} packs, {:.3} GB, pack length {}..{}", + fixture_build_seconds, + built.count, + built.total as f64 / 1e9, + built.min, + built.max + ); + drop(handles); + + // The `none` path at ~9%: mark only, repeated for the E3 median and range. + let mut none_runs = Vec::new(); + for run in 0..u8::try_from(runs_per_arm).expect("run count") { + let measured = measured(&store, SqlitePersistenceProfile::Disposable, 90 + run, 0); + assert_eq!(measured.path, "none", "~9% dead takes the none path"); + eprintln!( + "capacity_measure: none run {} -> {} us, dead {} B, shrink {} B", + run + 1, + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink + ); + none_runs.push(measured); + } + + // The `rebuild` path at the 10% threshold, then again after re-adding enough + // dead packs for the refined decision on each compacted Store. + let mut rebuild_runs = Vec::new(); + let mut dead_bytes = none_runs[0].dead_bytes_bound; + let mut dead_seed = 2_000_000_000u64; + for run in 0..u8::try_from(runs_per_arm).expect("run count") { + let (_crossed, added, seed) = cross_ten( + &store, + SqlitePersistenceProfile::Disposable, + dead_bytes, + dead_seed, + object_bytes, + ); + dead_seed = seed; + let measured = measured(&store, SqlitePersistenceProfile::Disposable, 100 + run, added); + assert_eq!(measured.path, "rebuild", "10% dead takes the rebuild path"); + eprintln!( + "capacity_measure: rebuild run {} -> {} us, dead {} B, shrink {} B, peak {} B", + run + 1, + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink, + measured.peak_temp_bytes + ); + dead_bytes = 0; + rebuild_runs.push(measured); + } + + // Optional WAL/FULL sensitivity run: production's default profile. + let mut durable_run = None; + if durable { + let (_crossed, added, _seed) = cross_ten( + &store, + SqlitePersistenceProfile::Durable, + dead_bytes, + dead_seed, + object_bytes, + ); + let measured = measured(&store, SqlitePersistenceProfile::Durable, 120, added); + assert_eq!(measured.path, "rebuild"); + eprintln!( + "capacity_measure: durable rebuild -> {} us, dead {} B, shrink {} B, peak {} B", + measured.micros, + measured.dead_bytes_bound, + measured.physical_shrink, + measured.peak_temp_bytes + ); + durable_run = Some(measured); + } + + let none_dead = none_runs[0].dead_bytes_bound; + let none_total = none_runs[0].pack_before.total; + let rebuild_dead = rebuild_runs[0].dead_bytes_bound; + let rebuild_total = rebuild_runs[0].pack_before.total; + let mut arms = format!( + "{},{}", + json_arm("none", &none_runs), + json_arm("rebuild", &rebuild_runs) + ); + if let Some(run) = &durable_run { + arms.push_str(&format!( + ",\n \"durableRebuild\": {{\n \"path\": \"rebuild\",\n \"runs\": 1,\n \ + \"micros\": [{}],\n \"deadBytesBound\": [{}],\n \ + \"physicalShrink\": [{}],\n \"peakTempBytes\": [{}],\n \ + \"addedDeadObjects\": [{}],\n \"phasesNanos\": {{\n \"mark\": [{}],\n \ + \"build\": [{}],\n \"verify\": [{}],\n \"install\": [{}]\n }}\n }}", + run.micros, + run.dead_bytes_bound, + run.physical_shrink, + run.peak_temp_bytes, + run.added_objects, + phase_value(run, "mark"), + phase_value(run, "build"), + phase_value(run, "verify"), + phase_value(run, "install"), + )); + } + let json = format!( + "{{\n \"schema\": \"t12.3a-sqlite-compact-capacity/1\",\n \ + \"actualBytes\": {},\n \"actualGib\": \"{:.3}\",\n \ + \"packCount\": {},\n \"packLengthMin\": {},\n \"packLengthMax\": {},\n \ + \"objectBytes\": {},\n \"groups\": {},\n \"groupSize\": {},\n \ + \"liveObjects\": {},\n \"initialDeadObjects\": {},\n \ + \"deadFractionPermille\": {},\n \"fixtureBuildSeconds\": {:.1},\n \ + \"noneDeadBytes\": {},\n \"noneTotalBytes\": {},\n \ + \"rebuildDeadBytes\": {},\n \"rebuildTotalBytes\": {},\n \ + \"profile\": \"{}\",\n \"platform\": \"{}-{}\",\n \ + \"rustc\": \"{}\",\n \"cpus\": {},\n \"memBytes\": {},{}\n}}\n", + built.total, + built.total as f64 / 1e9, + built.count, + built.min, + built.max, + object_bytes, + groups, + group_size, + live_objects, + initial_dead, + none_dead.saturating_mul(1000) / none_total.max(1), + fixture_build_seconds, + none_dead, + none_total, + rebuild_dead, + rebuild_total, + profile_label(SqlitePersistenceProfile::Disposable), + std::env::consts::OS, + std::env::consts::ARCH, + rustc_version(), + cpus(), + mem_bytes(), + arms, + ); + println!("{json}"); + if std::env::var_os("UPDATE_CAPACITY").is_some() { + let out = std::env::var_os("CAPACITY_OUT").map(PathBuf::from).unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../../engine/crates/layerfs-persistence/tests/fixtures/\ + sqlite_compact_capacity_measure.json", + ) + }); + update_fixture(&out, &json); + eprintln!("capacity_measure: wrote {}", out.display()); + } + if keep { + eprintln!("capacity_measure: kept {}", temp.path().display()); + std::mem::forget(temp); + } else { + eprintln!("capacity_measure: removing {}", temp.path().display()); + } +} + + +// ---- the peak-temp receipt covers the whole run's temp artifacts ------------ + +/// The maximum simultaneous extent of the rebuild's temp artifacts one sample +/// saw: the pre-truncate staged copy, its rollback journal, the visited scratch +/// and SQLite's unlinked rewrite temp database. +#[derive(Clone, Copy, Debug, Default)] +struct TempFootprint { + total: u64, + staged: u64, + journal: u64, + visited: u64, + rewrite: u64, +} + +/// Samples the run's temp artifacts from a second thread. The staged file, +/// its `-journal` and the visited scratch are visible in the Store's directory; +/// SQLite writes the rewrite's temp database to the process temp directory and +/// unlinks it, so it is reachable only through `/proc/self/fd` (Linux). +struct FootprintSampler { + stop: Arc, + handle: Option>, + total: Arc, + staged: Arc, + journal: Arc, + visited: Arc, + rewrite: Arc, +} + +impl FootprintSampler { + fn start(store: &Path) -> Self { + let name = store + .file_name() + .map(|file| file.to_string_lossy().into_owned()) + .unwrap_or_default(); + let directory = store + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let stop = Arc::new(AtomicBool::new(false)); + let total = Arc::new(AtomicU64::new(0)); + let staged = Arc::new(AtomicU64::new(0)); + let journal = Arc::new(AtomicU64::new(0)); + let visited = Arc::new(AtomicU64::new(0)); + let rewrite = Arc::new(AtomicU64::new(0)); + let handle = { + let stop = Arc::clone(&stop); + let total = Arc::clone(&total); + let staged_at = Arc::clone(&staged); + let journal_at = Arc::clone(&journal); + let visited_at = Arc::clone(&visited); + let rewrite_at = Arc::clone(&rewrite); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let (mut staged, mut journal, mut visited) = (0u64, 0u64, 0u64); + if let Ok(entries) = std::fs::read_dir(&directory) { + for entry in entries.flatten() { + let file = entry.file_name().to_string_lossy().into_owned(); + let Some(suffix) = file.strip_prefix(&name) else { + continue; + }; + let Some(suffix) = suffix.strip_prefix('.') else { + continue; + }; + let size = entry.metadata().map(|meta| meta.len()).unwrap_or(0); + if suffix == "compact" { + staged = staged.max(size); + } else if suffix.starts_with("compact-journal") + || suffix.starts_with("compact-mj") + { + journal = journal.max(size); + } else if suffix.starts_with("compact-visited") { + visited = visited.max(size); + } + } + } + let mut rewrite = 0u64; + if let Ok(fds) = std::fs::read_dir("/proc/self/fd") { + for fd in fds.flatten() { + let Ok(link) = std::fs::read_link(fd.path()) else { + continue; + }; + if !link.to_string_lossy().contains("etilqs_") { + continue; + } + if let Ok(meta) = std::fs::metadata(fd.path()) { + rewrite = rewrite.max(meta.len()); + } + } + } + staged_at.fetch_max(staged, Ordering::Relaxed); + journal_at.fetch_max(journal, Ordering::Relaxed); + visited_at.fetch_max(visited, Ordering::Relaxed); + rewrite_at.fetch_max(rewrite, Ordering::Relaxed); + total.fetch_max( + staged + journal + visited + rewrite, + Ordering::Relaxed, + ); + std::thread::sleep(Duration::from_micros(200)); + } + }) + }; + Self { + stop, + handle: Some(handle), + total, + staged, + journal, + visited, + rewrite, + } + } + + fn finish(mut self) -> TempFootprint { + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + TempFootprint { + total: self.total.load(Ordering::Relaxed), + staged: self.staged.load(Ordering::Relaxed), + journal: self.journal.load(Ordering::Relaxed), + visited: self.visited.load(Ordering::Relaxed), + rewrite: self.rewrite.load(Ordering::Relaxed), + } + } +} + +/// The §11 peak-temp receipt on a moderate fixture. The run really does create +/// the pre-truncate staged copy, a rollback journal (the prune and the rewrite) +/// and SQLite's unlinked rewrite temp database, and the receipt must bound all +/// of them; the old `new + visited` formula fails the sampled journal. The +/// fixture is ~40 MB, not the 50 GB T-G66 arm: the same code path, sampled. +#[test] +fn the_peak_temp_receipt_covers_the_staged_copy_and_the_rollback_journal() { + const GROUPS: u64 = 16; + const GROUP: u64 = 48; + const OBJECT_BYTES: usize = 40 * 1024; + const DEAD_OBJECTS: u64 = 240; + + let temp = Temp::new("sqlite-compact-peak-temp"); + let store = temp.join("store.sqlite"); + let handles = create_store(&store, SqlitePersistenceProfile::Disposable); + let mut seed = 0u64; + for group in 0..GROUPS { + let payloads: Vec> = (0..GROUP) + .map(|offset| payload(seed + offset, OBJECT_BYTES)) + .collect(); + seed += GROUP; + let roots = save_whole_files(&handles, &payloads); + publish_group(&handles, group, &roots); + } + add_dead(&handles, DEAD_OBJECTS, 1_000_000_000, OBJECT_BYTES); + drop(handles); + + let sampler = FootprintSampler::start(&store); + let handles = reopen(&store, SqlitePersistenceProfile::Disposable); + let receipt = handles.compact(&request(0x7a)).expect("compaction"); + let footprint = sampler.finish(); + + assert_eq!( + receipt.path, + CompactPath::Rebuild, + "the fixture must take the rebuild path" + ); + assert!( + footprint.staged > 0, + "the staged copy must be sampled: {footprint:?}" + ); + assert!( + footprint.journal > 0, + "the rollback journal must be sampled: {footprint:?}" + ); + assert!( + footprint.rewrite > 0, + "the unlinked rewrite temp database must be sampled: {footprint:?}" + ); + assert!( + footprint.visited > 0, + "the visited scratch must be sampled: {footprint:?}" + ); + let new_bytes = std::fs::metadata(&store).expect("installed Store").len(); + eprintln!( + "peak-temp fixture: new file {new_bytes} B, sampled temp footprint {footprint:?}, \ + receipt {} B", + receipt.peak_temp_bytes + ); + assert!( + receipt.peak_temp_bytes >= new_bytes + footprint.journal + footprint.visited, + "peakTempBytes {} must cover the new file {} plus the sampled {}-byte journal \ + and {}-byte visited scratch", + receipt.peak_temp_bytes, + new_bytes, + footprint.journal, + footprint.visited + ); + assert!( + receipt.peak_temp_bytes >= footprint.total, + "peakTempBytes {} must bound the sampled {}-byte temp footprint: {footprint:?}", + receipt.peak_temp_bytes, + footprint.total + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs index 79751dd869..d4fbc3eae3 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs @@ -38,6 +38,10 @@ struct Case { installed: bool, target: bool, visited: bool, + /// A real `SIGKILL` before the seal ends the process with the writer + /// session's WAL on disk (`-wal`, `-shm`); the in-process matrix stops and + /// drops cleanly, so only the real-process `BeforeSeal` arm sets this. + live_sidecars: bool, } fn create_store() -> (Temp, SqliteHandles, PathBuf) { @@ -313,26 +317,12 @@ fn assert_no_sidecars(store: &Path, case: Case) { } } -fn exercise(case: Case) { - let (_temp, handles, path) = create_store(); - let live = filesystem_root(&handles, 1); - let stack = initialize(&handles, 1, live); - let dead = whole_file(&handles, b"a pack no retained root reaches"); - let live_bytes = object_bytes(&handles, live); - let before = generation(&handles); - - let op = [41u8; 16]; - let error = handles - .compact(&request_at(op, case.failpoint)) - .unwrap_err(); - assert!( - matches!(error, layerfs_history::HistoryError::UnknownOutcome), - "{error:?}" - ); - +/// Asserts the operation record and the artifact set the failpoint leaves +/// beside the live Store, before any handle reopens it. +fn assert_crash_state(case: Case, store: &Path, before: u64, op: [u8; 16]) { // `store.status` (T10.1) projects the recorded operation, its phase and // its uncertain outcome; the engine reads them from the op record. - let status = compact_op_status(&path).expect("the op record remains"); + let status = compact_op_status(store).expect("the op record remains"); assert_eq!(status.op_id, op); assert_eq!(status.generation, i64::try_from(before).unwrap()); assert_eq!(status.phase, case.phase, "phase at {case:?}"); @@ -343,20 +333,65 @@ fn exercise(case: Case) { // The exact artifact set the point leaves beside the live Store. assert_eq!( - compact_path(&path).exists(), + compact_path(store).exists(), case.target, "staged Store at {case:?}" ); assert_eq!( - visited_path(&path).exists(), + visited_path(store).exists(), case.visited, "visited scratch at {case:?}" ); - assert!(op_path(&path).exists(), "op record at {case:?}"); - assert_no_sidecars(&path, case); - assert_no_sidecars(&compact_path(&path), case); + assert!(op_path(store).exists(), "op record at {case:?}"); + if case.live_sidecars { + // A real kill before the seal ends the process with the writer + // session's WAL still on disk: the next open replays it, and the crash + // state itself keeps both sidecars, exactly as a killed daemon does. + assert!( + with_suffix(store, "-wal").exists(), + "the killed session's WAL at {case:?}" + ); + assert!( + with_suffix(store, "-shm").exists(), + "the killed session's shared memory at {case:?}" + ); + assert!( + !with_suffix(store, "-journal").exists(), + "no rollback journal at {case:?}" + ); + } else { + assert_no_sidecars(store, case); + } + assert_no_sidecars(&compact_path(store), case); +} - let reopened = reopen(&path); +/// The fixtures one crash case needs to prove its recovery. +struct Recovery<'a> { + case: Case, + store: &'a Path, + before: u64, + live: ObjectId, + live_bytes: &'a [u8], + dead: ObjectId, + stack: LayerStackRecord, + op: [u8; 16], +} + +/// Reopens the crashed Store, asserts the §9 state (wholly old or wholly new, +/// live data readable, exact artifacts) and then proves the next +/// `store.compact` settles or rebuilds and leaves a clean Store. +fn assert_recovery(recovery: Recovery<'_>) { + let Recovery { + case, + store, + before, + live, + live_bytes, + dead, + stack, + op, + } = recovery; + let reopened = reopen(store); // The admission intent is released before the seal (T-G5), so only a // crash before that release still leaves it; a rebuilt Store was copied @@ -380,7 +415,7 @@ fn exercise(case: Case) { ); assert!(located(&reopened, live), "live root at {case:?}"); assert_eq!( - object_bytes(&reopened, live), + object_bytes(&reopened, live).as_slice(), live_bytes, "live bytes readable at {case:?}" ); @@ -397,7 +432,7 @@ fn exercise(case: Case) { // A crash during the copy can tear the staged file. The next run must // remove the recorded set instead of trusting it. if case.failpoint == CompactFailpoint::DuringCopy { - let target = compact_path(&path); + let target = compact_path(store); std::fs::OpenOptions::new() .write(true) .open(&target) @@ -410,11 +445,11 @@ fn exercise(case: Case) { // Put one back to prove the next run removes the whole set. if case.failpoint == CompactFailpoint::DuringSettle { std::fs::write( - with_suffix(&compact_path(&path), "-journal"), + with_suffix(&compact_path(store), "-journal"), b"partial settle", ) .unwrap(); - std::fs::write(visited_path(&path), b"partial settle").unwrap(); + std::fs::write(visited_path(store), b"partial settle").unwrap(); } // The next run judges the leftover and reaches a clean state: a rebuild @@ -435,17 +470,17 @@ fn exercise(case: Case) { receipt.residue ); - assert!(compact_op_status(&path).is_none(), "op settled at {case:?}"); - assert!(!compact_path(&path).exists(), "target removed at {case:?}"); - assert!(!visited_path(&path).exists(), "visited removed at {case:?}"); - assert!(!op_path(&path).exists(), "op removed at {case:?}"); - assert_no_sidecars(&path, case); + assert!(compact_op_status(store).is_none(), "op settled at {case:?}"); + assert!(!compact_path(store).exists(), "target removed at {case:?}"); + assert!(!visited_path(store).exists(), "visited removed at {case:?}"); + assert!(!op_path(store).exists(), "op removed at {case:?}"); + assert_no_sidecars(store, case); - let final_handles = reopen(&path); + let final_handles = reopen(store); assert_eq!(generation(&final_handles), before + 1, "final generation"); assert!(located(&final_handles, live), "final live root"); assert_eq!( - object_bytes(&final_handles, live), + object_bytes(&final_handles, live).as_slice(), live_bytes, "final bytes" ); @@ -456,6 +491,36 @@ fn exercise(case: Case) { clean(&final_handles); } +fn exercise(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + let op = [41u8; 16]; + let error = handles + .compact(&request_at(op, case.failpoint)) + .unwrap_err(); + assert!( + matches!(error, layerfs_history::HistoryError::UnknownOutcome), + "{error:?}" + ); + + assert_crash_state(case, &path, before, op); + assert_recovery(Recovery { + case, + store: &path, + before, + live, + live_bytes: &live_bytes, + dead, + stack, + op, + }); +} + macro_rules! crash_at { ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal) => { #[test] @@ -466,6 +531,7 @@ macro_rules! crash_at { installed: $installed, target: $target, visited: $visited, + live_sidecars: false, }); } }; @@ -575,3 +641,317 @@ crash_at!( false, false ); + +/// The same §9 crash points, but the compaction runs in a **separate process** +/// that the parent `SIGKILL`s while it is stopped inside the failpoint. This +/// is the crash arm the in-process matrix above cannot claim: no Rust `Drop`, +/// no connection close and no guard release runs after the kill. +/// +/// The child re-executes this test binary by name and reopens the fixture +/// Store the parent built. The parent learns the child reached the point from +/// a Unix-socket gate (no sleeps), then kills it, then makes the same +/// assertions as `assert_crash_state` / `assert_recovery`. +/// +/// Nine points are covered: the mark phase (`BeforeSeal`, `AfterSeal`); the +/// copy (`BeforeVacuum`, immediately before `VACUUM INTO` because SQLite runs +/// that copy as one uninterruptible call, so a kill cannot be injected inside +/// it); verification (`DuringVerify`); the rename boundary (`DuringRename`); +/// and the earlier build, install and settle points. +/// +/// Linux only: `SIGKILL` delivery and `ExitStatusExt::signal` are asserted +/// through the Unix process API. On other Unix targets the test below prints +/// a skip line instead of claiming coverage. +#[cfg(target_os = "linux")] +mod real_process { + use super::*; + use std::io::Read; + use std::os::unix::process::ExitStatusExt; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + + /// The operation identity the child records and the parent asserts. + const OP: [u8; 16] = [43u8; 16]; + const STORE_ENV: &str = "LAYERFS_COMPACT_CRASH_STORE"; + const FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + const GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; + + /// A short socket path: `Temp` lives under the (ext4) sandbox run + /// directory, whose absolute path exceeds `SUN_LEN`. + fn gate_socket() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + PathBuf::from(format!( + "/tmp/layerfs-compact-gate-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) + } + + fn parse_failpoint(text: &str) -> CompactFailpoint { + match text { + "BeforeSeal" => CompactFailpoint::BeforeSeal, + "AfterSeal" => CompactFailpoint::AfterSeal, + "BeforeVacuum" => CompactFailpoint::BeforeVacuum, + "DuringVerify" => CompactFailpoint::DuringVerify, + "DuringRename" => CompactFailpoint::DuringRename, + "AfterBuild" => CompactFailpoint::AfterBuild, + "BeforeInstall" => CompactFailpoint::BeforeInstall, + "AfterInstall" => CompactFailpoint::AfterInstall, + "DuringSettle" => CompactFailpoint::DuringSettle, + other => panic!("unknown failpoint {other}"), + } + } + + /// The child body. A normal test run returns immediately; the parent + /// re-executes this test by name with the fixture variables set. + #[test] + fn sigkill_at_failpoint() { + let Some(store) = std::env::var_os(STORE_ENV) else { + return; + }; + let point = parse_failpoint( + &std::env::var(FAILPOINT_ENV).expect("a failpoint for the crash child"), + ); + let handles = reopen(Path::new(&store)); + let outcome = handles.compact(&request_at(OP, point)); + panic!("the parent did not SIGKILL the run: {outcome:?}"); + } + + /// Kills the child on every path out of the test, including a panic. + struct KillOnDrop(Option); + + impl KillOnDrop { + fn new(child: Child) -> Self { + Self(Some(child)) + } + + fn child_mut(&mut self) -> &mut Child { + self.0.as_mut().expect("a running child") + } + + fn kill(&mut self) -> std::process::ExitStatus { + let mut child = self.0.take().expect("a running child"); + child.kill().expect("SIGKILL the crash child"); + child.wait().expect("the crash child exits") + } + } + + impl Drop for KillOnDrop { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + + /// Waits for the child's gate connection. `yield_now` keeps the bounded + /// wait from sleeping and still notices a child that ended early. + fn wait_for_gate( + listener: &std::os::unix::net::UnixListener, + child: &mut Child, + ) -> std::os::unix::net::UnixStream { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + match listener.accept() { + Ok((stream, _)) => return stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the crash child's status") { + panic!("the crash child ended before the gate: {status}"); + } + assert!( + Instant::now() < deadline, + "the crash child never reached the gate" + ); + std::thread::yield_now(); + } + Err(error) => panic!("crash gate accept: {error}"), + } + } + } + + fn exercise_sigkill(case: Case) { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + let stack = initialize(&handles, 1, live); + let dead = whole_file(&handles, b"a pack no retained root reaches"); + let live_bytes = object_bytes(&handles, live); + let before = generation(&handles); + + // The child owns the writable Store alone; the parent keeps only the + // fixture paths and the expected bytes. + drop(handles); + + let gate = gate_socket(); + let listener = + std::os::unix::net::UnixListener::bind(&gate).expect("the crash gate socket"); + listener + .set_nonblocking(true) + .expect("a nonblocking crash gate"); + let child = Command::new(std::env::current_exe().expect("the test binary")) + .args([ + "--exact", + "real_process::sigkill_at_failpoint", + "--nocapture", + ]) + .env(STORE_ENV, &path) + .env(FAILPOINT_ENV, format!("{:?}", case.failpoint)) + .env(GATE_ENV, &gate) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("a separate crash process"); + let mut child = KillOnDrop::new(child); + + let mut gate_stream = wait_for_gate(&listener, child.child_mut()); + gate_stream + .set_read_timeout(Some(Duration::from_secs(60))) + .expect("a bounded gate read"); + let mut ready = [0u8; 1]; + gate_stream + .read_exact(&mut ready) + .expect("the child reached the crash gate"); + assert_eq!(ready[0], b'R', "the child stopped at the crash gate"); + + // The real crash: SIGKILL while the child is still inside the run. + let status = child.kill(); + assert_eq!( + status.signal(), + Some(9), + "the crash child was SIGKILLed: {status:?}" + ); + drop(gate_stream); + let _ = std::fs::remove_file(&gate); + + assert_crash_state(case, &path, before, OP); + assert_recovery(Recovery { + case, + store: &path, + before, + live, + live_bytes: &live_bytes, + dead, + stack, + op: OP, + }); + } + + macro_rules! sigkill_at { + ($name:ident, $failpoint:expr, $phase:literal, $installed:literal, $target:literal, $visited:literal, $live_sidecars:literal) => { + #[test] + fn $name() { + exercise_sigkill(Case { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + live_sidecars: $live_sidecars, + }); + } + }; + } + + // The mark phase: the open session and admission intent at `BeforeSeal` + // (the real kill leaves the writer's WAL sidecars), and the sealed, + // intent-released Store at `AfterSeal`. + sigkill_at!( + sigkill_before_seal, + CompactFailpoint::BeforeSeal, + "mark", + false, + false, + false, + true + ); + sigkill_at!( + sigkill_after_seal, + CompactFailpoint::AfterSeal, + "mark", + false, + false, + false, + false + ); + // The copy: SQLite runs `VACUUM INTO` as one call that cannot be + // interrupted deterministically, so this arm stops immediately before it. + // The in-process `DuringCopy` arm keeps the torn-target state; this real + // kill proves the crash after the visited scratch exists and before the + // staged file does. + sigkill_at!( + sigkill_during_copy, + CompactFailpoint::BeforeVacuum, + "build", + false, + false, + true, + false + ); + // The verify pass and the rename boundary, at the same failpoints the + // in-process matrix uses. + sigkill_at!( + sigkill_during_verify, + CompactFailpoint::DuringVerify, + "verify", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_during_rename, + CompactFailpoint::DuringRename, + "install", + true, + false, + true, + false + ); + + sigkill_at!( + sigkill_after_build, + CompactFailpoint::AfterBuild, + "build", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_before_install, + CompactFailpoint::BeforeInstall, + "verify", + false, + true, + true, + false + ); + sigkill_at!( + sigkill_after_install, + CompactFailpoint::AfterInstall, + "install", + true, + false, + true, + false + ); + sigkill_at!( + sigkill_during_settle, + CompactFailpoint::DuringSettle, + "install", + true, + false, + false, + false + ); +} + +#[cfg(not(target_os = "linux"))] +#[test] +fn real_process_sigkill_skipped_outside_linux() { + eprintln!( + "skipped: the real-process SIGKILL compaction test requires Linux, not {}", + std::env::consts::OS + ); +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs index 3107ef5ccc..8ffc9736f3 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs @@ -1182,7 +1182,6 @@ fn rebuild_fixture() -> (Temp, SqliteHandles, FixtureIds) { .collect(); let mut delta_bases_sorted = vec![base1, base2]; delta_bases_sorted.sort(); - let mut objects_dead = objects_dead; objects_dead.push(dead_group.id()); objects_dead.extend([ root_a4, diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs new file mode 100644 index 0000000000..1b7a4b344e --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact_matrix.rs @@ -0,0 +1,1952 @@ +//! #41 T12.3a: the SQLite half of the section 12 acceptance matrix, executed +//! and reported. +//! +//! One runner covers AC-01..AC-60: every row is classified as executed here, +//! covered by a named existing test or fixture (asserted to exist), partial +//! (some SQLite arms evidenced, the rest named), deferred (the row is +//! PostgreSQL-only and belongs to T12.3b) or an explicit SQLite gap. The +//! previously uncovered priority rows with SQLite-only contract arms are +//! executed in this file: AC-33 (checkpoint busy and sidecar-at-install), +//! AC-46 (install fence), AC-51 (the recorded op phase), AC-55 (exact SQLite +//! refusal texts) and AC-59 (seal). AC-57 gets a three-run measurement arm; +//! AC-04 gets a seeded reachability property arm (the full random +//! fork/commit/merge/publish history generator is deferred). +//! +//! T12.3c closes the remaining SQLite gaps this runner can execute: AC-06 +//! (a `file_digest` row whose root the rebuild collected), and AC-07/AC-49 +//! (a mount and a compaction exclude each other; a Commit racing the rebuild +//! is fenced or lands). The derived-tree rows (AC-11..AC-14, AC-22), the bind +//! failure (AC-34) and the worker lost reply (AC-36) are named Store +//! service/control tests; their rows point at those files. +//! +//! The table is checked, not decorative: a `covered`/`partial` row names a +//! file and test function that must exist, and an `executed` row names an arm +//! this binary really runs. The report is written to +//! `core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md`; run with +//! `UPDATE_REPORT=1` to regenerate it, and `UPDATE_MEASURE=1` to refresh the +//! three-run measurement fixture. A normal run asserts both files are current +//! and shape-valid. +//! +//! This page adds no contract: the frozen spec section 12 and the acceptance +//! matrix win. Where a row is cross-backend, its PostgreSQL half is deferred +//! by name. +#![cfg(any(target_os = "linux", target_os = "macos"))] +mod support; + +use layerfs_content::{encode_whole_file_payload, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_history::{ + BranchId, CommitId, CommitMeta, CommitOutcome, CommitRequest, CommitVia, ForkRequest, + ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryError, LayerStackId, + LayerStackRecord, StackInitialization, +}; +use layerfs_history::records::{FileDigest, GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_persistence::{ + compact_op_status, CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, + SqliteHandles, SqlitePersistenceProfile, +}; +use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use support::Temp; + +const BINDING: &[u8] = b"sqlite-compact-matrix-binding"; +const REPORT_RELATIVE: &str = "core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md"; +const MEASURE_RELATIVE: &str = + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json"; + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn report_path() -> PathBuf { + repo_root().join(REPORT_RELATIVE) +} + +fn measure_path() -> PathBuf { + repo_root().join(MEASURE_RELATIVE) +} + +// ---- the Store helpers (the T12.1/T12.2 fixture pattern) ------------------- + +fn create_store() -> (Temp, SqliteHandles, PathBuf) { + let temp = Temp::new("sqlite-compact-matrix"); + let path = temp.join("store.sqlite"); + let handles = SqliteHandles::create( + PersistenceConfig::sqlite(path.clone()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + (temp, handles, path) +} + +fn reopen(path: &Path) -> SqliteHandles { + SqliteHandles::open_writable( + PersistenceConfig::sqlite(path.to_path_buf()) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + BINDING, + [71; 32], + ) + .unwrap() +} + +fn request(op: u8) -> CompactRequest { + CompactRequest { + op_id: [op; 16], + ..CompactRequest::default() + } +} + +fn request_at(op: [u8; 16], failpoint: CompactFailpoint) -> CompactRequest { + CompactRequest { + op_id: op, + failpoint: Some(failpoint), + ..CompactRequest::default() + } +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +fn compact_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact") +} + +fn op_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-op") +} + +fn visited_path(store: &Path) -> PathBuf { + with_suffix(store, ".compact-visited") +} + +fn whole_file(handles: &SqliteHandles, bytes: &[u8]) -> ObjectId { + let object = FinalizedObject::new( + ObjectRole::WholeFile, + encode_whole_file_payload(bytes).unwrap(), + ) + .unwrap(); + save_objects(handles, &[object]) +} + +fn save_objects(handles: &SqliteHandles, objects: &[FinalizedObject]) -> ObjectId { + let storage = Storage::new(handles.storage.clone()).unwrap(); + let save = storage.begin_save().unwrap(); + let mut last = None; + for object in objects { + last = Some(object.id()); + save.accept(object.clone()).unwrap(); + } + save.finish().unwrap(); + last.expect("at least one object") +} + +fn filesystem_root(handles: &SqliteHandles, seed: u8) -> ObjectId { + use layerfs_content::{ + filesystem::{ + attributes::PortableMetadata, + directory::{encode_directory_page, DirectoryPage}, + inode::{encode_inode_page, InodePage}, + root::profile_id, + scope_for_seed, FilesystemRoot, + }, + inode_leaf::{InodeKind, InodeMetadata, InodeValue}, + }; + let directory = FinalizedObject::new( + ObjectRole::DirectoryLeaf, + encode_directory_page(&DirectoryPage::Leaf { entries: Vec::new() }).unwrap(), + ) + .unwrap(); + let directory_id = directory.id(); + let inode = FinalizedObject::new( + ObjectRole::InodeLeaf, + encode_inode_page(&InodePage::Leaf { + entries: vec![( + 1, + InodeValue { + kind: InodeKind::Directory, + namespace_ref_count: 0, + content_root: directory_id, + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o755, + mtime_seconds: i64::from(seed), + mtime_nanoseconds: 0, + }), + }, + )], + }) + .unwrap(), + ) + .unwrap() + .with_references(vec![directory_id]); + let inode_id = inode.id(); + let root = FilesystemRoot::new(profile_id(), scope_for_seed([1; 32]), 1, inode_id).unwrap(); + let object = FinalizedObject::new(ObjectRole::FilesystemRoot, root.encode().unwrap()) + .unwrap() + .with_references(vec![inode_id]); + save_objects(handles, &[directory, inode, object]) +} + +fn initialize(handles: &SqliteHandles, tag: u8, genesis: ObjectId) -> LayerStackRecord { + handles + .history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: support::stack(tag), + name: support::name(&format!("project-{tag}")), + scope: layerfs_content::filesystem::scope_for_seed([1; 32]).object(), + profile: layerfs_content::filesystem::root::profile_id(), + genesis_root: genesis, + }) + .unwrap() +} + +fn generation(handles: &SqliteHandles) -> u64 { + handles.history.incarnation() +} + +fn located(handles: &SqliteHandles, id: ObjectId) -> bool { + let mut found = Vec::new(); + handles.storage.locate(&[id], &mut found).unwrap(); + !found.is_empty() +} + +fn object_bytes(handles: &SqliteHandles, id: ObjectId) -> Vec { + let storage = Storage::new(handles.storage.clone()).unwrap(); + storage + .reader() + .unwrap() + .read_objects(&[id]) + .unwrap() + .into_iter() + .next() + .expect("one object") +} + +fn clean(handles: &SqliteHandles) { + let mut out = Vec::new(); + handles + .diagnose( + layerfs_persistence::diagnostic::Options::default(), + &mut |observation| out.push(observation), + ) + .unwrap(); + assert!( + !out.iter().any(|observation| matches!( + observation.outcome, + layerfs_persistence::diagnostic::Outcome::Fail + | layerfs_persistence::diagnostic::Outcome::Unknown + )), + "{out:?}" + ); +} + +/// One live root, one Stack and a wholly dead pack: the minimal `rebuild` arm. +fn rebuild_store() -> (Temp, SqliteHandles, PathBuf, ObjectId) { + let (temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack no retained root reaches"); + (temp, handles, path, live) +} + +// ---- arms executed by the runner ------------------------------------------- + +fn arm_checkpoint_busy() { + let (_temp, handles, path, live) = rebuild_store(); + // A live reader's snapshot in the WAL makes wal_checkpoint(TRUNCATE) + // report busy > 0; step 1 refuses Busy before any artifact or op. + let reader = rusqlite::Connection::open(&path).expect("a second connection"); + let mode: String = reader + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .expect("journal mode"); + assert_eq!( + mode.to_ascii_lowercase(), + "wal", + "the disposable profile is WAL" + ); + reader.execute_batch("BEGIN").unwrap(); + let rows: i64 = reader + .query_row("SELECT count(*) FROM pack", [], |row| row.get(0)) + .unwrap(); + assert!(rows >= 1, "the reader holds a nonempty snapshot"); + + let error = handles.compact(&request(41)).unwrap_err(); + assert!(matches!(error, HistoryError::Busy), "{error:?}"); + assert!(!compact_path(&path).exists(), "no staged Store on Busy"); + assert!(!visited_path(&path).exists(), "no visited scratch on Busy"); + assert!(!op_path(&path).exists(), "no op record on Busy"); + let wal = with_suffix(&path, "-wal"); + assert!( + std::fs::metadata(&wal).map(|meta| meta.len()).unwrap_or(0) > 0, + "the refused seal never unlinks the WAL" + ); + + reader.execute_batch("ROLLBACK").unwrap(); + drop(reader); + // The Store is left wholly old at G and the next run rebuilds it. + let reopened = reopen(&path); + let receipt = reopened.compact(&request(42)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + assert!(located(&final_handles, live), "the live root survives"); + clean(&final_handles); +} + +/// The install hook writes a -mj* super-journal beside the sealed Store: +/// step 7's sidecar fence must refuse Conflict and write nothing. +fn place_super_journal(store: &Path) { + let mut name = store.file_name().expect("a file name").to_os_string(); + name.push("-mj-t123a"); + std::fs::write(store.with_file_name(name), b"t123a super-journal").unwrap(); +} + +fn arm_sidecar_at_install() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let hook_request = CompactRequest { + op_id: [43; 16], + before_install_hook: Some(place_super_journal), + ..CompactRequest::default() + }; + let error = handles.compact(&hook_request).unwrap_err(); + match error { + HistoryError::Conflict(text) => assert_eq!(text, "Store sidecar at install"), + other => panic!("expected the sidecar Conflict, got {other:?}"), + } + assert!(!compact_path(&path).exists(), "Conflict leaves no staged Store"); + assert!( + !visited_path(&path).exists(), + "Conflict leaves no visited scratch" + ); + assert!(!op_path(&path).exists(), "Conflict leaves no op record"); + let mut name = path.file_name().unwrap().to_os_string(); + name.push("-mj-t123a"); + let _ = std::fs::remove_file(path.with_file_name(name)); + + let reopened = reopen(&path); + assert_eq!(generation(&reopened), before, "generation is unchanged"); + assert!(located(&reopened, live), "the live root survives"); +} + +fn arm_status_phases() { + // One stop point per SQLite phase, including an installed one (install). + let cases: [(CompactFailpoint, &str, bool); 4] = [ + (CompactFailpoint::BeforeSeal, "mark", false), + (CompactFailpoint::BeforeCopy, "build", false), + (CompactFailpoint::DuringVerify, "verify", false), + (CompactFailpoint::DuringRename, "install", true), + ]; + let mut seen = BTreeSet::new(); + for (index, (failpoint, phase, installed)) in cases.iter().enumerate() { + let op = [u8::try_from(50 + index).unwrap(); 16]; + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let before = generation(&handles); + let error = handles.compact(&request_at(op, *failpoint)).unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome), "{error:?}"); + let status = compact_op_status(&path).expect("the op record remains"); + assert_eq!(status.op_id, op); + assert_eq!(status.generation, i64::try_from(before).unwrap()); + assert_eq!(status.phase, *phase); + assert!(status.uncertain, "an unfinished run is uncertain"); + seen.insert(status.phase.clone()); + + // A pre-install stop rebuilds at G + 1; an installed one settles. + let reopened = reopen(&path); + let receipt = reopened + .compact(&request(op[0].wrapping_add(20))) + .unwrap(); + if *installed { + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(receipt.generation_before, before + 1); + assert_eq!(receipt.generation_after, before + 1); + } else { + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_before, before); + assert_eq!(receipt.generation_after, before + 1); + } + assert!(compact_op_status(&path).is_none(), "the op settled"); + } + assert_eq!( + seen.iter().map(String::as_str).collect::>(), + vec!["build", "install", "mark", "verify"], + "every SQLite phase was observed" + ); +} + +fn arm_error_codes() { + // A stray staged name no record names is Exists, and is not deleted. + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + let stray = compact_path(&path); + std::fs::write(&stray, b"not a Store").unwrap(); + let error = handles.compact(&request(44)).unwrap_err(); + match error { + HistoryError::Exists(text) => assert_eq!(text, "unrecorded compaction artifact"), + other => panic!("expected Exists, got {other:?}"), + } + assert!(stray.exists(), "a name no record names is not deleted"); + + // The bounded-filesystem failpoint is the exact Capacity("disk space"). + let (_temp2, handles2, path2) = create_store(); + let live2 = filesystem_root(&handles2, 1); + initialize(&handles2, 1, live2); + let _dead2 = whole_file(&handles2, b"a pack the rebuild would collect"); + let before = generation(&handles2); + let error = handles2 + .compact(&request_at([45; 16], CompactFailpoint::CapacityAtBuild)) + .unwrap_err(); + match error { + HistoryError::Capacity(text) => assert_eq!(text, "disk space"), + other => panic!("expected Capacity, got {other:?}"), + } + assert!(!compact_path(&path2).exists()); + assert!(!visited_path(&path2).exists()); + assert!(!op_path(&path2).exists()); + let reopened = reopen(&path2); + assert_eq!(generation(&reopened), before, "the old Store is unchanged"); + assert!(located(&reopened, live2)); +} + +fn arm_seal() { + let (_temp, handles, path, live) = rebuild_store(); + let sealed = handles.seal().expect("the seal"); + let bytes = std::fs::metadata(&path).unwrap().len(); + assert_eq!(sealed.path, path); + assert_eq!(sealed.bytes, bytes); + assert!(sealed.bytes > 0); + assert!( + sealed.sqlite_version.starts_with("3."), + "{}", + sealed.sqlite_version + ); + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(&path, suffix).exists(), + "the seal leaves no {suffix} sidecar" + ); + } + let reopened = reopen(&path); + assert!(located(&reopened, live)); + clean(&reopened); +} + +/// §12 AC-06: a `file_digest` row named a root the rebuild collects; the row +/// goes with its root and a later Save of the same content records the newer +/// root without a stale row blocking it. +fn arm_digest_row_after_collection() { + let (_temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let dead_root = filesystem_root(&handles, 9); + let digest = [0xA6; 32]; + let length = 16 * 1024 * 1024u64; + handles + .history + .record_file_digests(&[FileDigest { + root: *dead_root.as_bytes(), + length, + digest, + }]) + .unwrap(); + assert_eq!( + handles.history.file_roots(&[(digest, length)]).unwrap(), + vec![Some(*dead_root.as_bytes())] + ); + + let receipt = handles.compact(&request(80)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let reopened = reopen(&path); + assert!(!located(&reopened, dead_root), "the named root was collected"); + assert_eq!( + reopened.history.file_roots(&[(digest, length)]).unwrap(), + vec![None], + "the row went with its root" + ); + + // A Save of bytes with that content records the new root; no stale row + // remains to block it. + let new_root = filesystem_root(&reopened, 2); + initialize(&reopened, 2, new_root); + reopened + .history + .record_file_digests(&[FileDigest { + root: *new_root.as_bytes(), + length, + digest, + }]) + .unwrap(); + assert_eq!( + reopened.history.file_roots(&[(digest, length)]).unwrap(), + vec![Some(*new_root.as_bytes())] + ); +} + +/// One mount intent naming its Workspace, Sandbox and machine. +fn mount_intent( + stack: LayerStackId, + branch: BranchId, + op: u8, + workspace: [u8; 32], + sandbox: &str, + machine: &str, +) -> GcIntent { + GcIntent::shared( + [op; 16], + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice().to_vec(), + ) + .in_project(stack.as_slice()) + .named(Some("project-1"), Some("work")) + .placed(Some(workspace), Some(sandbox), Some(machine)) + .owned(Some("runtime-t123c"), Some("boot-t123c")) +} + +fn hex16(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +/// A Stack, a Branch and one wholly dead pack: the fixture the admission arms +/// need. +fn mounted_store() -> (Temp, SqliteHandles, PathBuf, LayerStackId, BranchId) { + let (temp, handles, path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let stack = support::stack(1); + let branch = support::branch_id(1); + let head = handles.history.layer_stack(stack).unwrap().unwrap().head_layer; + handles + .history + .fork(&ForkRequest { + stack, + branch, + name: support::name("work"), + source: ForkSource::Layer(head), + }) + .unwrap(); + let _dead = whole_file(&handles, b"a pack the rebuild would collect"); + (temp, handles, path, stack, branch) +} + +/// §12 AC-07 / AC-49: a mount excludes compaction and a live compaction +/// excludes a mount and a second compaction. +fn arm_mount_and_compaction_exclude_each_other() { + let (_temp, handles, path, stack, branch) = mounted_store(); + let mount = mount_intent(stack, branch, 81, [0x81; 32], "sandbox-t123c", "machine-t123c"); + handles.history.gc_register(&mount).unwrap(); + + let error = handles.compact(&request(82)).unwrap_err(); + match error { + HistoryError::MountsActive(state) => { + assert_eq!(state.holders.len(), 1, "{state:?}"); + let holder = &state.holders[0]; + assert_eq!(holder.kind, GcIntentKind::Mount); + assert_eq!(holder.workspace.as_deref(), Some(hex16(&[0x81; 32]).as_str())); + assert_eq!(holder.sandbox.as_deref(), Some("sandbox-t123c")); + assert_eq!(holder.machine.as_deref(), Some("machine-t123c")); + assert_eq!(holder.op, hex16(&[81; 16])); + } + other => panic!("expected MountsActive, got {other:?}"), + } + assert!(!compact_path(&path).exists(), "a refused run stages no Store"); + assert!(!visited_path(&path).exists(), "and no visited scratch"); + assert!(!op_path(&path).exists(), "and no op record"); + let reopened = reopen(&path); + reopened.history.gc_release(&mount.op_id).unwrap(); + + // A live compaction intent refuses a mount and a second compaction. + let catalog = reopened.history.catalog_id(); + let compaction = GcIntent::operation( + [83; 16], + GcIntentKind::Compaction, + GcScopeKind::Store, + catalog.as_slice().to_vec(), + ) + .owned(Some("runtime-t123c"), Some("boot-t123c")) + .with_phase(Some("build")); + reopened.history.gc_register(&compaction).unwrap(); + let mount_again = mount_intent(stack, branch, 84, [0x84; 32], "s", "m"); + match reopened.history.gc_register(&mount_again).unwrap_err() { + HistoryError::BusyOp(state) => { + assert_eq!(state.kind, GcIntentKind::Compaction); + assert_eq!(state.entity, "store"); + assert_eq!(state.phase.as_deref(), Some("build")); + assert_eq!(state.op, hex16(&[83; 16])); + assert!(!state.uncertain); + } + other => panic!("expected BusyOp, got {other:?}"), + } + let second = reopen(&path).compact(&request(85)).unwrap_err(); + assert!(matches!(second, HistoryError::BusyOp(_)), "{second:?}"); + assert!(!op_path(&path).exists(), "a refused second run leaves no op"); + reopened.history.gc_release(&compaction.op_id).unwrap(); + drop(reopened); + + // The Store is still wholly old and the next run rebuilds it. + let receipt = reopen(&path).compact(&request(86)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + let mount = mount_intent(stack, branch, 87, [0x87; 32], "s", "m"); + final_handles.history.gc_register(&mount).unwrap(); + final_handles.history.gc_release(&mount.op_id).unwrap(); + clean(&final_handles); +} + +/// §12 AC-49: a mount started while a compaction is live is `Busy` with the +/// compaction's operation; the next run adopts the dead owner's leftover. +fn arm_mount_during_compaction_is_busy() { + let (_temp, handles, path, stack, branch) = mounted_store(); + let error = handles + .compact(&request_at([88; 16], CompactFailpoint::BeforeSeal)) + .unwrap_err(); + assert!(matches!(error, HistoryError::UnknownOutcome), "{error:?}"); + + // The stopped run left its live admission intent: a mount meets the + // compaction's operation, not a clean Store. + let reopened = reopen(&path); + let mount = mount_intent(stack, branch, 89, [0x89; 32], "s", "m"); + match reopened.history.gc_register(&mount).unwrap_err() { + HistoryError::BusyOp(state) => { + assert_eq!(state.kind, GcIntentKind::Compaction); + assert_eq!(state.op, hex16(&[88; 16])); + assert_eq!(state.phase.as_deref(), Some("mark")); + } + other => panic!("expected BusyOp, got {other:?}"), + } + + // The next run judges the leftover and rebuilds; the mount then proceeds. + drop(reopened); + let receipt = reopen(&path).compact(&request(90)).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let final_handles = reopen(&path); + final_handles.history.gc_register(&mount).unwrap(); + final_handles.history.gc_release(&mount.op_id).unwrap(); + clean(&final_handles); +} + +/// The Commit thread the install hook starts: `(Commit, root)` identities. +type RebuildCommit = Result<(Vec, [u8; 32]), String>; + +static REBUILD_COMMIT: std::sync::Mutex>> = + std::sync::Mutex::new(None); + +/// Commits one fresh filesystem root on `work`, opening the Store afresh in +/// this thread, so the rebuilt Store keeps a real tree. +fn commit_once(path: &Path) -> Result<(Vec, [u8; 32]), String> { + let handles = reopen(path); + let branch = support::branch_id(1); + let snapshot = handles + .history + .branch_snapshot(branch) + .map_err(|error| format!("{error:?}"))? + .ok_or("the test Branch is absent")?; + let candidate = filesystem_root(&handles, 2); + let outcome = handles.history.commit_branch(&CommitRequest { + branch, + expected_head: snapshot.branch.head_commit, + expected_base: snapshot.branch.base_layer, + expected_root: snapshot.effective_root, + candidate_root: candidate, + profile: snapshot.profile, + scope: snapshot.scope, + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + }); + match outcome { + Ok(CommitOutcome::Committed { record, .. }) => { + Ok((record.id.as_slice().to_vec(), *candidate.as_bytes())) + } + Ok(other) => Err(format!("{other:?}")), + Err(error) => Err(format!("{error:?}")), + } +} + +/// The §10 install hook: start a Commit while the rebuild holds the lane and +/// before the rename, so its outcome is the fence or a landed Commit. +fn commit_during_rebuild(path: &Path) { + let path = path.to_path_buf(); + let handle = std::thread::spawn(move || commit_once(&path)); + *REBUILD_COMMIT.lock().unwrap() = Some(handle); + std::thread::sleep(std::time::Duration::from_millis(150)); +} + +/// §12 AC-49: a Commit started during the rebuild is never lost silently: the +/// install fence refuses its stale handle and the retry lands on `G + 1`. +fn arm_commit_during_rebuild() { + let (_temp, handles, path, _stack, branch) = mounted_store(); + let hook_request = CompactRequest { + op_id: [91; 16], + before_install_hook: Some(commit_during_rebuild), + ..CompactRequest::default() + }; + let receipt = handles.compact(&hook_request).unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + let committed = REBUILD_COMMIT + .lock() + .unwrap() + .take() + .expect("the install hook spawned the Commit") + .join() + .unwrap(); + let (id, root) = match committed { + // The thread may have started after the rename, when its handles are + // already the rebuilt Store's: the Commit landed on `G + 1`. + Ok(committed) => committed, + Err(text) => { + assert!( + text.contains("StaleStore"), + "the racing Commit was refused for another reason: {text}" + ); + commit_once(&path).expect("the Commit retried on G + 1") + } + }; + let reopened = reopen(&path); + let id = CommitId::from_slice(&id).unwrap(); + let record = reopened + .history + .commit(id) + .unwrap() + .expect("the Commit survives the rebuild"); + assert_eq!(record.root.as_bytes(), &root); + assert_eq!( + reopened.history.branch(branch).unwrap().unwrap().head_commit, + Some(id) + ); + clean(&reopened); +} + +fn arm_random_store_shapes() { + // A seeded property arm over reachability shapes: every live object stays + // readable and every wholly dead object is collected, for the retained + // closure to be unchanged. + let mut state = 0x9E37_79B9_7F4A_7C15u64; + for case in 0..6u8 { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let live_count = 12 + usize::try_from(state % 9).unwrap(); + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let dead_count = live_count; + let (_temp, handles, path) = create_store(); + let mut live = Vec::new(); + for index in 0..live_count { + let seed = case * 32 + u8::try_from(index).unwrap() + 1; + let root = filesystem_root(&handles, seed); + let tag = u8::try_from(index + 1).unwrap(); + initialize(&handles, tag, root); + live.push((root, object_bytes(&handles, root))); + } + let mut dead = Vec::new(); + for index in 0..dead_count { + dead.push(whole_file( + &handles, + format!("dead-{case}-{index:03}").as_bytes(), + )); + } + let before = handles.history.mark().unwrap().live_heads_digest(); + let before_generation = generation(&handles); + let receipt = handles + .compact(&request(60 + case)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild, "case {case}"); + assert_eq!(receipt.generation_before, before_generation); + assert_eq!(receipt.generation_after, before_generation + 1); + let reopened = reopen(&path); + for (id, bytes) in &live { + assert!(located(&reopened, *id), "live {id} at case {case}"); + assert_eq!(&object_bytes(&reopened, *id), bytes, "bytes at case {case}"); + } + for id in &dead { + assert!(!located(&reopened, *id), "dead {id} at case {case}"); + } + assert_eq!( + reopened.history.mark().unwrap().live_heads_digest(), + before, + "the retained closure digest at case {case}" + ); + clean(&reopened); + } +} + +/// Three runs of the same rebuild: the E3 shape (median and range), on a small +/// fixture. The 50 GB and PostgreSQL speed/space arms stay deferred (section 13). +fn arm_measurement() { + let mut runs = Vec::new(); + for run in 0..3u8 { + let (_temp, handles, path) = create_store(); + let mut live = Vec::new(); + for index in 0..20 { + let seed = run * 32 + u8::try_from(index).unwrap() + 1; + let root = filesystem_root(&handles, seed); + initialize(&handles, u8::try_from(index + 1).unwrap(), root); + live.push(root); + } + for index in 0..20 { + let _ = whole_file( + &handles, + format!("measure-dead-{run}-{index:03}").as_bytes(), + ); + } + let before = generation(&handles); + let started = Instant::now(); + let receipt = handles + .compact(&request(70 + run)) + .unwrap(); + let micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(receipt.generation_after, before + 1); + assert!(receipt.dead_bytes_bound > 0); + assert!(receipt.physical_shrink > 0); + assert!(receipt.peak_temp_bytes > 0); + let reopened = reopen(&path); + for id in &live { + assert!(located(&reopened, *id)); + } + clean(&reopened); + runs.push(( + micros, + receipt.dead_bytes_bound, + receipt.physical_shrink, + receipt.peak_temp_bytes, + )); + } + let mut times: Vec = runs.iter().map(|run| run.0).collect(); + times.sort_unstable(); + let median = times[1]; + assert!((times[0]..=times[2]).contains(&median)); + let (dead, shrink, peak) = (runs[0].1, runs[0].2, runs[0].3); + for run in &runs[1..] { + assert_eq!(run.1, dead, "dead bytes are deterministic"); + assert_eq!(run.2, shrink, "physical shrink is deterministic"); + assert_eq!(run.3, peak, "peak temp is deterministic"); + } + let json = format!( + "{{\n \"schema\": \"t12.3a-sqlite-compact-measure/1\",\n \ + \"runs\": {},\n \"micros\": [{}, {}, {}],\n \"medianMicros\": {},\n \ + \"minMicros\": {},\n \"maxMicros\": {},\n \"deadBytesBound\": {},\n \ + \"physicalShrink\": {},\n \"peakTempBytes\": {},\n \ + \"liveRoots\": 20,\n \"deadPacks\": 20,\n \ + \"profile\": \"SqlitePersistenceProfile::Disposable (WAL/OFF)\",\n \ + \"platform\": \"{}-{}\",\n \"rustc\": \"{}\"\n}}\n", + runs.len(), + runs[0].0, + runs[1].0, + runs[2].0, + median, + times[0], + times[2], + dead, + shrink, + peak, + std::env::consts::OS, + std::env::consts::ARCH, + rustc_version(), + ); + let path = measure_path(); + if std::env::var_os("UPDATE_MEASURE").is_some() { + std::fs::write(&path, json.as_bytes()).unwrap(); + } else { + let committed = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; run UPDATE_MEASURE=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix", + path.display() + ) + }); + // Shape-valid on any machine: three runs, median inside its own range. + assert!(committed.contains("\"runs\": 3"), "{committed}"); + assert!(committed.contains("\"medianMicros\""), "{committed}"); + assert!(committed.contains("\"minMicros\""), "{committed}"); + assert!(committed.contains("\"maxMicros\""), "{committed}"); + } +} + +fn rustc_version() -> String { + std::process::Command::new("rustc") + .arg("--version") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|text| text.trim().to_owned()) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +const ARMS: [(&str, fn()); 11] = [ + ("arm_checkpoint_busy", arm_checkpoint_busy), + ("arm_sidecar_at_install", arm_sidecar_at_install), + ("arm_status_phases", arm_status_phases), + ("arm_error_codes", arm_error_codes), + ("arm_seal", arm_seal), + ("arm_digest_row_after_collection", arm_digest_row_after_collection), + ("arm_mount_and_compaction_exclude_each_other", arm_mount_and_compaction_exclude_each_other), + ("arm_mount_during_compaction_is_busy", arm_mount_during_compaction_is_busy), + ("arm_commit_during_rebuild", arm_commit_during_rebuild), + ("arm_random_store_shapes", arm_random_store_shapes), + ("arm_measurement", arm_measurement), +]; + +// ---- the acceptance matrix ------------------------------------------------ + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Backend { + Both, + Sqlite, + Pg, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Status { + /// An arm in this binary runs the SQLite half. + Executed, + /// A named existing test or fixture runs it. + Covered, + /// Some SQLite arms are named; the rest is named as missing. + Partial, + /// PostgreSQL-only (or the PostgreSQL half): T12.3b. + Deferred, + /// SQLite-applicable with no named evidence yet. + Gap, +} + +struct Row { + id: &'static str, + backend: Backend, + tier: &'static str, + status: Status, + /// `repo/path.rs#test_fn`, `repo/path.json`, or `runner#arm_name`. + evidence: &'static [&'static str], + note: &'static str, +} + +const MATRIX: [Row; 60] = [ + Row { id: "AC-01", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_verify", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle"], + note: "The SQLite section 9 crash matrix through named failpoints; a real kill -9 of a separate daemon process is T12.2's in-process stop points, not run here." }, + Row { id: "AC-02", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "The SQLite arm runs through the suite; the converted 1.0 Store arm and the PostgreSQL backend matrix are deferred to T12.3b." }, + Row { id: "AC-03", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_wal_switch", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_after_install"], + note: "SQLite mark/build/verify/install and WAL-switch stop points; lost COMMIT and power-loss arms belong to the PostgreSQL/fsync halves." }, + Row { id: "AC-04", backend: Backend::Both, tier: "E3", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "runner#arm_random_store_shapes"], + note: "Deterministic closure fixture plus a seeded six-history reachability property arm; the full random fork/commit/merge/publish/removal generator is not built here." }, + Row { id: "AC-05", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "runner#arm_random_store_shapes"], + note: "Full doctor (including the diagnostic scan) on the compacted Store, also in the property arm." }, + Row { id: "AC-06", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_digest_row_after_collection"], + note: "A Save's `file_digest` row named a root the rebuild collected; the row goes with its root and recording the same content again succeeds. The PostgreSQL half is T12.3b." }, + Row { id: "AC-07", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_mount_and_compaction_exclude_each_other", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction"], + note: "SQLite half executed: bypass gives `StaleStore`, a mount blocks compaction (`MountsActive`, naming the holder) and a live compaction refuses a mount and a second compaction (`BusyOp`). The open one-shot SDK transaction arm is PostgreSQL, T12.3b." }, + Row { id: "AC-08", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner"], + note: "One winner; the loser sees MountsActive, and a removed Branch is Missing." }, + Row { id: "AC-09", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_removed_branch_is_hidden_and_the_epoch_bumps_once", + "engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "api/crates/layerfs-api/tests/gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches"], + note: "Reserved main, absent Branch and holder details are asserted; project-side name collisions are in AC-30." }, + Row { id: "AC-10", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position"], + note: "Removed/UpToDate codes and the frontier; 2.0 frees no bytes for the removed path." }, + Row { id: "AC-11", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount"], + note: "Non-FF whiteouts, the fast-forward control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute. A merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed." }, + Row { id: "AC-12", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation", + "engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit"], + note: "A removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept, and only an ordinary edit conflict on it is reported." }, + Row { id: "AC-13", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit"], + note: "The earlier Commit on an unmerged Branch is hidden by the marker; a Commit after the frontier does not inherit the path." }, + Row { id: "AC-14", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount", + "engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes"], + note: "A Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount; its next Commit records the deletion." }, + Row { id: "AC-15", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Retention of the only-reaching-path object is proven; marker hiding in reads/listings and derived whiteouts are not in one named test." }, + Row { id: "AC-16", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "T12.1 pins a retained retired Branch name and a collected unreferenced one; the branch.remove-then-rebuild arm is exercised through those markers." }, + Row { id: "AC-17", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/kept_objects.rs#a_delta_base_is_enqueued_and_reported"], + note: "Depth-2 chain copied, pack ids preserved." }, + Row { id: "AC-18", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Collected digest row gone, retained one kept." }, + Row { id: "AC-19", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers", + "engine/crates/layerfs-persistence/tests/path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position"], + note: "Path-grammar refusals and the marker frontier are named; the full form table (symlink/hardlink, >4096 bytes, >256 components) is not one test." }, + Row { id: "AC-20", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date", + "engine/crates/layerfs-persistence/tests/path_removal.rs#an_unknown_project_is_missing"], + note: "UpToDate and never-existing paths are named; the re-creation and other-project arms are partial." }, + Row { id: "AC-21", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back", + "engine/crates/layerfs-persistence/tests/epoch_fence.rs#a_concurrent_commit_and_removal_have_one_winner"], + note: "One-shot writes racing removals roll back as Conflict; the merge-racing-remove_file arm is not named." }, + Row { id: "AC-22", backend: Backend::Both, tier: "E3", status: Status::Covered, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible", + "engine/crates/layerfs-persistence/tests/epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back"], + note: "A deterministic barrier starts a merge and a `remove_file` together for three rounds: the loser sees the marker or gets `Conflict` and retries with the whiteout, and a tree derived afterwards never shows the path. The PostgreSQL half is T12.3b." }, + Row { id: "AC-23", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/project_delete.rs#deleting_a_project_hides_it_and_marks_once"], + note: "Deleted-project rows and markers are collected by the rebuild; a dedicated path_removed cleanup arm is not named." }, + Row { id: "AC-24", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order", + "engine/crates/layerfs-persistence/tests/gc_backfill.rs#existing_serials_are_kept_and_new_ones_start_above_the_maximum"], + note: "Serials backfilled, next_history_serial never falls." }, + Row { id: "AC-25", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Retained branch_retired/layer_stack_retired markers survive; collected rows take their markers." }, + Row { id: "AC-26", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/project_delete.rs#retry_classifies_retired_live_and_absent", + "engine/crates/layerfs-persistence/tests/project_delete.rs#deleting_a_project_hides_it_and_marks_once"], + note: "Hidden, entry present, retry returns the stored deleted_ms." }, + Row { id: "AC-27", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/project_delete.rs#paging_skips_a_hidden_project", + "engine/crates/layerfs-persistence/tests/project_delete.rs#every_layer_stack_lookup_carries_the_marker_filter"], + note: "Stale home entries stay out of list/lookup." }, + Row { id: "AC-28", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase C cleanup and its failpoints: T12.3b." }, + Row { id: "AC-29", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved", + "engine/crates/layerfs-persistence/tests/admission.rs#a_mount_blocks_all_three_removals_until_released"], + note: "Owner liveness and Unresolved settlement; machine_stopped and the other-home intent are partial." }, + Row { id: "AC-30", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names", + "api/crates/layerfs-api/tests/gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect"], + note: "Holder detail and refusal codes are named; the full exact-message matrix (AC-55) is partial." }, + Row { id: "AC-31", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["api/crates/layerfs-api/tests/gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log", + "api/crates/layerfs-api/tests/gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt"], + note: "Byte-count path arguments and the keyed trace hash are asserted." }, + Row { id: "AC-32", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-persistence/tests/admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved"], + note: "Live/uncertain ownership is named; live-but-unreachable, machine_stopped and another home's intent are partial." }, + Row { id: "AC-33", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_checkpoint_busy", "runner#arm_sidecar_at_install"], + note: "A live reader makes wal_checkpoint(TRUNCATE) Busy with no artifact and no op; a -mj* super-journal at install is Conflict with no writes." }, + Row { id: "AC-34", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active"], + note: "A committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains. The failed-delete sub-arm is not inducible on SQLite: the storage schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register too; the explicit `Cleanup` report is covered by the seam's code path, not executed here. The PostgreSQL half is T12.3b." }, + Row { id: "AC-35", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL advisory-key refcount: T12.3b." }, + Row { id: "AC-36", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["control/crates/layerfs-control/tests/project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry", + "control/crates/layerfs-store/tests/project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry", + "control/crates/layerfs-store/tests/project_delete.rs#an_absent_stack_finishes_a_stale_entry"], + note: "A lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent. The PostgreSQL half is T12.3b." }, + Row { id: "AC-37", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/gc_trace.rs#the_redacted_set_covers_every_path_bearing_call", + "api/crates/layerfs-api/tests/gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt"], + note: "Redaction and the keyed hash are named; salt backfill on an upgraded Store is partial." }, + Row { id: "AC-38", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle"], + note: "The SQLite artifact set is removed exactly; the PostgreSQL schema/name arms are T12.3b." }, + Row { id: "AC-39", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order"], + note: "One project_epoch row per stack, gc_op not copied, next_history_serial never lowered." }, + Row { id: "AC-40", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/kept_objects.rs#the_digest_is_stable_and_moves_with_a_head"], + note: "SQLite live_heads_digest is identical before/after and moves with a head; PostgreSQL agreement is deferred." }, + Row { id: "AC-41", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase M mark transfer: T12.3b." }, + Row { id: "AC-42", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL reserved pack/Phase B: T12.3b." }, + Row { id: "AC-43", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL Phase C cleanup resume: T12.3b." }, + Row { id: "AC-44", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL privileges and lost COMMIT reply: T12.3b." }, + Row { id: "AC-45", backend: Backend::Sqlite, tier: "E1", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#group_rows_indexed_layout_opens_compacts_and_takes_none", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#acquisition_layout_opens_compacts_and_clears_working_rows", + "engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs#explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order"], + note: "Monolithic/GroupRows/GroupRowsIndexed and the acquisition variants compact; working rows are cleared." }, + Row { id: "AC-46", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_sidecar_at_install", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction"], + note: "Sealed-state/sidecar change at install gives Conflict with no writes; a replaced Store gives StaleStore. The in-flight-transaction exclusion is a documented lane limit, not probed." }, + Row { id: "AC-47", backend: Backend::Sqlite, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_settle", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted", + "engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript"], + note: "Crash leaves the full set, the next compact removes it, none leaves no artifact, a stray name is Exists." }, + Row { id: "AC-48", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#an_empty_store_takes_the_none_path", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_capacity_failure_leaves_the_store_unchanged"], + note: "9/10/30 percent, empty-store zeroReason, capacity abort; verify-failure and re-refine arms are partial." }, + Row { id: "AC-49", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_mount_during_compaction_is_busy", + "runner#arm_commit_during_rebuild", + "engine/crates/layerfs-persistence/tests/admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner"], + note: "SQLite half executed: a mount during compact is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild gets `StaleStore` or lands, and its retry is never lost. The PG Phase A/Phase C arms are T12.3b." }, + Row { id: "AC-50", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["api/crates/layerfs-api/tests/calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover", + "api/crates/layerfs-api/tests/gc_calls.rs#the_four_gc_calls_take_ids_only"], + note: "Folder scope refuses the four calls; the admin profile is the only allowance." }, + Row { id: "AC-51", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_status_phases", + "control/crates/layerfs-store/tests/compact_report_status.rs#status_projects_the_sidecar_phase_and_state", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_seal"], + note: "The op sidecar records mark/build/verify/install (runner); T10 projects it through store.status, and the named control test asserts the phase, state and phases list." }, + Row { id: "AC-52", backend: Backend::Both, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json", + "control/crates/layerfs-store/tests/compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape"], + note: "Every section 11 field pinned with units and sampling point (golden) and rendered by the T10 wire-shape test; markBytes is a metric, not a result field." }, + Row { id: "AC-53", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["api/crates/layerfs-api/tests/project_export.rs#export_omits_a_path_hidden_by_remove_file", + "api/crates/layerfs-api/tests/gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers"], + note: "One hidden-path export case is named; the full read-surface table (read/stat/list/glob/diff/revision/edit/grep/blame/cp/apply) is not executed." }, + Row { id: "AC-54", backend: Backend::Both, tier: "E2", status: Status::Partial, + evidence: &["control/crates/layerfs-store/tests/restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install", + "control/crates/layerfs-store/tests/restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it"], + note: "Restore install guards and restored records are named; restore before/after a removal and the stale-entry listing arm are partial. The restore-window arm needs a lock-capable filesystem (ext4/xfs/btrfs), so on this Sandbox it reports Unsupported while the other two cases pass." }, + Row { id: "AC-55", backend: Backend::Both, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_error_codes", "runner#arm_checkpoint_busy", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted"], + note: "Exact SQLite texts asserted: Busy, Conflict(\"Store sidecar at install\"), Exists(\"unrecorded compaction artifact\"), Capacity(\"disk space\"); the full engine/API message matrix stays partial." }, + Row { id: "AC-56", backend: Backend::Both, tier: "E3", status: Status::Partial, + + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_golden.json"], + note: "The literal SQLite vector is pinned (absent head 33 zero bytes); the shared PostgreSQL literal and the three-run E3 shape are deferred." }, + Row { id: "AC-57", backend: Backend::Both, tier: "E3", status: Status::Partial, + evidence: &["runner#arm_measurement", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_measure.json", + "engine/crates/layerfs-persistence/tests/sqlite_compact_capacity.rs#capacity_measure", + "engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json"], + note: "The tiny three-run arm is median 28.2 ms. T-G61 measured a real 9.987 GB Store with 3 runs per arm: `none` at 8.88% dead (median 149.318 s) and `rebuild` at 10.00% dead (median 246.371 s, mark 149.089 s / build 94.317 s / verify 0.312 s / install 1.916 s), plus one WAL/FULL rebuild at 284.078 s; profile, host and toolchain are declared in the fixture, and the warm page cache is declared below. The section 13 50 GB arms, the 30% arm, each removal type, the plain `VACUUM INTO` comparison and PostgreSQL timings remain deferred/[E]." }, + Row { id: "AC-58", backend: Backend::Sqlite, tier: "E2", status: Status::Covered, + evidence: &["engine/crates/layerfs-persistence/tests/sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_before_copy", + "engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs#crash_during_copy"], + note: "none deletes the scratch and leaves no .compact; rebuild keeps the visited scratch through the build; step 2 writes only on rebuild." }, + Row { id: "AC-59", backend: Backend::Sqlite, tier: "E2", status: Status::Executed, + evidence: &["runner#arm_checkpoint_busy", "runner#arm_seal", + "engine/crates/layerfs-persistence/tests/sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_stale"], + note: "busy != 0 refuses Busy and the WAL is never unlinked; a successful seal records (size, mtime, counter) as fenced by the StaleStore arm." }, + Row { id: "AC-60", backend: Backend::Pg, tier: "E2", status: Status::Deferred, + evidence: &[], note: "PostgreSQL synchronous standby handling: T12.3b." }, +]; + +// ---- report --------------------------------------------------------------- + +fn evidence_label(entry: &str) -> String { + if let Some(arm) = entry.strip_prefix("runner#") { + return format!("this runner #{arm}"); + } + let (file, function) = entry.split_once('#').unwrap_or((entry, "")); + let base = file.rsplit('/').next().unwrap_or(file); + // T-G61's opt-in capacity suite is labelled with its ticket in the matrix + // evidence; its measured numbers are in the notes and run-environment page. + if base == "sqlite_compact_capacity.rs" { + return format!("T-G61 {base}#{function}"); + } + if function.is_empty() { + base.to_owned() + } else { + format!("{base}#{function}") + } +} + +fn backend_label(backend: Backend) -> &'static str { + match backend { + Backend::Both => "both", + Backend::Sqlite => "SQLite", + Backend::Pg => "PG", + } +} + +fn status_label(status: Status) -> &'static str { + match status { + Status::Executed => "executed", + Status::Covered => "covered", + Status::Partial => "partial", + Status::Deferred => "deferred (T12.3b)", + Status::Gap => "gap", + } +} + +/// The crash suite's test count and its real-process SIGKILL arms, read from +/// its source so the report's T-G28/T-G60 note cannot drift when an arm is +/// added: the in-process `crash_at!` arms, the `sigkill_at!` arms and the +/// crash child itself are every test the suite declares on Linux. +fn crash_suite_counts() -> (usize, usize) { + let path = + repo_root().join("engine/crates/layerfs-persistence/tests/sqlite_compact_crash.rs"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; the crash suite the report counts", + path.display() + ) + }); + let sigkill_arms = text.matches("sigkill_at!(").count(); + let total = text.matches("crash_at!(").count() + + sigkill_arms + + text.matches("fn sigkill_at_failpoint(").count(); + (total, sigkill_arms) +} + +fn generate_report() -> String { + use std::fmt::Write as _; + let count = |status: Status| MATRIX.iter().filter(|row| row.status == status).count(); + let both = MATRIX.iter().filter(|row| row.backend == Backend::Both).count(); + let sqlite = MATRIX + .iter() + .filter(|row| row.backend == Backend::Sqlite) + .count(); + let pg = MATRIX.iter().filter(|row| row.backend == Backend::Pg).count(); + let mut out = String::new(); + writeln!(out, "# T12.3a: SQLite acceptance-matrix runner report").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "> Generated by `cargo test -p layerfs-persistence --test sqlite_compact_matrix`." + ) + .unwrap(); + writeln!( + out, + "> The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) section 12 and" + ) + .unwrap(); + writeln!( + out, + "> [`gc-acceptance-matrix.md`](gc-acceptance-matrix.md) AC-01..AC-60 win; this" + ) + .unwrap(); + writeln!( + out, + "> page records evidence and adds no contract. Regenerate with" + ) + .unwrap(); + writeln!( + out, + "> `UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix`." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Scope and counts").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- Rows: {} (AC-01..AC-60); backend split: both {both}, SQLite-only {sqlite}, PG-only {pg}.", + MATRIX.len() + ) + .unwrap(); + writeln!( + out, + "- Executed here: {}; covered by a named existing test/fixture: {}; partial: {}; deferred to T12.3b: {}; SQLite gaps: {}.", + count(Status::Executed), + count(Status::Covered), + count(Status::Partial), + count(Status::Deferred), + count(Status::Gap) + ) + .unwrap(); + let (crash_tests, crash_sigkill_arms) = crash_suite_counts(); + writeln!( + out, + "- **T-G28/T-G60 follow-up.** T-G28 later added the real-process SIGKILL arms" + ) + .unwrap(); + writeln!( + out, + " and T-G60 extended them to mark, copy, verify and rename, so" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_crash` now has {crash_tests} tests ({crash_sigkill_arms} real-process SIGKILL arms and the" + ) + .unwrap(); + writeln!( + out, + " child placeholder), not the 13 recorded below. Those arms still kill a" + ) + .unwrap(); + writeln!( + out, + " re-executed test child, so T-G65 added" + ) + .unwrap(); + writeln!( + out, + " `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs`: 8 SIGKILL" + ) + .unwrap(); + writeln!( + out, + " arms on the built `layerfs store compact` process at mark, build, verify and" + ) + .unwrap(); + writeln!( + out, + " install, so AC-01 is `covered` (the matrix now reads covered 30 / partial" + ) + .unwrap(); + writeln!( + out, + " 16). This page keeps the T12.3c run's original counts." + ) + .unwrap(); + writeln!( + out, + "- A `covered`/`partial` row names a file and test function that this runner" + ) + .unwrap(); + writeln!( + out, + " asserts exists; an `executed` row names an arm this binary really runs." + ) + .unwrap(); + writeln!( + out, + "- A cross-backend row records its SQLite half only; every PostgreSQL half" + ) + .unwrap(); + writeln!( + out, + " and the PostgreSQL-only rows are deferred to T12.3b by name." + ) + .unwrap(); + writeln!( + out, + "- **T-G61 capacity follow-up.** A separate opt-in test," + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_capacity.rs#capacity_measure` (ignored by default), measured a" + ) + .unwrap(); + writeln!( + out, + " real **9.987 GB** Store: `none` at 8.88% dead (three runs, median 149.318 s)" + ) + .unwrap(); + writeln!( + out, + " and `rebuild` at 10.00% dead (three runs, median 246.371 s; mark 149.089 s," + ) + .unwrap(); + writeln!( + out, + " build 94.317 s, verify 0.312 s, install 1.916 s; dead 1.011 GB, shrink" + ) + .unwrap(); + writeln!( + out, + " 1.017 GB, peak temp 9.144 GB), plus one WAL/FULL rebuild (284.078 s). The" + ) + .unwrap(); + writeln!( + out, + " numbers are in `tests/fixtures/sqlite_compact_capacity_measure.json`; the" + ) + .unwrap(); + writeln!( + out, + " 50 GB arms and the 30% arm remain [E]." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Matrix (SQLite half)").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "| Case | Backend | Matrix tier | Disposition | Evidence | Note |" + ) + .unwrap(); + writeln!(out, "|---|---|---|---|---|---|").unwrap(); + for row in &MATRIX { + let evidence = if row.evidence.is_empty() { + "-".to_owned() + } else { + row.evidence + .iter() + .map(|entry| evidence_label(entry)) + .collect::>() + .join("; ") + }; + writeln!( + out, + "| {} | {} | {} | {} | {} | {} |", + row.id, + backend_label(row.backend), + row.tier, + status_label(row.status), + evidence, + row.note + ) + .unwrap(); + } + writeln!(out).unwrap(); + writeln!(out, "## Executed in this runner").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- AC-33 / AC-59: `arm_checkpoint_busy` - a live reader makes" + ) + .unwrap(); + writeln!( + out, + " `wal_checkpoint(TRUNCATE)` report busy, the compaction refuses `Busy`," + ) + .unwrap(); + writeln!( + out, + " leaves no `.compact`, `.compact-visited` or `.compact-op`, and the WAL is" + ) + .unwrap(); + writeln!( + out, + " never unlinked; the next run rebuilds at `G -> G+1`." + ) + .unwrap(); + writeln!( + out, + "- AC-33 / AC-46: `arm_sidecar_at_install` - a `-mj*` super-journal placed" + ) + .unwrap(); + writeln!( + out, + " at install is `Conflict(\"Store sidecar at install\")` with no artifact," + ) + .unwrap(); + writeln!( + out, + " no op and an unchanged generation." + ) + .unwrap(); + writeln!( + out, + "- AC-51: `arm_status_phases` - stop points at mark, build, verify and install" + ) + .unwrap(); + writeln!( + out, + " record the same phase in `.compact-op`; every phase is observed." + ) + .unwrap(); + writeln!( + out, + "- AC-55: `arm_error_codes` - exact `Exists(\"unrecorded compaction artifact\")` and" + ) + .unwrap(); + writeln!( + out, + " `Capacity(\"disk space\")` texts, with no artifact after the capacity abort." + ) + .unwrap(); + writeln!( + out, + "- AC-59: `arm_seal` - a successful seal leaves no sidecar, reports the main-file" + ) + .unwrap(); + writeln!( + out, + " bytes and a SQLite version, and the file reopens." + ) + .unwrap(); + writeln!( + out, + "- AC-04 / AC-05: `arm_random_store_shapes` - a seeded property arm over six" + ) + .unwrap(); + writeln!( + out, + " reachability shapes: every live object stays readable, every wholly dead one" + ) + .unwrap(); + writeln!( + out, + " is collected, the retained-closure digest is unchanged and the doctor is clean." + ) + .unwrap(); + writeln!( + out, + "- AC-06: `arm_digest_row_after_collection` - a `file_digest` row named a" + ) + .unwrap(); + writeln!( + out, + " root the rebuild collected, so the row goes; recording the same content" + ) + .unwrap(); + writeln!( + out, + " again succeeds and points at the new root." + ) + .unwrap(); + writeln!( + out, + "- AC-07 / AC-49: `arm_mount_and_compaction_exclude_each_other` - a live mount" + ) + .unwrap(); + writeln!( + out, + " makes compaction `MountsActive` naming it, and a live compaction makes a" + ) + .unwrap(); + writeln!( + out, + " mount and a second compaction `BusyOp`; the refused runs leave no op." + ) + .unwrap(); + writeln!( + out, + "- AC-49: `arm_mount_during_compaction_is_busy` - the stopped run's live" + ) + .unwrap(); + writeln!( + out, + " compaction intent refuses a mount with `BusyOp`, and the next run adopts" + ) + .unwrap(); + writeln!( + out, + " the leftover and rebuilds." + ) + .unwrap(); + writeln!( + out, + "- AC-49: `arm_commit_during_rebuild` - a Commit started inside the install" + ) + .unwrap(); + writeln!( + out, + " fence is `StaleStore` or lands on `G + 1`; a retry is never lost." + ) + .unwrap(); + writeln!( + out, + "- AC-57: `arm_measurement` - three runs of the same rebuild with median and" + ) + .unwrap(); + writeln!( + out, + " range in `engine/crates/layerfs-persistence/tests/fixtures/" + ) + .unwrap(); + writeln!( + out, + " sqlite_compact_measure.json` (refresh with `UPDATE_MEASURE=1`). The separate" + ) + .unwrap(); + writeln!( + out, + " T-G61 `sqlite_compact_capacity.rs#capacity_measure` measures a real 9.987 GB" + ) + .unwrap(); + writeln!( + out, + " Store (three `none` + three `rebuild` runs, one WAL/FULL control) into" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_capacity_measure.json`; it is opt-in because the fixture is" + ) + .unwrap(); + writeln!( + out, + " ~10 GB and the ignored test is not part of a normal suite." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Deferred to T12.3b (PostgreSQL)").unwrap(); + writeln!(out).unwrap(); + for row in MATRIX.iter().filter(|row| row.status == Status::Deferred) { + writeln!(out, "- {} - {}", row.id, row.note).unwrap(); + } + writeln!( + out, + "- The PostgreSQL half of every `both` row (digest agreement, artifact" + ) + .unwrap(); + writeln!( + out, + " names, Phase M/B/C, standby, lost reply, 50 GB arms)." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## SQLite gaps (not claimed by this runner)").unwrap(); + writeln!(out).unwrap(); + if MATRIX.iter().all(|row| row.status != Status::Gap) { + writeln!( + out, + "- none; every SQLite gap this runner can execute is executed or covered." + ) + .unwrap(); + } else { + for row in MATRIX.iter().filter(|row| row.status == Status::Gap) { + writeln!(out, "- {} - {}", row.id, row.note).unwrap(); + } + } + writeln!(out).unwrap(); + writeln!(out, "## E3 arms").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- AC-04: the T12.1 closure fixture (deterministic) plus the seeded" + ) + .unwrap(); + writeln!( + out, + " reachability property arm; the full random history generator is deferred." + ) + .unwrap(); + writeln!( + out, + "- AC-22: a deterministic barrier starts a merge and a `remove_file` together" + ) + .unwrap(); + writeln!( + out, + " for three rounds; the loser retries and a derived tree never shows the path." + ) + .unwrap(); + writeln!( + out, + "- AC-56: the SQLite literal vector is pinned by the T12.1 golden; the shared" + ) + .unwrap(); + writeln!(out, " PostgreSQL literal is deferred.").unwrap(); + writeln!( + out, + "- AC-57: three-run measurement arm executed on this Sandbox (small fixture)," + ) + .unwrap(); + writeln!( + out, + " plus T-G61's real 9.987 GB capacity fixture with three runs per arm" + ) + .unwrap(); + writeln!( + out, + " (`none` 149.318 s median, `rebuild` 246.371 s median, phase sums recorded)." + ) + .unwrap(); + writeln!( + out, + " Its 50 GB projection, the 30% arm, PG `ACCESS EXCLUSIVE` duration and the" + ) + .unwrap(); + writeln!( + out, + " plain `VACUUM INTO` comparison are deferred/[E]." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Status and report projection").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "T10 (merged from main after this branch forked) projects `.compact-op`" + ) + .unwrap(); + writeln!( + out, + "through `store.status`: `op {{id, name, phase, started, uncertain}}`, the" + ) + .unwrap(); + writeln!( + out, + "busy/unresolved `state` and the backend `phases`. The named control tests" + ) + .unwrap(); + writeln!( + out, + "in AC-51 and AC-52 assert that projection and the snake_case section 11" + ) + .unwrap(); + writeln!( + out, + "render; the runner still asserts the recorded SQLite phases directly, so" + ) + .unwrap(); + writeln!( + out, + "AC-51 does not depend on the projection alone." + ) + .unwrap(); + writeln!(out).unwrap(); + writeln!(out, "## Run environment").unwrap(); + writeln!(out).unwrap(); + writeln!( + out, + "- T-G61 capacity run (separate opt-in test): Docker Sandbox `g61sbx`" + ) + .unwrap(); + writeln!( + out, + " (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 overlay, 4.0 TB free), rustc" + ) + .unwrap(); + writeln!( + out, + " 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF) plus one" + ) + .unwrap(); + writeln!( + out, + " `Durable` control, `CARGO_TARGET_DIR=/var/tmp/target-g61`, fixture under" + ) + .unwrap(); + writeln!( + out, + " `/var/tmp/g61big`; the test ran 1678 s. Caches: the ~10 GB Store fits in the" + ) + .unwrap(); + writeln!( + out, + " 135 GB page cache and no cache drop ran between the three runs of an arm;" + ) + .unwrap(); + writeln!( + out, + " the mark is CPU-bound at ~62 MB/s, not cache-bound." + ) + .unwrap(); + writeln!( + out, + "- Docker Sandbox `t123csbx` (Linux x86_64, tmpfs `/tmp`, overlayfs" + ) + .unwrap(); + writeln!( + out, + " `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable`" + ) + .unwrap(); + writeln!( + out, + " (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`." + ) + .unwrap(); + writeln!( + out, + "- Verified here: `sqlite_compact_matrix` (1 test); the persistence suites" + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact` (20), `sqlite_compact_crash` (13)," + ) + .unwrap(); + writeln!( + out, + " `sqlite_compact_fixtures` (1), `admission` (5), `path_removal` (3)," + ) + .unwrap(); + writeln!( + out, + " `branch_removal` (3), `project_delete` (5), `epoch_fence` (7)," + ) + .unwrap(); + writeln!( + out, + " `gc_backfill` (4), `kept_objects` (5); the Store-service path suites" + ) + .unwrap(); + writeln!( + out, + " `sdk_path_merge_removal` (5), `sdk_mount_bind_failure` (1)," + ) + .unwrap(); + writeln!( + out, + " `sdk_path_whiteouts` (7), `sdk_path_hidden_semantics` (2)," + ) + .unwrap(); + writeln!( + out, + " `sdk_path_removal` (2); and `layerfs-control --features postgresql" + ) + .unwrap(); + writeln!( + out, + " project_delete_worker` (5), `layerfs-store --features postgresql" + ) + .unwrap(); + writeln!( + out, + " project_delete` (15) and `compact_report_status` (3)." + ) + .unwrap(); + writeln!( + out, + " All of those pass on the merged branch." + ) + .unwrap(); + writeln!( + out, + "- The control `restore_install` arm that opens a Store needs ext4/xfs/btrfs;" + ) + .unwrap(); + writeln!( + out, + " this Sandbox has no such direct filesystem, so that one case reports" + ) + .unwrap(); + writeln!( + out, + " `Unsupported` while its other two cases pass. It is an environment limit," + ) + .unwrap(); + writeln!( + out, + " not a matrix result; the cited API tests are validated by source name only" + ) + .unwrap(); + writeln!( + out, + " in this run." + ) + .unwrap(); + out +} + +fn validate_registry(executed: &BTreeSet) { + assert_eq!(MATRIX.len(), 60, "AC-01..AC-60 are 60 rows"); + for (index, row) in MATRIX.iter().enumerate() { + let expected = format!("AC-{:02}", index + 1); + assert_eq!(row.id, expected, "rows are in id order"); + if row.status == Status::Deferred { + assert!( + row.backend == Backend::Pg || row.note.contains("PostgreSQL"), + "{} is deferred but not PostgreSQL-only", + row.id + ); + continue; + } + if row.status == Status::Gap { + assert!( + row.evidence.is_empty(), + "{} is a gap but names evidence", + row.id + ); + continue; + } + assert!(!row.evidence.is_empty(), "{} has no evidence", row.id); + for entry in row.evidence { + if let Some(arm) = entry.strip_prefix("runner#") { + assert!(executed.contains(arm), "{} names unexecuted arm {arm}", row.id); + continue; + } + let (file, function) = entry.split_once('#').unwrap_or((entry, "")); + let path = repo_root().join(file); + assert!(path.exists(), "{}: evidence file missing: {file}", row.id); + if !function.is_empty() { + let text = std::fs::read_to_string(&path).unwrap(); + assert!( + text.contains(function), + "{}: {file} does not name {function}", + row.id + ); + } + } + if row.status == Status::Executed { + assert!( + row.evidence.iter().any(|entry| entry.starts_with("runner#")), + "{} is executed but names no arm", + row.id + ); + } + } +} + +#[test] +fn the_sqlite_half_of_the_acceptance_matrix_is_executed_and_reported() { + let mut executed = BTreeSet::new(); + for (name, arm) in ARMS { + arm(); + executed.insert(name.to_owned()); + } + validate_registry(&executed); + + let report = generate_report(); + let path = report_path(); + if std::env::var_os("UPDATE_REPORT").is_some() { + std::fs::write(&path, report.as_bytes()).unwrap(); + } else { + let committed = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "missing {}; run UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix", + path.display() + ) + }); + assert_eq!( + committed, report, + "the report is stale; rerun with UPDATE_REPORT=1" + ); + } + + let count = |status: Status| MATRIX.iter().filter(|row| row.status == status).count(); + println!( + "T12.3a SQLite matrix: {} rows | executed {} | covered {} | partial {} | deferred {} | gap {}", + MATRIX.len(), + count(Status::Executed), + count(Status::Covered), + count(Status::Partial), + count(Status::Deferred), + count(Status::Gap) + ); +} diff --git a/engine/crates/layerfs-persistence/tests/support/metrics.rs b/engine/crates/layerfs-persistence/tests/support/metrics.rs new file mode 100644 index 0000000000..e1024b545d --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/support/metrics.rs @@ -0,0 +1,91 @@ +#![allow(dead_code)] +//! Shared metric assertions for the compaction captures. +//! +//! `compact_metrics` (SQLite §7) and `postgresql_compact` (PostgreSQL §8) +//! both read a `CompactMetricsCapture`, so the closed label dimensions and +//! values are checked once here for both backends. + +use layerfs_persistence::compact_metrics::CompactMetrics; + +/// The value of one sample, or `None` when this run did not emit it. +pub fn metric(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> Option { + metrics + .samples() + .into_iter() + .find(|sample| { + sample.name == name + && labels + .iter() + .all(|(key, value)| sample.label(key) == Some(*value)) + }) + .map(|sample| sample.value) +} + +/// The value of one sample the run must emit. +pub fn value(metrics: &CompactMetrics, name: &str, labels: &[(&str, &str)]) -> u64 { + metric(metrics, name, labels).unwrap_or_else(|| { + panic!( + "no sample {name} {labels:?}; emitted: {:?}", + metrics.samples() + ) + }) +} + +/// Bounded cardinality: every label key is a closed dimension and every value +/// is one the enums name, never a Store name, path, operation id or artifact. +pub fn assert_bounded(metrics: &CompactMetrics) { + let keys = [ + "outcome", + "code", + "kind", + "reason", + "state", + "objective", + "result", + "phase", + "statistic", + "which", + "part", + ]; + let values = [ + "rebuild", + "none", + "failed", + "uncertain", + "busy", + "unresolved", + "mark", + "build", + "verify", + "install", + "pg_mark", + "pg_build", + "pg_swap", + "pg_cleanup", + "run", + "leftover", + "count", + "sum", + "max", + "before", + "after", + "bound", + "refined", + "dead_in_retained_packs", + "delta_bases", + "inode_blocks", + "residue_free", + "generation", + "pass", + "fail", + ]; + for sample in metrics.samples() { + for (key, label) in &sample.labels { + assert!(keys.contains(key), "non-closed label key {key}"); + assert!( + values.contains(label), + "non-closed label value {label} for {key}" + ); + } + } +} diff --git a/engine/crates/layerfs-persistence/tests/support/mod.rs b/engine/crates/layerfs-persistence/tests/support/mod.rs index c7d718d7dc..3a68d87e6f 100644 --- a/engine/crates/layerfs-persistence/tests/support/mod.rs +++ b/engine/crates/layerfs-persistence/tests/support/mod.rs @@ -140,3 +140,8 @@ pub mod sqlite { } pub mod doctor; + +pub mod metrics; + +#[cfg(feature = "postgresql")] +pub mod pg; diff --git a/engine/crates/layerfs-persistence/tests/support/pg.rs b/engine/crates/layerfs-persistence/tests/support/pg.rs new file mode 100644 index 0000000000..5952125420 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/support/pg.rs @@ -0,0 +1,281 @@ +#![allow(dead_code)] +//! Shared fixture for the PostgreSQL runtime-evidence tests (T1.3b, T6, T6.5, +//! T7.1, T7.2). One disposable schema per test, driven only through the public +//! `PostgresHandles`/`HistoryCatalog` API except where a test proves the +//! physical layout (a trigger, a dropped table, an epoch row). +use layerfs_content::ObjectId; +use layerfs_history::records::{ + AddLayerOutcome, AddLayerRequest, BranchMergeRequest, CommitMeta, CommitOutcome, CommitRequest, + CommitVia, ForkRequest, ForkSource, MergeCommit, MergeHead, MergeSide, StackInitialization, +}; +use layerfs_history::{ + BranchId, CommitId, CommitRecord, HistoryCatalog, HistoryCatalogConfig, LayerId, LayerRecord, + LayerStackId, NodeId, +}; +use layerfs_persistence::{PostgresHandles, PostgresSelection}; +use layerfs_storage::port::PersistenceError; +use layerfs_storage::StoragePolicy; + +use super::{name, root, stack}; + +/// The URL a PostgreSQL-backed test runs against, or `None` with the usual +/// NOT_RUN note. +pub fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +/// One `PostgresSelection` under a disposable schema named for the test. +pub struct Schema { + pub url: String, + pub name: String, +} + +impl Schema { + pub fn new(url: String, tag: &str) -> Self { + let name = format!("lfsg6a_{tag}_{}", std::process::id()); + let schema = Self { url, name }; + schema.drop_schema(); + schema + } + + pub fn selection(&self) -> PostgresSelection { + PostgresSelection::new(&self.url, &self.name) + } + + pub fn create(&self, binding: &[u8], cursor: [u8; 32]) -> PostgresHandles { + PostgresHandles::create( + &self.selection(), + StoragePolicy::frozen_default(), + &catalog(binding, cursor, 3), + ) + .expect("PostgreSQL Store creation") + } + + pub fn open( + &self, + binding: &[u8], + cursor: [u8; 32], + ) -> Result { + PostgresHandles::open_writable(&self.selection(), binding, cursor) + } + + /// An admin connection with `search_path` pointing at the test schema. + pub fn admin(&self) -> postgres::Client { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).unwrap(); + client + .batch_execute(&format!("SET search_path TO {}", self.name)) + .unwrap(); + client + } + + /// The live column definition the Store records, exactly as + /// `history_meta.schema_definition` holds it. + pub fn definition(&self) -> String { + self.admin().query_one(DEFINITION, &[]).unwrap().get(0) + } + + pub fn drop_schema(&self) { + if let Ok(mut client) = postgres::Client::connect(&self.url, postgres::NoTls) { + let _ = client.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", self.name)); + } + } +} + +impl Drop for Schema { + fn drop(&mut self) { + self.drop_schema(); + } +} + +/// The live column definition, as `postgresql_position_columns.rs` uses it. +pub const DEFINITION: &str = "SELECT string_agg(table_name::text||' '||column_name::text||' '||data_type::text||' '||is_nullable::text, chr(10) ORDER BY table_name, ordinal_position)||chr(10) FROM information_schema.columns WHERE table_schema=current_schema()"; + +pub fn catalog(binding: &[u8], cursor: [u8; 32], incarnation: u64) -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: binding.to_vec(), + cursor_key: cursor, + incarnation, + } +} + +/// Project `tag` with one Branch `work`, at its genesis Layer. +pub fn project(history: &dyn HistoryCatalog, tag: u8) -> (LayerStackId, BranchId, LayerId) { + project_with_genesis(history, tag, root(1)) +} + +/// Project `tag` with one Branch `work`, rooted at an already-published +/// genesis object (a mark needs a locator for every retained root). +pub fn project_with_genesis( + history: &dyn HistoryCatalog, + tag: u8, + genesis_root: ObjectId, +) -> (LayerStackId, BranchId, LayerId) { + let record = history + .initialize_layerstack(&StackInitialization { + logical: 10, + stack: stack(tag), + name: name(&format!("project-{tag}")), + scope: root(200), + profile: root(201), + genesis_root, + }) + .unwrap(); + let branch = branch_id(tag); + history + .fork(&ForkRequest { + stack: stack(tag), + branch, + name: name("work"), + source: ForkSource::Layer(record.head_layer), + }) + .unwrap(); + (stack(tag), branch, record.head_layer) +} + +pub fn branch_id(tag: u8) -> BranchId { + super::branch_id(tag) +} + +pub fn commit_request( + history: &dyn HistoryCatalog, + branch: BranchId, + candidate_root: ObjectId, +) -> CommitRequest { + let record = history.branch(branch).unwrap().unwrap(); + let expected_root = match record.head_commit { + Some(head) => history.commit(head).unwrap().unwrap().root, + None => history.layer(record.base_layer).unwrap().unwrap().root, + }; + CommitRequest { + branch, + expected_head: record.head_commit, + expected_base: record.base_layer, + expected_root, + candidate_root, + profile: root(201), + scope: root(200), + second_parent: None, + meta: CommitMeta::new(CommitVia::Files), + force: false, + } +} + +pub fn commit(history: &dyn HistoryCatalog, branch: BranchId, tag: u8) -> CommitRecord { + commit_with(history, branch, root(tag)) +} + +pub fn commit_with( + history: &dyn HistoryCatalog, + branch: BranchId, + candidate_root: ObjectId, +) -> CommitRecord { + match history + .commit_branch(&commit_request(history, branch, candidate_root)) + .unwrap() + { + CommitOutcome::Committed { record, .. } => *record, + other => panic!("commit: {other:?}"), + } +} + +pub fn layer_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, +) -> AddLayerRequest { + let record = history.branch(branch).unwrap().unwrap(); + let head = history.layer_stack(stack).unwrap().unwrap(); + AddLayerRequest { + stack, + branch, + commit: record.head_commit.expect("a committed Branch"), + expected_stack_head: head.head_layer, + expected_branch_base: record.base_layer, + } +} + +pub fn publish(history: &dyn HistoryCatalog, stack: LayerStackId, branch: BranchId) -> LayerRecord { + match history + .add_layer(&layer_request(history, stack, branch)) + .unwrap() + { + AddLayerOutcome::Added { layer } => layer, + other => panic!("publish: {other:?}"), + } +} + +/// A merge of `work`'s Commit with a second Branch still at the genesis Layer: +/// one new Commit, second parent the genesis node, root `tag`. +pub fn merge_request( + history: &dyn HistoryCatalog, + stack: LayerStackId, + branch: BranchId, + commit: CommitId, + genesis: LayerId, + tag: u8, +) -> BranchMergeRequest { + let side = branch_id(2); + history + .fork(&ForkRequest { + stack, + branch: side, + name: name("side"), + source: ForkSource::Layer(genesis), + }) + .unwrap(); + BranchMergeRequest { + stack, + branch: branch_id(9), + name: name("merged"), + sides: [ + (MergeSide::Branch(branch), NodeId::Commit(commit)), + (MergeSide::Branch(side), NodeId::Layer(genesis)), + ], + head: MergeHead::Commit(Box::new(MergeCommit { + base: genesis, + parent: Some(commit), + second_parent: NodeId::Layer(genesis), + root: root(tag), + meta: CommitMeta::new(CommitVia::Merge), + })), + } +} + +/// `(commit rows, layer rows)` to prove a refusal wrote nothing. +pub fn counts(admin: &mut postgres::Client) -> (i64, i64) { + let count = |admin: &mut postgres::Client, table: &str| -> i64 { + admin + .query_one(&format!("SELECT count(*) FROM \"{table}\""), &[]) + .unwrap() + .get(0) + }; + (count(admin, "commit"), count(admin, "layer")) +} + +pub fn epoch(admin: &mut postgres::Client, stack: LayerStackId) -> i64 { + admin + .query_one( + "SELECT epoch FROM project_epoch WHERE layer_stack_id=$1", + &[&stack.as_slice()], + ) + .unwrap() + .get(0) +} + +/// Name of every base table in the test schema, in name order. +pub fn tables(admin: &mut postgres::Client) -> Vec { + admin + .query( + "SELECT table_name::text FROM information_schema.tables \ + WHERE table_schema=current_schema() AND table_type='BASE TABLE' ORDER BY 1", + &[], + ) + .unwrap() + .into_iter() + .map(|row| row.get(0)) + .collect() +} diff --git a/engine/crates/layerfs-project/src/store/init.rs b/engine/crates/layerfs-project/src/store/init.rs index b2bb16565e..37af0fc0ee 100644 --- a/engine/crates/layerfs-project/src/store/init.rs +++ b/engine/crates/layerfs-project/src/store/init.rs @@ -2,8 +2,8 @@ use crate as layerfs_project; use layerfs_bridge::provision::{check_destination, ProviderKind, StoreManifest, StoreProfile}; use layerfs_history::{ - BranchId, BranchSnapshot, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, - HistoryError, HistoryName, InitStatistics, LayerStackId, + error::NameTakenState, BranchId, BranchSnapshot, ForkRequest, ForkSource, HistoryCatalog, + HistoryCatalogConfig, HistoryError, HistoryName, InitStatistics, LayerStackId, }; use layerfs_persistence::{ PersistenceConfig, SealedStore, SqliteAcquisitionSchema, SqliteHandles, @@ -77,9 +77,10 @@ pub enum InitError { Project(ProjectError), /// First Branch publication. History(HistoryError), - /// Adding a project (#506): the Store already has a project of that - /// name; nothing was written. - Exists, + /// Adding a project (#506): the Store still holds that project name. + /// It is `NameTaken` whether the holder is live, retired or deleted; + /// nothing was written. + NameTaken(Box), /// Adding a project (#506): the Store was made without the acquisition /// tables (schema versions 16 to 18), refused before anything was /// scanned or written. LayerFS adds no tables to an existing Store. diff --git a/engine/crates/layerfs-project/src/store/live.rs b/engine/crates/layerfs-project/src/store/live.rs index 143ff54cb0..e63fe2cdd4 100644 --- a/engine/crates/layerfs-project/src/store/live.rs +++ b/engine/crates/layerfs-project/src/store/live.rs @@ -15,7 +15,7 @@ use crate::store::{live_open::Opened, InitError}; use crate::{InitCounters, InitStep, Initialized, ProjectError}; use layerfs_history::{ BranchId, BranchSnapshot, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, - HistoryName, HistoryResult, LayerStackId, LayerStackRecord, Page, + HistoryError, HistoryName, HistoryResult, LayerStackId, LayerStackRecord, Page, }; use layerfs_persistence::{ SealedStore, SqliteAcquisitionSchema, SqliteHandles, SqlitePersistenceProfile, StoreSelection, @@ -132,11 +132,14 @@ pub(crate) fn add( disposable(&request.store)?; let handles = Opened::writable(&request.store, &request.binding, request.cursor_key)?; let history = handles.history(); - let taken = history - .layer_stack_named(&request.stack_name) - .map_err(InitError::History)?; - if taken.is_some() { - return Err(InitError::Exists); + // A held name (live, retired or deleted) is refused before the folder + // is scanned or any object written. The publication re-checks it, so + // a collision that appears meanwhile is refused the same way. + if let Some(state) = history + .layer_stack_name_taken(&request.stack_name) + .map_err(InitError::History)? + { + return Err(InitError::NameTaken(Box::new(state))); } let storage = Storage::new(handles.storage()).map_err(InitError::Storage)?; let imported = crate::init_observed( @@ -195,6 +198,11 @@ fn refused_acquisition(error: ProjectError) -> InitError { ProjectError::Acquisition(AcquisitionError::Persistence(PersistenceError::Refused { status, })) if status == "acquisition working rows remain" => InitError::Busy, + // The catalog's collision query read the holder (live, retired or + // deleted) and wrote nothing: keep its typed state for the SDK. + ProjectError::History(HistoryError::NameTaken(state)) => { + InitError::NameTaken(state.clone()) + } _ => InitError::Project(error), } } diff --git a/engine/crates/layerfs-project/tests/init_postgresql.rs b/engine/crates/layerfs-project/tests/init_postgresql.rs index b19f3ca345..a0dff6d834 100644 --- a/engine/crates/layerfs-project/tests/init_postgresql.rs +++ b/engine/crates/layerfs-project/tests/init_postgresql.rs @@ -7,7 +7,10 @@ //! sslmode=disable`). Without it the test reports NOT_RUN and passes nothing. #![cfg(all(unix, feature = "postgresql"))] use layerfs_content::{filesystem::root::FilesystemRootId, read_all, FilesystemRead, LogicalPath}; -use layerfs_history::{BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId}; +use layerfs_history::{ + error::{HolderKind, HolderState, NameSource}, + BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId, +}; use layerfs_persistence::{ PersistenceConfig, PostgresHandles, PostgresSelection, SqlitePersistenceProfile, StoreSelection, }; @@ -176,11 +179,15 @@ fn a_folder_inits_into_postgresql_with_the_root_sqlite_gives() { let work = on_postgresql.initialized.namespace_work; assert_eq!((work.wide_directories, work.regular_aliases), (1, 2)); - // A taken name is refused before anything is written; another project is added. - assert!(matches!( - add(postgresql(), &source, "app"), - Err(InitError::Exists) - )); + // A taken name is the typed `NameTaken`; another project is added. + let taken = add(postgresql(), &source, "app"); + let Err(InitError::NameTaken(state)) = taken else { + panic!("a held project name is NameTaken: {taken:?}"); + }; + assert_eq!(state.holder.kind, HolderKind::Project); + assert_eq!(state.holder.name, "app"); + assert_eq!(state.holder.state, HolderState::Live); + assert_eq!(state.source, NameSource::Store); let other = scratch.path.join("other"); fs::create_dir_all(&other).unwrap(); fs::write(other.join("README"), b"other").unwrap(); diff --git a/engine/crates/layerfs-store-service/src/history/admission.rs b/engine/crates/layerfs-store-service/src/history/admission.rs index 84b421dca8..cf614f48b2 100644 --- a/engine/crates/layerfs-store-service/src/history/admission.rs +++ b/engine/crates/layerfs-store-service/src/history/admission.rs @@ -66,10 +66,13 @@ pub fn project_removal_at(stack: LayerStackId, op: [u8; 16]) -> GcIntent { } /// A mount intent at Branch scope, naming its project identity for the -/// PostgreSQL key derivation. +/// PostgreSQL key derivation. The Workspace is always placed; the Sandbox +/// and machine are placed only when the caller that mounted knows them +/// (a daemon mount does; a private Store-call Workspace does not). pub fn mount_intent( workspace: layerfs_history::WorkspaceId, snapshot: &layerfs_history::BranchSnapshot, + place: &layerfs_bridge::control::MountPlace, ) -> GcIntent { let body: [u8; 32] = workspace.as_slice().try_into().unwrap_or([0u8; 32]); GcIntent::shared( @@ -80,7 +83,11 @@ pub fn mount_intent( ) .in_project(snapshot.branch.stack.as_slice()) .named(None, Some(snapshot.branch.name.as_str())) - .placed(Some(body), None, None) + .placed( + Some(body), + place.sandbox.as_deref(), + place.machine.as_deref(), + ) } /// A short-lived shared intent (Init or merge), released on success or drop. diff --git a/engine/crates/layerfs-store-service/src/history/branch.rs b/engine/crates/layerfs-store-service/src/history/branch.rs index eeeb766ebd..1d60ff3922 100644 --- a/engine/crates/layerfs-store-service/src/history/branch.rs +++ b/engine/crates/layerfs-store-service/src/history/branch.rs @@ -346,7 +346,7 @@ pub fn list_layers(store: &Arc, stack: &LayerStackRecord, limit: usize, p Some(id) => match names.get(&id) { Some(name) => Some(name.clone()), None => catalog - .branch(id) + .branch_any(id) .map_err(history)? .map(|branch| branch.name.as_str().to_owned()), }, diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index c197adebc4..d3d581edca 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -73,7 +73,8 @@ pub fn history(error: HistoryError) -> SdkRefusal { refuse(code, message, detail) } -fn name_taken_detail(state: &NameTakenState) -> SdkValue { +/// The `NameTaken` detail: the holder and where it is held. +pub fn name_taken_detail(state: &NameTakenState) -> SdkValue { SdkValue::map([ ( "holder", @@ -128,21 +129,21 @@ fn mounts_detail(state: &layerfs_history::MountHolders) -> SdkValue { holder .workspace .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ( "sandbox", holder .sandbox .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ( "machine", holder .machine .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + .map_or(SdkValue::Null, SdkValue::text), ), ("op", SdkValue::text(&holder.op)), ]) @@ -162,10 +163,7 @@ fn op_detail(state: &layerfs_history::OpConflict) -> SdkValue { ), ( "phase", - state - .phase - .as_ref() - .map_or(SdkValue::Null, |value| SdkValue::text(value)), + state.phase.as_ref().map_or(SdkValue::Null, SdkValue::text), ), ("op", SdkValue::text(&state.op)), ("uncertain", SdkValue::Bool(state.uncertain)), diff --git a/engine/crates/layerfs-store-service/src/history/saved.rs b/engine/crates/layerfs-store-service/src/history/saved.rs index f14c40282f..31b28b2db7 100644 --- a/engine/crates/layerfs-store-service/src/history/saved.rs +++ b/engine/crates/layerfs-store-service/src/history/saved.rs @@ -18,7 +18,7 @@ pub fn reference(store: &Arc, node: NodeId) -> Result { let branch = record .meta .branch - .map(|id| store.published(|history| history.branch(id))) + .map(|id| store.published(|history| history.branch_any(id))) .transpose() .map_err(args::published)? .flatten() @@ -83,7 +83,7 @@ pub fn record(store: &Arc, node: NodeId) -> Result { .transpose()?; let source_branch = r .source_branch - .map(|id| store.published(|history| history.branch(id))) + .map(|id| store.published(|history| history.branch_any(id))) .transpose() .map_err(args::published)? .flatten(); diff --git a/engine/crates/layerfs-store-service/src/methods/file/blame.rs b/engine/crates/layerfs-store-service/src/methods/file/blame.rs index 1c9a8dcaf4..176f29da1c 100644 --- a/engine/crates/layerfs-store-service/src/methods/file/blame.rs +++ b/engine/crates/layerfs-store-service/src/methods/file/blame.rs @@ -17,8 +17,10 @@ pub fn blame(service: &StoreService, args: &SdkValue) -> Answer { &snap, &path, &seen, - line(args, "fromLine", 1), - line(args, "lineCount", 2000), + blame::Window { + from: line(args, "fromLine", 1), + count: line(args, "lineCount", 2000), + }, &mut blame::Limits { versions: crate::read::query::MAXIMUM_QUERY_WORK, bytes: 64 * 1024 * 1024, diff --git a/engine/crates/layerfs-store-service/src/operator/files.rs b/engine/crates/layerfs-store-service/src/operator/files.rs index ec5e04acaa..078be27e11 100644 --- a/engine/crates/layerfs-store-service/src/operator/files.rs +++ b/engine/crates/layerfs-store-service/src/operator/files.rs @@ -24,7 +24,7 @@ pub fn revision( let branch = r .meta .branch - .map(|id| store.history().branch(id)) + .map(|id| store.history().branch_any(id)) .transpose() .map_err(args::history)? .flatten(); diff --git a/engine/crates/layerfs-store-service/src/operator/inspect.rs b/engine/crates/layerfs-store-service/src/operator/inspect.rs index 9dc541f56b..60bdfb4224 100644 --- a/engine/crates/layerfs-store-service/src/operator/inspect.rs +++ b/engine/crates/layerfs-store-service/src/operator/inspect.rs @@ -190,9 +190,17 @@ pub fn snapshot(store: &Arc, args: &SdkValue, views: &[WorkspaceView]) -> let mut next: Option = Some(stack.head_layer); while let Some(id) = next { let record = args::layer(store, id)?; - let source = record - .source_branch - .and_then(|branch| names.get(&branch).map(|(name, _)| name.clone())); + let source = match record.source_branch { + Some(branch) => match names.get(&branch) { + Some((name, _)) => Some(name.clone()), + None => store + .history() + .branch_any(branch) + .map_err(args::history)? + .map(|record| record.name.as_str().to_owned()), + }, + None => None, + }; let value = layer_value(&record, source.as_deref()); if layer_spent + cost(&value) > LAYER_BUDGET { break; diff --git a/engine/crates/layerfs-store-service/src/read/attribution.rs b/engine/crates/layerfs-store-service/src/read/attribution.rs index e08f7df31a..fc67983235 100644 --- a/engine/crates/layerfs-store-service/src/read/attribution.rs +++ b/engine/crates/layerfs-store-service/src/read/attribution.rs @@ -61,8 +61,18 @@ fn owners( versions: budget.work, bytes: budget.bytes, }; - let page = - blame::blame(store, reader, snap, path, visibility, Ok(from), Ok(2000), &mut limits)?; + let page = blame::blame( + store, + reader, + snap, + path, + visibility, + blame::Window { + from: Ok(from), + count: Ok(2000), + }, + &mut limits, + )?; budget.work = limits.versions; budget.bytes = limits.bytes; exact &= !page.get_bool("comparisonCoarse") diff --git a/engine/crates/layerfs-store-service/src/read/blame.rs b/engine/crates/layerfs-store-service/src/read/blame.rs index a0c1ba9bd1..4ca272f6e9 100644 --- a/engine/crates/layerfs-store-service/src/read/blame.rs +++ b/engine/crates/layerfs-store-service/src/read/blame.rs @@ -129,6 +129,12 @@ pub struct Limits { pub bytes: u64, } +/// The line window a blame page covers. +pub struct Window { + pub from: Arg, + pub count: Arg, +} + /// The origin Commit of each line of the regular file at `path`, from line /// `from` on, along persisted first parents. pub fn blame( @@ -137,8 +143,7 @@ pub fn blame( snap: &Snap, path: &str, visibility: &Visibility, - from: Arg, - count: Arg, + window: Window, limits: &mut Limits, ) -> Answer { let tip = snapshot::existing(reader, snap, path)?; @@ -152,8 +157,8 @@ pub fn blame( SdkValue::Null, )); } - let from = from?; - let count = count?; + let from = window.from?; + let count = window.count?; if count == 0 || count > window::LINES as u64 { return Err(args::invalid("lineCount must be in 1..=2000")); } diff --git a/engine/crates/layerfs-store-service/src/read/visibility.rs b/engine/crates/layerfs-store-service/src/read/visibility.rs index f388a1b1a5..906659b46b 100644 --- a/engine/crates/layerfs-store-service/src/read/visibility.rs +++ b/engine/crates/layerfs-store-service/src/read/visibility.rs @@ -66,10 +66,9 @@ impl Visibility { /// `load` for a stack id and an explicit `history_serial`, for reads that /// walk several nodes and therefore ask `hidden_at` per node. pub fn for_stack(store: &Arc, stack: LayerStackId, serial: u64) -> Self { - let markers = match store.published(|history| history.path_removals(stack)) { - Ok(markers) => markers, - Err(_) => Vec::new(), - }; + let markers = store + .published(|history| history.path_removals(stack)) + .unwrap_or_default(); Self { serial, markers } } diff --git a/engine/crates/layerfs-store-service/src/store/bind.rs b/engine/crates/layerfs-store-service/src/store/bind.rs index c4bd4b6065..fd169256c6 100644 --- a/engine/crates/layerfs-store-service/src/store/bind.rs +++ b/engine/crates/layerfs-store-service/src/store/bind.rs @@ -4,6 +4,7 @@ use super::{ NamedBindRequest, Store, }; use crate::OwnerClient; +use layerfs_bridge::control::MountPlace; use layerfs_content::filesystem::{ attributes::{read_portable, AttributeReadWork}, directory::decode_directory_page, @@ -24,7 +25,19 @@ impl Store { owner: OwnerClient, request: BindRequest, ) -> Result { - self.bind_with(owner, BindIntent::Id(request), false) + self.bind_with(owner, BindIntent::Id(request), MountPlace::default(), false) + } + /// [`Self::bind`] of a mount whose caller knows the Sandbox and machine + /// its daemon serves: the registered holder reports them beside the + /// Workspace. An unknown place is the plain [`Self::bind`]. + #[allow(clippy::result_large_err)] + pub fn bind_placed( + self: &Arc, + owner: OwnerClient, + request: BindRequest, + place: MountPlace, + ) -> Result { + self.bind_with(owner, BindIntent::Id(request), place, false) } /// [`Self::bind`] of the Branch a project and Branch name select (#64), /// found on the Workspace's own reader with its snapshot in one read. @@ -46,7 +59,23 @@ impl Store { request: BindIntent::Named(request), }); } - self.bind_with(owner, BindIntent::Named(request), false) + self.bind_with( + owner, + BindIntent::Named(request), + MountPlace::default(), + false, + ) + } + /// [`Self::bind_named`] of a mount whose caller knows the Sandbox and + /// machine its daemon serves. + #[allow(clippy::result_large_err)] + pub fn bind_named_placed( + self: &Arc, + owner: OwnerClient, + request: NamedBindRequest, + place: MountPlace, + ) -> Result { + self.bind_with(owner, BindIntent::Named(request), place, false) } /// [`Self::bind`] for an SDK call's private Workspace: its reader comes /// from, and returns to, the Store's idle readers (#352 F-025 fix 4). @@ -56,13 +85,25 @@ impl Store { owner: OwnerClient, request: BindRequest, ) -> Result { - self.bind_with(owner, BindIntent::Id(request), true) + self.bind_with(owner, BindIntent::Id(request), MountPlace::default(), true) + } + /// [`Self::bind_private`] of a caller that knows the Sandbox and + /// machine the Workspace belongs to. + #[allow(clippy::result_large_err)] + pub fn bind_private_placed( + self: &Arc, + owner: OwnerClient, + request: BindRequest, + place: MountPlace, + ) -> Result { + self.bind_with(owner, BindIntent::Id(request), place, true) } #[allow(clippy::result_large_err)] fn bind_with( self: &Arc, owner: OwnerClient, request: BindIntent, + place: MountPlace, pooled: bool, ) -> Result { let mut phase = BindPhase::Snapshot; @@ -94,7 +135,7 @@ impl Store { // Admission before any root demand: a live removal is `Busy`, an // uncertain owner `Unresolved`, and the intent lives until this // Workspace's unmount deletes it. - let intent = crate::history::admission::mount_intent(workspace, &captured); + let intent = crate::history::admission::mount_intent(workspace, &captured, &place); match self.history().gc_register(&intent) { Ok(()) => gc_op = Some(intent.op_id), // A provider without admission cannot exclude a removal; it diff --git a/engine/crates/layerfs-store-service/src/write/files.rs b/engine/crates/layerfs-store-service/src/write/files.rs index 36925dcd95..683cdfdbe9 100644 --- a/engine/crates/layerfs-store-service/src/write/files.rs +++ b/engine/crates/layerfs-store-service/src/write/files.rs @@ -223,7 +223,7 @@ fn change( let tree = TreeTable::open(&*reader, FilesystemRootId(base.effective_root)) .map_err(args::content)?; let removed = |path: &str| -> Result, SdkRefusal> { - let exists = snapshot::entry(&*reader, &tree, path)?.is_some(); + let exists = snapshot::entry(&reader, &tree, path)?.is_some(); Ok(exists .then(|| visibility.hidden(path)) .flatten() diff --git a/engine/crates/layerfs-store-service/tests/admission.rs b/engine/crates/layerfs-store-service/tests/admission.rs index d8db93e9c8..7889ea2250 100644 --- a/engine/crates/layerfs-store-service/tests/admission.rs +++ b/engine/crates/layerfs-store-service/tests/admission.rs @@ -50,6 +50,9 @@ fn a_mount_refuses_every_removal_through_the_service() { }; assert_eq!(holders[0].get_text("kind"), Some("mount")); assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + assert_eq!(holders[0].get_text("machine"), Some("machine-9")); + let workspace = lookup::hex(&[9; 32]); + assert_eq!(holders[0].get_text("workspace"), Some(workspace.as_str())); let refused = deletion::delete_stack(&fixture.service.store, stack.id, 1_000) .expect_err("the mount holds"); diff --git a/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt b/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt index 0fb8409998..cb3af3dd9b 100644 --- a/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt +++ b/engine/crates/layerfs-store-service/tests/golden/sdk_methods.txt @@ -5,8 +5,8 @@ project.list_branches -> refused InvalidInput "pageSize must be in 1..=1000" nul branch.fork -> ok {kind: "Forked", branch: "feat", base: "", head: null} branch.fork -> ok {kind: "Forked", branch: "other", base: "", head: null} branch.fork -> refused ReservedName "\"main\" names the project's main line" null -branch.fork -> refused NameTaken "Branch feat already exists" null -branch.fork -> refused InvalidInput "`nope` is not a Commit or Layer identity" null +branch.fork -> refused NameTaken "name taken: NameHolder { kind: Branch, name: \"feat\", state: Live }" {holder: {kind: "branch", name: "feat", state: "live"}, source: "store"} +branch.fork -> refused NoSuchBranch "Branch nope does not exist" null fs.write -> ok {kind: "Committed", branch: "feat", head: "", commit: {id: "", branch: "feat", parent: null, secondParent: null, base: "", root: "", via: "fs", message: "first", mergedFrom: null, time: , logical: 43, newCanonical: 957, newStored: , newMetadata: , displaced: null, lww: [], lwwTruncated: false}} fs.write -> refused ReadOnly "main changes only through publish" null fs.write -> ok {kind: "UpToDate", branch: "feat", head: "", root: ""} @@ -68,7 +68,7 @@ branch.fork -> ok {kind: "Forked", branch: "right", base: "", head: null} fs.write -> ok {kind: "Committed", branch: "right", head: "", commit: {id: "", branch: "right", parent: null, secondParent: null, base: "", root: "", via: "fs", message: "Apply filesystem changes", mergedFrom: null, time: , logical: 42, newCanonical: 958, newStored: , newMetadata: , displaced: null, lww: [], lwwTruncated: false}} branch.merge -> ok {kind: "Merged", fastForward: false, branch: "both", head: "", root: "", sources: [{ref: "left", node: ""}, {ref: "right", node: ""}], mergeBase: "", conflicts: [], lww: [], conflictsTruncated: false, lwwTruncated: false, units: [], unitsTruncated: false} fs.list -> ok {kind: "ok", snapshot: "both@", root: "", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "left.txt", path: "/left.txt", type: "file", inode: 328}, {name: "right.txt", path: "/right.txt", type: "file", inode: 392}], hasMore: false, nextPagePos: null, removed_count: 0, at: {id: "", kind: "Commit", branch: "both", root: ""}} -branch.merge -> refused NameTaken "Branch both already exists" null +branch.merge -> refused NameTaken "name taken: NameHolder { kind: Branch, name: \"both\", state: Live }" {holder: {kind: "branch", name: "both", state: "live"}, source: "store"} branch.merge -> refused InvalidInput "merge takes two different sources" null branch.merge -> refused ReservedName "\"main\" names the project's main line" null branch.publish -> ok {kind: "Added", source: {branch: "feat", commit: ""}, publishedLayer: {overwroteCount: 0, logical: 55, id: "", parent: "", root: "", sourceBranch: "feat", sourceCommit: ""}, basedOn: "", overwroteCount: 0, overwrote: [], overwroteHasMore: false} diff --git a/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs b/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs new file mode 100644 index 0000000000..07a8a7a982 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_mount_bind_failure.rs @@ -0,0 +1,95 @@ +//! T12.3c: §12 AC-34, the mount bind-failure path over the SQLite Store +//! service. A committed mount intent whose bind fails after the admission is +//! deleted in the error path, so no `MountsActive` remains. The failed-delete +//! arm is not inducible on a SQLite Store: the storage schema guard rejects a +//! trigger on `gc_intent`, and an outside lock held across the bind would +//! refuse the register too, so the row names that clause as uncovered. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::SdkValue; +use layerfs_history::{BranchId, HistoryName, LayerStackId, WorkspaceId}; +use layerfs_store_service::store::{BindError, BindPhase, BindRequest}; +use support::Fixture; + +/// The installed fixture's project and a Branch `broken` whose stack profile +/// no longer matches its root: the bind's root check must fail after the +/// admission intent commits. The Commit itself is service-written, so every +/// history identity stays valid; only the stack row's profile changes. +fn break_root(fixture: &Fixture) -> BranchId { + let stack = LayerStackId::from_authority([23; 16]); + let branch = BranchId::named(stack, &HistoryName::new("broken").unwrap()); + fixture.fork("broken"); + fixture.apply("broken", vec![support::write("/b", "b\n")]); + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + let changed = connection + .execute( + "UPDATE layer_stack SET profile_id = ?1 WHERE layer_stack_id = ?2", + rusqlite::params![[0x42u8; 32].to_vec(), stack.as_slice().to_vec()], + ) + .unwrap(); + assert_eq!(changed, 1, "the fixture has one LayerStack"); + branch +} + +fn bind( + fixture: &Fixture, + branch: BranchId, + id: u8, +) -> Result<(), layerfs_store_service::store::BindRefusal> { + fixture + .service + .store + .bind( + fixture.owner.client(), + BindRequest { + workspace: WorkspaceId::from_authority([id; 32]).unwrap(), + branch, + }, + ) + .map(|_| ()) +} + +/// The committed admission intents the Store holds. +fn intents(fixture: &Fixture) -> Vec> { + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + let mut statement = connection.prepare("SELECT op_id FROM gc_intent").unwrap(); + statement + .query_map([], |row| row.get(0)) + .unwrap() + .map(|row| row.unwrap()) + .collect() +} + +#[test] +fn a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active() { + let fixture = Fixture::new("t123c-bind", |source| { + std::fs::write(source.join("a"), "a\n").unwrap(); + 1 + }); + let branch = break_root(&fixture); + + // The admission intent commits, the root check then fails: the refusal + // names the root phase and the error path deleted the intent. + let refusal = bind(&fixture, branch, 71).expect_err("the root is not a filesystem root"); + assert_eq!(refusal.phase, BindPhase::Root, "{refusal:?}"); + assert!( + matches!(refusal.error, BindError::Content { .. }), + "{refusal:?}" + ); + assert!( + intents(&fixture).is_empty(), + "a failed bind leaves no committed intent" + ); + // No `MountsActive` remains: the removal is admitted. + fixture.call( + "project.remove_file", + vec![("path", SdkValue::text("/a"))], + ); + + fixture.cleanup(); +} diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs b/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs new file mode 100644 index 0000000000..36c4fe95a9 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/sdk_path_merge_removal.rs @@ -0,0 +1,290 @@ +//! T12.3c: `project.remove_file` in derived trees (#52 §12 rows AC-11..AC-14 +//! and AC-22). A merge of at-or-below heads materializes the whiteout, a +//! post-frontier re-creation wins, a fast-forward merge creates no tree, a +//! publish of an at-or-below Commit yields a hidden Layer whose mount +//! whiteouts, an unmerged Branch that existed before the frontier hides but +//! does not inherit, and a merge started together with the removal never +//! leaves the path visible. +#[path = "support/installed_store.rs"] +#[allow(dead_code)] +mod installed; +#[path = "support/sdk.rs"] +#[allow(dead_code)] +mod support; + +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::fs; +use support::{write, Change, Fixture}; + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +fn remove(fixture: &Fixture, path: &str) -> SdkValue { + fixture.call("project.remove_file", vec![("path", text(path))]) +} + +fn try_remove(fixture: &Fixture, path: &str) -> Result { + fixture.try_call("project.remove_file", vec![("path", text(path))]) +} + +fn read(fixture: &Fixture, branch: &str, path: &str) -> Result { + fixture.try_call( + "fs.read", + vec![("branch", text(branch)), ("path", text(path))], + ) +} + +/// The text of a present file. +fn visible(fixture: &Fixture, branch: &str, path: &str) -> String { + let value = + read(fixture, branch, path).unwrap_or_else(|refusal| panic!("{branch}{path}: {refusal:?}")); + value + .get_text("text") + .unwrap_or_else(|| panic!("{branch}{path} has no text: {value:?}")) + .to_owned() +} + +/// The path is hidden by the marker at this node: the read refuses `Removed` +/// and names the frontier. +fn hidden(fixture: &Fixture, branch: &str, path: &str) { + let refusal = + read(fixture, branch, path).expect_err(&format!("{branch}{path} should be hidden")); + assert_eq!(refusal.code, "Removed", "{branch}{path}: {refusal:?}"); + let paths = match refusal.detail.get("paths") { + Some(SdkValue::List(paths)) => paths, + other => panic!("Removed detail without paths: {other:?}"), + }; + assert_eq!(paths.len(), 1, "{paths:?}"); + assert_eq!(paths[0].get_text("path"), Some(path), "{paths:?}"); + assert!( + paths[0].get_int("frontier_serial").is_some(), + "{paths:?}" + ); +} + +/// The path is absent from the committed tree: above the frontier the read +/// is `NoSuchPath`, never `Removed`. +fn absent(fixture: &Fixture, branch: &str, path: &str) { + let refusal = + read(fixture, branch, path).expect_err(&format!("{branch}{path} should be absent")); + assert_eq!(refusal.code, "NoSuchPath", "{branch}{path}: {refusal:?}"); +} + +/// Rows of one history table in the installed Store. +fn rows(fixture: &Fixture, table: &str) -> i64 { + let connection = rusqlite::Connection::open(&fixture.host.config.path).unwrap(); + connection + .query_row(&format!("SELECT count(*) FROM \"{table}\""), [], |row| { + row.get(0) + }) + .unwrap() +} + +fn merge(fixture: &Fixture, left: &str, right: &str, name: &str) -> Result { + fixture.try_call( + "branch.merge", + vec![ + ("branch", text(left)), + ("with", text(right)), + ("name", text(name)), + ], + ) +} + +#[test] +fn a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden() { + let fixture = Fixture::new("t123c-ff", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("left"); + fixture.apply("left", vec![write("/l", "l\n")]); + // `fast` starts at left's Commit and commits one child: left contains no + // `fast` node, while `fast` contains every `left` node. + fixture.call( + "branch.fork", + vec![("name", text("fast")), ("from", text("left"))], + ); + fixture.apply("fast", vec![write("/r", "r\n")]); + remove(&fixture, "/a"); + let (commits, layers) = (rows(&fixture, "commit"), rows(&fixture, "layer")); + + // One side contains the other: a fast-forward Branch at the newer node. + let merged = merge(&fixture, "left", "fast", "merged").expect("a fast-forward merge"); + assert!(merged.get_bool("fastForward"), "{merged:?}"); + assert_eq!( + rows(&fixture, "commit"), + commits, + "a fast-forward merge writes no new Commit" + ); + assert_eq!(rows(&fixture, "layer"), layers, "and no new Layer"); + + // The node it points at is at or below the frontier: the marker hides it + // without any whiteout, and the other files stay visible. + hidden(&fixture, "merged", "/a"); + assert_eq!(visible(&fixture, "merged", "/l"), "l\n"); + assert_eq!(visible(&fixture, "merged", "/r"), "r\n"); + fixture.cleanup(); +} + +#[test] +fn publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount() { + let fixture = Fixture::new("t123c-publish", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + fixture.fork("work"); + fixture.apply("work", vec![write("/w", "w\n")]); + remove(&fixture, "/a"); + + // `work`'s Commit is at or below the frontier: publishing republishes its + // tree as a Layer at the same position, so the marker hides it on main. + let published = fixture.call("branch.publish", vec![("branch", text("work"))]); + assert_eq!(published.get_text("kind"), Some("Added"), "{published:?}"); + hidden(&fixture, "main", "/a"); + assert_eq!(visible(&fixture, "main", "/w"), "w\n"); + + // A Branch forked from main is layer-headed (no head Commit) and its base + // Layer is the published, at-or-below one: the mount whiteouts /a, and the + // next Commit records the deletion rather than inheriting it. + fixture.fork("layer"); + fixture.workspace_commit("layer", &[Change::Create("/n", b"n")], 41, || {}); + absent(&fixture, "layer", "/a"); + assert_eq!(visible(&fixture, "layer", "/n"), "n"); + assert_eq!(visible(&fixture, "layer", "/b"), "b\n"); + fixture.cleanup(); +} + +#[test] +fn an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit() { + let fixture = Fixture::new("t123c-unmerged", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + // The Branch and its Commit exist before the marker. + fixture.fork("old"); + fixture.apply("old", vec![write("/x", "x\n")]); + remove(&fixture, "/a"); + + // The earlier Commit's path is hidden by the marker. + hidden(&fixture, "old", "/a"); + assert_eq!(visible(&fixture, "old", "/x"), "x\n"); + + // A Commit after the frontier on that unmerged Branch does not inherit + // it: the derived tree records the absence. + fixture.apply("old", vec![write("/y", "y\n")]); + absent(&fixture, "old", "/a"); + assert_eq!(visible(&fixture, "old", "/x"), "x\n"); + assert_eq!(visible(&fixture, "old", "/y"), "y\n"); + fixture.cleanup(); +} + +#[test] +fn a_recreated_removed_path_still_conflicts_as_an_ordinary_edit() { + let fixture = Fixture::new("t123c-recreate-conflict", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + 1 + }); + // `side` stays at the base node (at or below the frontier). + fixture.fork("side"); + fixture.apply("side", vec![write("/s", "s\n")]); + remove(&fixture, "/a"); + + // Two Branches re-create /a after the frontier; both are above it, so the + // marker no longer applies and this is an ordinary edit conflict. + fixture.fork("one"); + fixture.apply("one", vec![write("/a", "one\n")]); + fixture.fork("two"); + fixture.apply("two", vec![write("/a", "two\n")]); + assert_eq!(visible(&fixture, "one", "/a"), "one\n"); + assert_eq!(visible(&fixture, "two", "/a"), "two\n"); + + let refusal = merge(&fixture, "one", "two", "conflicted").expect_err("an ordinary edit conflict"); + assert_eq!(refusal.code, "Conflict", "{refusal:?}"); + let conflicts = match refusal.detail.get("conflicts") { + Some(SdkValue::List(conflicts)) => conflicts, + other => panic!("Conflict detail without conflicts: {other:?}"), + }; + let paths: Vec<&str> = conflicts + .iter() + .map(|conflict| conflict.get_text("path").unwrap_or("")) + .collect(); + assert_eq!(paths, ["/a"], "{conflicts:?}"); + // A conflicted merge creates nothing. + let missing = fixture + .try_call( + "fs.read", + vec![("branch", text("conflicted")), ("path", text("/a"))], + ) + .expect_err("the conflicted merge created no Branch"); + assert_eq!(missing.code, "NoSuchBranch", "{missing:?}"); + fixture.cleanup(); +} + +#[test] +fn a_merge_and_a_removal_started_together_never_leave_the_path_visible() { + let fixture = Fixture::new("t123c-race", |source| { + fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("b"), "b\n").unwrap(); + 2 + }); + for round in 0..3u8 { + let left = format!("left-{round}"); + let right = format!("right-{round}"); + let merged = format!("merged-{round}"); + fixture.fork(&left); + fixture.fork(&right); + fixture.apply(&left, vec![write(&format!("/l-{round}"), "l\n")]); + fixture.apply(&right, vec![write(&format!("/r-{round}"), "r\n")]); + + // A deterministic barrier starts the merge and the removal together. + let barrier = std::sync::Barrier::new(2); + let (merge_result, remove_result) = std::thread::scope(|scope| { + let merge_barrier = &barrier; + let remove_barrier = &barrier; + let merging = scope.spawn(|| { + merge_barrier.wait(); + merge(&fixture, &left, &right, &merged) + }); + let removing = scope.spawn(|| { + remove_barrier.wait(); + try_remove(&fixture, "/a") + }); + (merging.join().unwrap(), removing.join().unwrap()) + }); + + // The loser retries: a removal refused by the live merge (or its + // operation) is repeated after it, and a merge refused by the epoch + // fence is repeated with the marker in place. + if let Err(refusal) = &remove_result { + assert!( + matches!(refusal.code.as_str(), "MountsActive" | "Busy" | "Unresolved"), + "round {round}: {refusal:?}" + ); + remove(&fixture, "/a"); + } + if let Err(refusal) = &merge_result { + assert_eq!(refusal.code, "Conflict", "round {round}: {refusal:?}"); + merge(&fixture, &left, &right, &merged).expect("the retried merge"); + } + + // Neither order may leave the removed path visible on the merge. + match read(&fixture, &merged, "/a") { + Ok(value) => panic!("round {round}: /a is visible: {value:?}"), + Err(refusal) => assert!( + matches!(refusal.code.as_str(), "Removed" | "NoSuchPath"), + "round {round}: {refusal:?}" + ), + } + // A tree derived above the frontier records the absence. + fixture.apply(&merged, vec![write(&format!("/z-{round}"), "z\n")]); + absent(&fixture, &merged, "/a"); + assert_eq!(visible(&fixture, &merged, &format!("/l-{round}")), "l\n"); + assert_eq!(visible(&fixture, &merged, &format!("/r-{round}")), "r\n"); + } + fixture.cleanup(); +} diff --git a/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs b/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs index 6fff9e6b2c..3b19bf76a9 100644 --- a/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs +++ b/engine/crates/layerfs-store-service/tests/sdk_path_whiteouts.rs @@ -76,41 +76,67 @@ fn a_one_shot_commit_after_remove_stays_absent() { } #[test] -fn a_workspace_commit_bound_before_the_marker_materializes_it() { - // Bound first, then the marker is written before the Commit: the mount - // saw no whiteout, so the Commit path itself must remove the inherited - // entry rather than silently resurrect it. +fn a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes() { + // Admission refuses while any mount intent holds the + // project, so a marker can never appear under a live Workspace. The Commit + // path still has to materialize a marker written before the Workspace + // binds, which the second Workspace below commits through. let fixture = Fixture::new("t55-workspace", |source| { - fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("a"), "a +").unwrap(); 1 }); fixture.fork("work"); fixture.workspace_commit("work", &[Change::Create("/w", b"w")], 31, || { - remove(&fixture, "/a"); + let refusal = fixture + .try_call("project.remove_file", vec![("path", text("/a"))]) + .expect_err("a marker under a live mount"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + let holders = match refusal.detail.get("holders") { + Some(SdkValue::List(holders)) => holders, + other => panic!("MountsActive detail without holders: {other:?}"), + }; + assert_eq!(holders.len(), 1, "{holders:?}"); + assert_eq!(holders[0].get_text("kind"), Some("mount"), "{holders:?}"); + assert_eq!( + holders[0].get_text("workspace"), + Some("1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f"), + "{holders:?}" + ); }); + remove(&fixture, "/a"); + // A Workspace bound after the marker materializes the whiteout into its + // overlay: a Commit of a change elsewhere must not resurrect /a. + fixture.workspace_commit("work", &[Change::Create("/v", b"v")], 33, || {}); absent(&fixture, "work", "/a"); assert_eq!(visible(&fixture, "work", "/w"), "w"); + assert_eq!(visible(&fixture, "work", "/v"), "v"); let listed = fixture.call( "fs.list", vec![("branch", text("work")), ("path", text("/"))], ); - assert!(!paths(&listed).contains(&"/a")); + let names = paths(&listed); + assert!(!names.contains(&"/a"), "{names:?}"); + assert!(names.contains(&"/w") && names.contains(&"/v"), "{names:?}"); fixture.cleanup(); } #[test] -fn a_workspace_commit_keeps_a_re_creation_written_before_the_marker() { - // The workspace rewrote the inherited file, then the marker was written, - // then the Commit: the rewrite is a re-creation and stays visible. +fn a_workspace_commit_keeps_a_re_creation_above_the_marker() { + // The marker comes first, as admission requires while no mount is live. + // The Workspace then binds below the frontier, materializes the whiteout, + // and writes the path again above it: the re-creation stays visible. let fixture = Fixture::new("t55-workspace-recreate", |source| { - fs::write(source.join("a"), "a\n").unwrap(); + fs::write(source.join("a"), "a +").unwrap(); 1 }); fixture.fork("work"); - fixture.workspace_commit("work", &[Change::Write("/a", b"again\n")], 32, || { - remove(&fixture, "/a"); - }); - assert_eq!(visible(&fixture, "work", "/a"), "again\n"); + remove(&fixture, "/a"); + fixture.workspace_commit("work", &[Change::Create("/a", b"again +")], 32, || {}); + assert_eq!(visible(&fixture, "work", "/a"), "again +"); fixture.cleanup(); } diff --git a/tools/ci/test_store_ext4.sh b/tools/ci/test_store_ext4.sh new file mode 100755 index 0000000000..f70e03652c --- /dev/null +++ b/tools/ci/test_store_ext4.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +# Store tests need a filesystem that carries SQLite's locks. +# +# Why this exists +# --------------- +# A Store opens directly only inside a private (0700) directory, owned by this +# user, on a local filesystem whose locks reach every opener: ext4, xfs or +# btrfs on Linux; apfs or hfs on macOS. Anything else is `Unsupported` +# (engine/crates/layerfs-store-service/src/kernel_guard.rs), because SQLite's +# WAL locks and the writers' directory flock cannot coordinate across kernels +# or on tmpfs, overlay, FUSE or network mounts. +# +# The Store suites place their Stores in `std::env::temp_dir()` (TMPDIR) and in +# `env!("CARGO_TARGET_TMPDIR")` (CARGO_TARGET_DIR/tmp). On a host whose /tmp is +# tmpfs -- or inside a container whose checkout is an overlay -- the guard +# refuses every Store and suites such as `layerfs-store-service --test +# kernel_guard` fail with "does not carry SQLite locks". That is an environment +# failure, not a product failure. +# +# This script requires LAYERFS_TEST_TMPDIR on ext4, xfs or btrfs (apfs or hfs on +# macOS), refuses anything else before cargo starts, and points TMPDIR, +# CARGO_TARGET_DIR and CARGO_HOME at it. See +# docs/architecture/engine/store/testing.md. +# +# Usage +# LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh --quick +# LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests tools/ci/test_store_ext4.sh +# tools/ci/test_store_ext4.sh --list +# +# Options +# --quick a small cross-crate subset (heads, sqlite_publication, +# kernel_guard, commit_pipeline, create_directory, store_family) +# --full every Store suite (the default) +# --list print the cargo commands, run nothing +# --help this text +# +# Environment +# LAYERFS_TEST_TMPDIR required; must be on ext4, xfs or btrfs (Linux) +# or apfs or hfs (macOS) +# LAYERFS_TEST_JOBS cargo -j; default min(nproc, 16) +# LAYERFS_TEST_CARGO_HOME default $LAYERFS_TEST_TMPDIR/cargo-home +# LAYERFS_TEST_CARGO_TARGET_DIR +# default $LAYERFS_TEST_TMPDIR/target +# LAYERFS_TEST_POSTGRES_URL libpq connection string; enables the postgresql +# feature on every suite and the PostgreSQL cases. +# Full-mode diagnostics need amcheck created in +# that database (see testing.md); the harness +# warns when psql can see that it is missing. +# LAYERFS_TEST_EXPECT_AMCHECK +# assert a Pass relation summary in +# postgresql_diagnostic (optional) +# Arguments after `--` are appended to every `cargo test` (e.g. -- --nocapture). +# +# Exit status: 0 every suite passed; 1 at least one failed; 2 wrong command line +# or a refused LAYERFS_TEST_TMPDIR; 127 cargo is not installed. +set -euo pipefail + +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" + +usage() { + sed -n '2,/^set -euo pipefail$/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//' +} + +refuse() { + printf 'test_store_ext4: %s\n' "$1" >&2 + shift || true + for line in "$@"; do printf '%s\n' "$line" >&2; done + exit 2 +} + +mode=full +list_only=0 +passthrough=() +while [ "$#" -gt 0 ]; do + case "$1" in + --quick) mode=quick ;; + --full) mode=full ;; + --list) list_only=1 ;; + -h|--help) usage; exit 0 ;; + --) shift; passthrough=("$@"); break ;; + *) refuse "unknown argument: $1 (try --help)" ;; + esac + shift +done + +# ---- The directory and its filesystem ------------------------------------- + +dir=${LAYERFS_TEST_TMPDIR:-} +if [ -z "$dir" ]; then + refuse "LAYERFS_TEST_TMPDIR is not set." \ + "" \ + "The Store suites need their temporary Stores on a filesystem that carries" \ + "SQLite's locks: ext4, xfs or btrfs on Linux; apfs or hfs on macOS." \ + "Choose such a directory and re-run, for example:" \ + "" \ + " LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests $0 --quick" \ + "" \ + "Inside a Docker Sandbox the run directory is ext4 (\$HOME and" \ + "/layerfs/data/machine/native/run/); /tmp is tmpfs there." +fi + +case "$dir" in + /*) ;; + *) dir="$PWD/$dir" ;; +esac +if command -v realpath >/dev/null 2>&1; then + dir=$(realpath -m -- "$dir") +fi + +mkdir -p -- "$dir" 2>/dev/null || refuse "cannot create LAYERFS_TEST_TMPDIR=$dir" +[ -d "$dir" ] || refuse "LAYERFS_TEST_TMPDIR=$dir is not a directory" + +uname_s=$(uname -s 2>/dev/null || printf 'Linux') + +filesystem_type() { + local target=$1 found= + if [ "$uname_s" = Darwin ]; then + # macOS: the mount table names the filesystem in parentheses; `stat -f %T` + # is a fallback for a mount line this cannot parse. + local device type + device=$(df -P "$target" 2>/dev/null | awk 'NR == 2 { print $1 }') + type=$(mount 2>/dev/null | awk -v device="$device" '$1 == device { print; exit }' \ + | sed -n 's/.*(\([^,)]*\).*/\1/p') + if [ -n "$type" ]; then + printf '%s\n' "$type" + return 0 + fi + stat -f %T "$target" 2>/dev/null || true + return 0 + fi + if command -v findmnt >/dev/null 2>&1; then + found=$(findmnt -n -o FSTYPE --target "$target" 2>/dev/null || true) + if [ -n "$found" ]; then + printf '%s\n' "$found" + return 0 + fi + fi + found=$(stat -f -c %T "$target" 2>/dev/null || true) + case "$found" in ext2/ext3) found=ext4 ;; esac + printf '%s\n' "$found" +} + +if [ "$uname_s" = Darwin ]; then + supported='apfs hfs' + supported_text='apfs or hfs' +else + supported='ext4 xfs btrfs' + supported_text='ext4, xfs or btrfs' +fi + +refuse_filesystem() { + local target=$1 label=$2 found=$3 + refuse "$label=$target is on '$found', which cannot carry a Store." \ + "" \ + "Direct Store access needs a local filesystem that carries SQLite's locks:" \ + "$supported_text. The kernel guard refuses tmpfs, overlay, FUSE and network" \ + "mounts before any Store file is opened, so the suites would fail with" \ + "'does not carry SQLite locks' -- an environment failure, not a product one." \ + "" \ + "Point LAYERFS_TEST_TMPDIR at a supported directory, for example:" \ + " LAYERFS_TEST_TMPDIR=/mnt/ext4/layerfs-store-tests $0 --quick" \ + "" \ + "Inside a Docker Sandbox the run directory is ext4 (\$HOME and" \ + "/layerfs/data/machine/native/run/); /tmp is tmpfs there." +} + +check_supported() { + local target=$1 label=$2 found + found=$(filesystem_type "$target") + case " $supported " in + *" ${found:-unknown} "*) ;; + *) refuse_filesystem "$target" "$label" "${found:-unknown}" ;; + esac +} + +check_supported "$dir" LAYERFS_TEST_TMPDIR + +probe="$dir/.layerfs-store-tests.$$" +if ! ( : > "$probe" ) 2>/dev/null; then + refuse "LAYERFS_TEST_TMPDIR=$dir is not writable by $(id -un 2>/dev/null || echo this user)." +fi +rm -f -- "$probe" + +# ---- Point the tests at it ------------------------------------------------ + +export TMPDIR="$dir/tmp" +export CARGO_TARGET_DIR="${LAYERFS_TEST_CARGO_TARGET_DIR:-$dir/target}" +export CARGO_HOME="${LAYERFS_TEST_CARGO_HOME:-$dir/cargo-home}" +mkdir -p -- "$TMPDIR" "$CARGO_TARGET_DIR" "$CARGO_HOME" 2>/dev/null \ + || refuse "cannot create TMPDIR, CARGO_TARGET_DIR and CARGO_HOME under $dir" + +# Unix-domain sockets live under TMPDIR, and Linux rejects a socket path of 108 +# bytes or more. A Sandbox run directory is long enough that its own path plus +# a test directory would overrun that, so TMPDIR itself is short: a private +# wrapper under the system temporary directory whose `tmp` entry is a symlink +# to the real one. statfs follows the symlink, so the Stores still land on +# `$dir` filesystem; the wrapper holds nothing but the link. +system_tmp=${LAYERFS_TEST_SYSTEM_TMP:-/tmp} +if [ -d "$system_tmp" ] && [ -w "$system_tmp" ]; then + tmp_wrapper=$(mktemp -d "$system_tmp/layerfs-store-tests.XXXXXX") \ + || refuse "cannot create a short TMPDIR wrapper under $system_tmp" + ln -s "$dir/tmp" "$tmp_wrapper/tmp" \ + || { rm -rf -- "$tmp_wrapper"; refuse "cannot link $system_tmp/.../tmp to $dir/tmp"; } + export TMPDIR="$tmp_wrapper/tmp" + trap 'if [ -n "${tmp_wrapper:-}" ]; then rm -rf -- "$tmp_wrapper"; fi' EXIT +else + export TMPDIR="$dir/tmp" +fi + +# `CARGO_TARGET_TMPDIR` follows CARGO_TARGET_DIR, and the tests also read TMPDIR: +# both must be on the supported filesystem, including when they are overridden. +check_supported "$TMPDIR" TMPDIR +check_supported "$CARGO_TARGET_DIR" CARGO_TARGET_DIR + +# The pinned fuser dependency is fetched over git. The git CLI honours the +# host's proxy and credential configuration where libgit2 does not, and some +# TLS-inspecting hosts need verification off for that fetch. Scoped to this +# process; not a production setting. +export CARGO_NET_GIT_FETCH_WITH_CLI=true +export GIT_SSL_NO_VERIFY=true + +# The official Rust image keeps rustup and the toolchain outside HOME. +if [ -z "${RUSTUP_HOME:-}" ] && [ -d /usr/local/rustup ] && [ -x /usr/local/cargo/bin/rustup ]; then + export RUSTUP_HOME=/usr/local/rustup +fi + +if ! command -v cargo >/dev/null 2>&1; then + for candidate in "$CARGO_HOME/bin" "${HOME:-}/.cargo/bin" /usr/local/cargo/bin; do + if [ -x "$candidate/cargo" ]; then + PATH="$candidate:$PATH" + export PATH + break + fi + done +fi +if ! command -v cargo >/dev/null 2>&1; then + refuse "cargo is not installed or not on PATH; install Rust 1.85.1 (rustup) first." +fi + +cargo_cmd=(cargo) +if command -v rustup >/dev/null 2>&1 && rustup toolchain list 2>/dev/null | grep -q '^1\.85\.1'; then + cargo_cmd=(cargo +1.85.1) +fi + +if [ -n "${LAYERFS_TEST_JOBS:-}" ]; then + case "$LAYERFS_TEST_JOBS" in + ''|*[!0-9]*) refuse "LAYERFS_TEST_JOBS must be a positive integer, not '$LAYERFS_TEST_JOBS'." ;; + esac + jobs=$LAYERFS_TEST_JOBS +else + jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '4') + if [ "$jobs" -gt 16 ]; then + jobs=16 + fi +fi +[ "$jobs" -ge 1 ] || refuse "LAYERFS_TEST_JOBS must be a positive integer, not '$jobs'." + +# ---- The suites ----------------------------------------------------------- + +ran=0 +failed=0 + +show_command() { + printf ' ' + printf ' %q' "$@" + printf '\n' +} + +run_cargo() { + local package=$1 + shift + local -a cmd=( "${cargo_cmd[@]}" test --locked -j "$jobs" -p "$package" ) + if [ -n "${LAYERFS_TEST_POSTGRES_URL:-}" ]; then + cmd+=( --features postgresql ) + fi + cmd+=( "$@" ) + if [ "${#passthrough[@]}" -gt 0 ]; then + cmd+=( -- "${passthrough[@]}" ) + fi + show_command "${cmd[@]}" + if [ "$list_only" -eq 1 ]; then + return 0 + fi + printf '\n==> %s:%s\n' "$package" "$*" + ran=$((ran + 1)) + if "${cmd[@]}"; then + printf 'PASS %s\n' "$package" + else + failed=$((failed + 1)) + printf 'FAIL %s\n' "$package" >&2 + fi +} + +# ---- The PostgreSQL prerequisites ----------------------------------------- +# +# The full-mode doctor calls amcheck's verify_heapam and bt_index_check on +# every heap and ready B-tree relation, and the PostgreSQL compaction suites +# assert a clean full audit. A database without amcheck reports +# store.database.relations=Unknown, so those tests fail with a diagnostic +# finding, not an environment refusal. The engine never installs the +# extension: warn, never fail, so the SQLite-only lane and a deliberately +# amcheck-free server stay usable. psql is optional too; without it the check +# is skipped with a clear message. +postgresql_extension_count() { + psql "$LAYERFS_TEST_POSTGRES_URL" -X -A -t -v ON_ERROR_STOP=1 \ + -c "SELECT count(*) FROM pg_catalog.pg_extension WHERE extname='$1'" 2>/dev/null +} + +warn_about_postgresql_prerequisites() { + if ! command -v psql >/dev/null 2>&1; then + printf '%s\n' \ + "test_store_ext4: LAYERFS_TEST_POSTGRES_URL is set but psql is not on PATH;" \ + " the amcheck precondition was not checked. Full-mode diagnostics report" \ + " store.database.relations=Unknown without it. Create it out of band:" \ + " CREATE EXTENSION IF NOT EXISTS amcheck;" >&2 + return 0 + fi + local extension count + for extension in amcheck pgstattuple; do + if ! count=$(postgresql_extension_count "$extension"); then + printf 'test_store_ext4: cannot query LAYERFS_TEST_POSTGRES_URL; the %s precondition was not checked.\n' "$extension" >&2 + return 0 + fi + if [ "${count:-0}" -gt 0 ]; then + continue + fi + if [ "$extension" = amcheck ]; then + printf '%s\n' \ + "test_store_ext4: warning: the PostgreSQL test database has no amcheck extension." \ + " The full-mode doctor reports store.database.relations=Unknown and the" \ + " PostgreSQL compaction suites fail their clean-audit assertions (T-G14)." \ + " Create it in that database, then re-run:" \ + " psql \"\$LAYERFS_TEST_POSTGRES_URL\" -c 'CREATE EXTENSION IF NOT EXISTS amcheck;'" >&2 + else + printf 'test_store_ext4: note: %s is not installed (companion extension; amcheck is the required one).\n' "$extension" >&2 + fi + done + return 0 +} + +printf '%s\n' \ + "Store test harness" \ + " repository: $root" \ + " LAYERFS_TEST_TMPDIR: $dir ($(filesystem_type "$dir"))" \ + " TMPDIR: $TMPDIR" \ + " CARGO_TARGET_DIR: $CARGO_TARGET_DIR" \ + " CARGO_HOME: $CARGO_HOME" \ + " jobs (-j): $jobs" \ + " mode: $mode" \ + " PostgreSQL: $([ -n "${LAYERFS_TEST_POSTGRES_URL:-}" ] && echo 'on (LAYERFS_TEST_POSTGRES_URL set)' || echo 'off (LAYERFS_TEST_POSTGRES_URL unset; PostgreSQL cases print NOT_RUN)')" + +if [ -n "${LAYERFS_TEST_POSTGRES_URL:-}" ] && [ "$list_only" -eq 0 ]; then + warn_about_postgresql_prerequisites +fi + +if [ "$mode" = quick ]; then + run_cargo layerfs-persistence --test heads --test sqlite_publication + run_cargo layerfs-store-service --test kernel_guard --test commit_pipeline + run_cargo layerfs-store --test create_directory + run_cargo layerfs-api --test store_family +else + run_cargo layerfs-persistence + run_cargo layerfs-store-service + run_cargo layerfs-store + api_store_targets=() + for file in api/crates/layerfs-api/tests/store_*.rs; do + [ -f "$file" ] || continue + target=${file##*/} + api_store_targets+=( --test "${target%.rs}" ) + done + [ "${#api_store_targets[@]}" -gt 0 ] \ + || refuse "no api/crates/layerfs-api/tests/store_*.rs targets found; is $root the repository root?" + run_cargo layerfs-api "${api_store_targets[@]}" +fi + +if [ "$list_only" -eq 1 ]; then + exit 0 +fi + +printf '\n%d suite(s), %d failed\n' "$ran" "$failed" +if [ "$failed" -ne 0 ]; then + exit 1 +fi