diff --git a/.agents/skills/layerfs-js-sdk/SKILL.md b/.agents/skills/layerfs-js-sdk/SKILL.md index 83de9a92e3..2d67c5be08 100644 --- a/.agents/skills/layerfs-js-sdk/SKILL.md +++ b/.agents/skills/layerfs-js-sdk/SKILL.md @@ -285,20 +285,23 @@ For Rust projects, follow the [Cargo workflow](references/rules/cargo.md). ## Calls -61 calls in nine families. Each family's page has arguments, results, refusals +85 calls in twelve families. Each family's page has arguments, results, refusals and examples. | Family | Calls | | --- | --- | -| [project](references/methods/project.md) | `init(project, folder, { timeout, on_progress })`, `list(store?, { page_size, page_pos })`, `get(project)`, `export(project, destination, { branch, on_progress })`, `remove(project)` | -| [branch](references/methods/branch.md) | `list(project)`, `get(project, branch)`, `fork(project, name, from_branch, from_commit?)`, `merge(project, name, a, b, { message, units })`, `publish(project, branch, commit?)`, `diff(project, from_branch, to_branch, { paths_only, page_size, page_pos, cursor })` | -| [commit](references/methods/commit.md) | `list(project, branch)`, `get(project, branch, commit)`, `diff(project, from_commit, to_commit, { paths_only, … })` | -| [file](references/methods/file.md) | reads `read`, `stat`, `list`, `glob`, `grep`, `blame`, `diff`; writes `write`, `edit`, `rm`, `mkdir`, `mv`, `cp`, `apply` | -| [sandbox](references/methods/sandbox.md) | `create(project, name, machine?, { image })`, `list(project?, { page_size, page_pos })`, `get(name, { page_size, page_pos })`, `close(name, { force })` | -| [workspace](references/methods/workspace.md) | `mount(sandbox, branch, { name })`, `exec(ws, command, { message, stream, stdout, stderr })`, `commit(ws, { message, force })`, `unmount(ws, { discard, force })`, `status(ws)` | -| [machine](references/methods/machine.md) | `list`, `platforms`, `get`, `status`, `start`, `stop`, `create`, `remove`, `update` | -| [store](references/methods/store.md) | `list`, `get`, `status`, `create`, `connect`, `push`, `pull`, `remove`, `disconnect` | -| [backup](references/methods/backup.md) | `connect`, `disconnect`, `list`, `create`, `restore` | +| home (admin) | `init()`, `get()`: create or convert this home, and read the bundle and daemon this build would use; no skill page yet, see `docs/architecture/api/sdk/home.md` in the repository | +| [project](references/methods/project.md) | `init(project, folder, { timeout, on_progress })`, `list(store?, { page_size, page_pos })`, `get(project)`, `stats(project, branch?, commit?)`, `export(project, destination, { branch, on_progress })`, `delete(project)`, `remove_file(project, path)` | +| [branch](references/methods/branch.md) | `list(project, { page_size, page_pos, page })`, `get(project, branch)`, `commits(project, branch, { page, page_size, at_commit, include_storage })`, `get_commit(project, branch, commit)`, `fork(project, name, from_branch, from_commit?)`, `merge(project, name, a, b, { message, units })`, `publish(project, branch, commit?)`, `diff(project, from_branch, to_branch, { paths_only, page_size, page_pos, cursor, page, from_commit, to_commit, path_prefix, include_attribution, continuation })`, `remove(project, branch)` | +| [commit](references/methods/commit.md) | `list(project, branch, { page_size, page_pos })`, `get(project, branch, commit)`, `diff(project, from_commit, to_commit, { paths_only, page_size, page_pos, cursor })` | +| [file](references/methods/file.md) | reads `read(project, branch, path, commit?, { start_line, end_line })`, `stat(project, branch, path, commit?)`, `list(project, branch, path, commit?, { page_size, page, page_pos })`, `glob(project, branch, path, pattern, commit?, { page_size, page, page_pos })`, `grep(project, branch, path, pattern, commit?, { page_size, page, page_pos })`, `revision(project, branch, path, { at_commit, traversal, page, page_size, continuation })`, `blame(project, branch, path, commit?, { from_line, line_count })`, `diff(project, left_branch, left_path, right_branch, right_path, { left_commit, right_commit, include_attribution, allow_missing, page, page_size })`; writes `write(project, branch, path, content, { message, mode })`, `edit(project, branch, path, { old, new, message })`, `rm(project, branch, path, { message })`, `mkdir(project, branch, path, { message, mode })`, `mv(project, branch, from_path, to_path, { message })`, `cp(project, branch, from_path, to_path, { message })`, `apply(project, branch, changes, { message })` | +| [sandbox](references/methods/sandbox.md) | `create(project, name, machine?, { image, on_progress })`, `list(project?, { page_size, page_pos })`, `get(name, { page_size, page_pos })`, `close(name, { force })`, `files(sandbox, path?, { depth, cursor, limit, offset, length })` | +| [workspace](references/methods/workspace.md) | `mount(sandbox, branch, { name })`, `exec(ws, command, { message, stream, stdout, stderr, exec_id, on_progress })`, `exec_cancel(ws, exec_id, { grace_ms })`, `commit(ws, { message, force })`, `unmount(ws, { discard, force })`, `status(ws)` | +| [machine](references/methods/machine.md) | `list()`, `platforms()`, `get(name)`, `images(name)`, `load(name, image, { on_progress })`, `prune(name, image?, { unused })`, `status(name)`, `stats(machine)`, `space(machine)`, `files(machine, path?, { depth, cursor, limit, offset, length })`, `start(name, { on_progress })`, `stop(name, { force })`, `create(name, platform, { cpus, memory, disk, image, os_image, on_progress })`, `remove(name)`, `update(name, { cpus, memory, image, docker, wsl, daemon })` | +| [store](references/methods/store.md) | `list()`, `get(name)`, `status(name)`, `stats(name)`, `create(name)`, `connect(name, url, { password, synchronous_commit })`, `attach(name, machine, { url, password })`, `detach(name, machine)`, `push(name, machine, { on_progress })`, `pull(name, { on_progress })`, `remove(name)`, `compact(name)`, `disconnect(name)` | +| [backup](references/methods/backup.md) | `connect(name, url, { key, region, part_size })`, `disconnect(name)`, `list(bucket?)`, `create(store, bucket, file?, { on_progress })`, `restore(store, bucket, file?, { on_progress })` | +| [logs](references/methods/logs.md) | `list({ op, project, client, tag, outcome, since, cursor, limit })`, `reset()`, `machine(machine, { cursor, before, limit })`, `sandbox(sandbox, { cursor, before, limit })` | +| [doctor](references/methods/doctor.md) | `fix(id)` | One top-level function is outside the families: `doctor({ home })` (shell `layerfs doctor [--text]`) reports what is wrong in the home as `{ ok, diff --git a/.agents/skills/layerfs-js-sdk/references/methods/branch.md b/.agents/skills/layerfs-js-sdk/references/methods/branch.md index 0cc8d154ea..783c744fb2 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/branch.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/branch.md @@ -5,12 +5,15 @@ > command in a private home on 2026-11-02; ids are shortened. ```js -branch.list(project, { page_size, page_pos }) +branch.list(project, { page_size, page_pos, page }) branch.get(project, branch) +branch.commits(project, branch, { page, page_size, at_commit, include_storage }) +branch.get_commit(project, branch, commit) branch.fork(project, name, from_branch, from_commit?) branch.merge(project, name, a, b, { message, units }) -branch.publish(project, branch, commit?) -branch.diff(project, from_branch, to_branch, { paths_only, page_size, page_pos, cursor }) +branch.publish(project, branch, commit?, { expected_base, expected_head, force }) +branch.diff(project, from_branch, to_branch, { paths_only, page_size, page_pos, cursor, page, from_commit, to_commit, path_prefix, include_attribution, continuation }) +branch.remove(project, branch) ``` `main` is not a Branch: it is the project's published line of Layers. It shows @@ -19,7 +22,7 @@ in `branch.list`, you can read it and fork from it, and it changes only through ## `branch.list` and `branch.get` -`branch.list(project, { page_size, page_pos })`, `branch.get(project, branch)` +`branch.list(project, { page_size, page_pos, page })`, `branch.get(project, branch)` ```bash layerfs branch list shop/api --text @@ -36,6 +39,46 @@ Layer underneath; `head` is the newest Commit, `null` before the first one. `branch.get` returns one item, or `NoSuchBranch` (`no Branch `; `NotFound` before `88b033cfa9`). +## `branch.commits` + +`branch.commits(project, branch, { page, page_size, at_commit, include_storage })` + +Saved side-Branch Commits, or `main`'s publication Layers, newest first from +an immutable start. `page` and `page_size` number the pages; `at_commit` (a +full saved id) starts from another point; `include_storage` adds each row's +`new_canonical`, `new_stored` and `new_metadata` (`null` for a main Layer, +which performs no Save). + +```json +{"branch":"work","at_commit":{"id":"12ad2061f04f…","kind":"Commit","branch":"work","root":"f55fa9660a73…"}, + "items":[{"id":"12ad2061f04f…","parent":"12c7ef811a49…","base":"3275838904b6…", + "time":1793574061147,"via":"fs","message":"add retries","message_truncated":false, + "details_omitted":true}], + "page":1,"page_size":50,"has_more":false,"scan":{"complete":true,"examined":1,"reason":null}} +``` + +Each item carries a full `ref: { id, kind, branch, root }`; `branch.get_commit` +answers the rest. A side Branch follows its first-parent chain; `main` follows +publication Layer parents, and a Layer's `time` is `null` with +`time_status: "unavailable"` (its `source_time` is the source Commit's own +time). A side Branch with no Commit has no rows. + +## `branch.get_commit` + +`branch.get_commit(project, branch, commit)` + +Full details of one saved Commit or `main` publication Layer admitted by the +Branch. `commit` is the complete id: no prefix matching (unlike `commit.get`). + +The answer keeps the recorded metadata and adds typed `ref`, `parent_ref`, +`second_parent_ref`, `base_layer`, `source_commit_ref`, `source_time` and +`time_status`; a relationship that does not apply is null. For a publication, +`parent` is the previous Layer and `source_commit` what was published. `main` +admits its Layer lineage; a side Branch admits its first-parent and base +lineage and its own recorded Commits, displaced ones included. The context +lookup visits at most 10,000 records, and the whole answer, merge-unit +reports included, must fit 48 KiB: either limit is `Capacity`. + ## `branch.fork` `branch.fork(project, name, from_branch, from_commit?)` @@ -185,5 +228,24 @@ result is the same shape as [`commit.diff`](commit.md#commitdiff): layerfs branch diff shop/api main work --paths-only ``` -`paths_only` leaves out each item's `hunks` (they come back `null`). Resume a +`paths_only` leaves out each item's `hunks` (they come back `null`). `page`, +`page_size` and `page_pos` page the items; `from_commit` and `to_commit` pin +the compared versions; `path_prefix` keeps one subtree; `include_attribution` +adds each path's last writers; `continuation` resumes a saved query. Resume a long diff with `cursor: next_cursor`. Refused under a folder scope. + +## `branch.remove` + +`branch.remove(project, branch)` + +Retires one Branch in its Store. Its row is hidden from listings and name +resolution; fork, merge, publish and mount of it are refused, and its Commits +and objects stay until a later `store.compact`. `main` is `ReservedName`. + +```json +{"kind":"Removed","project":"shop/api","branch":"work","removed_ms":1793574061147} +``` + +A repeat is `NoSuchBranch` (only `project.delete` is retry-idempotent); a +concurrent remove is `Busy` with phase `"retire"`. `MountsActive` while a +mount of the Branch is live. diff --git a/.agents/skills/layerfs-js-sdk/references/methods/doctor.md b/.agents/skills/layerfs-js-sdk/references/methods/doctor.md new file mode 100644 index 0000000000..34106ba617 --- /dev/null +++ b/.agents/skills/layerfs-js-sdk/references/methods/doctor.md @@ -0,0 +1,32 @@ +# doctor calls + +> **Status:** Current general guide for LayerFS 2.0 (#533). Part of the +> `layerfs-js-sdk` skill. `doctor` is the report function, outside the table; +> `doctor.fix` is the family's one call. A folder scope refuses it. + +```js +doctor.fix(id) +``` + +`layerfs doctor` reports what is wrong in the home as +`{ ok, findings: [{ severity, entity: { kind, name }, message, fix }] }`, each +finding naming the call that fixes it. It changes nothing and starts no +machine. `doctor.fix(id)` removes the one leftover `id` names. + +## `doctor.fix` + +`doctor.fix(id)`: `id` is a leftover's `::` from the +report; it never holds a path. One attempt: nothing is retried, and nothing +but the one entry is removed. The kinds are `run-folder`, `empty-root`, +`init-copy`, `backup-staging`, `image-tmp` and `image-part`. + +```json +{"id":"native:run-folder:web","machine":"native","kind":"run-folder", + "path":"/Users/me/.layerfs/machine/native/run/web","removed":true,"bytes":11833344} +``` + +`bytes` is what the removed files held, `null` for a folder copy and an image +temporary. Errors: `NotFound` (no leftover has that id now), `InUse` (the +kind's guard does not hold; nothing removed), `InvalidInput` (a kind that is +only reported, a missing `id`, a folder scope), `Unavailable` (the machine is +stopped), `Unknown` (the worker's answer was lost), `Failed`. diff --git a/.agents/skills/layerfs-js-sdk/references/methods/file.md b/.agents/skills/layerfs-js-sdk/references/methods/file.md index 92295dd20c..2088f71303 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/file.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/file.md @@ -9,11 +9,12 @@ // reads: the Branch's head, or the Commit given file.read(project, branch, path, commit?, { start_line, end_line }) file.stat(project, branch, path, commit?) -file.list(project, branch, path, commit?, { page_size, page_pos }) -file.glob(project, branch, path, pattern, commit?, { page_size, page_pos }) -file.grep(project, branch, path, pattern, commit?, { page_size, page_pos }) -file.blame(project, branch, path, commit?, { from_line }) -file.diff(project, left_branch, left_path, right_branch, right_path, { left_commit, right_commit }) +file.list(project, branch, path, commit?, { page_size, page, page_pos }) +file.glob(project, branch, path, pattern, commit?, { page_size, page, page_pos }) +file.grep(project, branch, path, pattern, commit?, { page_size, page, page_pos }) +file.revision(project, branch, path, { at_commit, traversal, page, page_size, continuation }) +file.blame(project, branch, path, commit?, { from_line, line_count }) +file.diff(project, left_branch, left_path, right_branch, right_path, { left_commit, right_commit, include_attribution, allow_missing, page, page_size }) // writes: one Commit on the Branch's head each file.write(project, branch, path, content, { message, mode }) @@ -166,6 +167,35 @@ over. It searches everything under `path`, `node_modules` included, so keep "skipped":0,"has_more":false,"next_page_pos":null} ``` +## `file.revision` + +Saved changes to one literal path, deletions and metadata-only changes +included; no rename or copy ancestry is inferred. + +```js +const rev = await layerfs.file.revision(P, "work", "/src/index.js", { page_size: 20 }); +console.log(rev.items.map((c) => `${c.kind} ${c.ref.id.slice(0, 12)}`).join("\n")); +``` + +```json +{"kind":"ok","at":{"id":"12ad2061f04f…","kind":"Commit","branch":"work","root":"f55fa9660a73…"}, + "path":"/src/index.js","traversal":"provenance", + "items":[{"candidate_index":0,"ref":{"id":"12ad2061f04f…","kind":"Commit","branch":"work","root":"f55fa9660a73…"}, + "compared_to":{"id":"12c7ef811a49…","kind":"Commit","branch":"work","root":"0a12a8d94231…"}, + "kind":"Modified","changes":{"content":true,"metadata":true,"identity":false,"attribution":null}, + "message":"add retries","recorded_at":1793574061147,"time_status":"recorded"}], + "page":1,"page_size":20,"has_more":false,"end_of_query":true,"continuation":null, + "scan":{"complete":true,"candidates_examined":1,"reasons":[]}} +``` + +`traversal` is `provenance` (default: a Commit's first parent or base Layer, +a Layer's source Commit or parent Layer, as blame walks) or `branch` (a side +Branch's first parents, or `main`'s Layers in publication order). `at_commit` +pins a full Commit or Layer id; reuse the first answer's `at.id`. A path that +never existed answers an exhausted empty list. The request admits 10,000 +visited records and 64 MiB of content comparison; past either it answers a +partial page with `scan.complete: false`, never a silent cap. + ## `file.blame` The Commit of each line, from `from_line` (default 1). diff --git a/.agents/skills/layerfs-js-sdk/references/methods/logs.md b/.agents/skills/layerfs-js-sdk/references/methods/logs.md new file mode 100644 index 0000000000..c3c16f7971 --- /dev/null +++ b/.agents/skills/layerfs-js-sdk/references/methods/logs.md @@ -0,0 +1,63 @@ +# logs calls + +> **Status:** Current general guide for LayerFS 2.0 (#533). Part of the +> `layerfs-js-sdk` skill. An agent working on a project rarely needs this +> family; it belongs to whoever watches a run. A folder scope refuses every +> `logs` call. + +```js +logs.list({ op, project, client, tag, outcome, since, cursor, limit }) +logs.reset() +logs.machine(machine, { cursor, before, limit }) +logs.sandbox(sandbox, { cursor, before, limit }) +``` + +`logs.list` reads the home's call log (`trace/calls.jsonl`, then +`trace/calls.1.jsonl`); `logs.machine` and `logs.sandbox` read a machine's +worker log and a Sandbox's `daemon.log`. A call-log line is +`{ v, at, ms, client, argv, outcome, kind?, commit?, layer?, tag? }`, where +`at` is when the call started (ms since the epoch) and `ms` its duration. + +## `logs.list` + +`logs.list({ op, project, client, tag, outcome, since, cursor, limit })`: the +calls made against this home, newest first. Every option given must match: +`op` is `family` or `family.call`; `project` matches the lines' `argv`; +`client` is `cli`, `node`, `python` or `rust`; `tag` is an exact tag or a +prefix ending in `:`; `outcome` is `ok` or a refusal code; `since` keeps lines +at or after a time (ms). `limit` is 1 to 1000 (default 100), and `cursor` is a +previous answer's `next`. + +Returns `{ lines, next?, has_more, skipped }`; `skipped` counts torn lines +that do not parse, which are never answered. Errors: `InvalidInput` (a bad +option, `limit` out of range, a malformed or stale cursor), `Failed` (a log +file cannot be read). + +## `logs.reset` + +`logs.reset()`: clears the call log, truncating `trace/calls.jsonl` in place +and removing `trace/calls.1.jsonl`. Returns `{ kind: "Reset", removed }`, the +lines both files held. A cursor from before the reset is stale afterwards. +Error: `Failed`. + +## `logs.machine` + +`logs.machine(machine, { cursor, before, limit })`: one window of a machine's +`log/worker.log`, with `worker.1.log`. `limit` is 1 to 1000 (default 200); +`cursor` reads the lines after another answer's `cursor`, `before` the lines +before another answer's `before` (not with `cursor`). + +Returns `{ lines, cursor, before?, has_more }`: `lines` are oldest first, each +`{ text }`, or `{ text, continued: true }` for a piece of a longer line. +`cursor` is always text; pass it back to read what was written since. One call +reads at most 64 KiB, so an answer may hold fewer than `limit` lines with +`has_more` true. Errors: `InvalidInput` (both options, `limit` out of range, a +malformed or stale cursor), `Unsupported` (`native` has no worker), +`NotFound`, `Unavailable`, `Busy`, `Failed`. + +## `logs.sandbox` + +`logs.sandbox(sandbox, { cursor, before, limit })`: the same, over a +Sandbox's `daemon.log` in its run folder. Options, result and rules are +`logs.machine`'s. Errors add `NoSuchSandbox` and `Unsupported` (a retired +docker Sandbox record). diff --git a/.agents/skills/layerfs-js-sdk/references/methods/machine.md b/.agents/skills/layerfs-js-sdk/references/methods/machine.md index 6a3114ead8..65a7045951 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/machine.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/machine.md @@ -16,12 +16,18 @@ no `machine.connect` or `machine.disconnect`. See machine.list() machine.platforms() machine.get(name) +machine.images(name) +machine.load(name, image, { on_progress }) +machine.prune(name, image?, { unused }) machine.status(name) -machine.start(name) -machine.stop(name) -machine.create(name, platform, { cpus, memory, disk, image }) +machine.stats(machine) +machine.space(machine) +machine.files(machine, path?, { depth, cursor, limit, offset, length }) +machine.start(name, { on_progress }) +machine.stop(name, { force }) +machine.create(name, platform, { cpus, memory, disk, image, os_image, on_progress }) machine.remove(name) -machine.update(name, { cpus, memory, image, docker, wsl }) +machine.update(name, { cpus, memory, image, docker, wsl, daemon }) ``` A machine is one kernel where Sandboxes run. Two are built in and always @@ -77,15 +83,89 @@ or `machine.remove`). `NotFound` for a name that isn't recorded. +### `machine.images` + +`machine.images(name)`: the Sandbox images the machine holds and the host's +Docker images it could copy, read without starting it. `mode` is `images`, or +`host` on macOS `native` (no rows, `docker: null`). `available` lists the +host's tagged Docker images not already unpacked; a row for another OS or +architecture is `usable: false` with a `reason`. A Docker failure fills +`docker.available: false` and never fails the call. + +```text +{ machine, mode, architecture, default, free, + images: [{ id, refs, size, architecture, source, sandboxes, last_used }], + docker: null | { available: true } | { available: false, reason, message }, + available: [{ id, refs, size, os, architecture, usable, reason?, need, fits }] } +``` + +Errors: `NotFound`; `Unavailable` when the machine's worker does not answer. + +### `machine.stats` + +`machine.stats(machine)`: what the machine has and uses now, for a UI that +polls. It answers the present only: processor values are counters, and a +client keeps its own history and turns two reads into a rate. + +```text +{ machine, platform, observed_at, worker, resources, data_disk, stores, + sandboxes: { recorded, live, items: [{ name, live, live_reason?, started_at, usage, usage_reason? }] }, + complete, reason? } +``` + +`worker` is the machine's worker process (`null` on `native`); `resources` +carries `scope`: `host`, `vm` or `engine_host`. `sandboxes.items` has one row +per recorded Sandbox; a worker of an older build answers none. `complete: +false` with `reason: "deadline"` means the 4 s soft budget ended the Store or +Sandbox part early. Nothing is locked, opened or started, and PostgreSQL +Stores are not listed. Errors: `NotFound`, `Unavailable`, `Busy`, +`InvalidInput`. + +### `machine.space` + +`machine.space(machine)`: where the data disk goes, for a UI that warns +before it fills. Every size is the blocks a part takes. + +```text +{ machine, platform, observed_at, disk, layerfs, outside, + parts: { stores, sandboxes, images, other_runs, logs, other }, + stores: [{ name, bytes }], + sandboxes: [{ name, bytes, overlays, writes, home, log, runtime, reason? }], + unreadable, complete, reason? } +``` + +`outside` is `disk.used` less `layerfs`, never below 0. `complete: false` +when the walk stopped at 250,000 entries or its 4 s budget (`reason` joins +`deadline`, `unreadable`, `disk_not_read`); a part not reached is `null`. It +takes no lock, dials no daemon and writes nothing, the call log included. +Errors as `machine.stats`. + +### `machine.files` + +`machine.files(machine, path?, { depth, cursor, limit, offset, length })`: +reads what is on a machine, for a UI that shows a tree. Its options, a folder +page, a file window and their fields are [`sandbox.files`](sandbox.md)'s; +what differs is where a path may lead. + +With no `path` it answers the roots that exist, each with a `role` (`data`, +`stores`, `sandbox_records`, `run_folders`, `images`, `staging`, `logs`, +`state`, `bundle`, `image`, `machine`); a client offers a "whole filesystem" +view only when the answer has the `machine` root. On `native` that root is +absent: a path outside the LayerFS folders is `InvalidInput`. It takes no +lock, lease or daemon call, and writes nothing, the call log included. +Errors: `InvalidInput`, `NotFound`, `Unavailable`, `Busy`, `Unsupported`. + ## Changes (not run here) | Call | What it does | Notes | | --- | --- | --- | | `start(name)` | Starts a stopped machine and its worker and waits until ready | A running machine and `native` answer at once. Returns the machine as `get` does. `Busy` while another call holds the machine (its `op`) | | `stop(name)` | Stops the worker and, where there is one, the VM; disks stay | `InUse` while Sandboxes run on it; `detail.related` lists them as `{ kind: "sandbox", name }`. `native` answers at once. Later calls that need the machine are refused `Unavailable` until `start` | -| `create(name, platform, …)` | Makes and starts an enabled VM (`apple-container`, `lima`) | `Unsupported` when the platform's tool is missing; `InvalidInput` for `native` or `docker`; `Unsupported` for the removed `firecracker`, `ssh` and `microsandbox`. `memory`, `disk`: bytes or text with `K`, `M`, `G` | +| `load(name, image, …)` | Copies an image to the machine as `sandbox.create`'s `image` does, without a Sandbox | `image` is a digest, Docker image Id or reference. An image the machine holds is `reused: true` and needs neither Docker nor a byte sent. Progress `inspect`, `export`, `send`, `unpack`. `Unsupported` (`host` on macOS `native`; `arch`/`os`), `InvalidInput` (Docker has no such image), `Unavailable`, `NoSpace` | +| `prune(name, image?, …)` | Removes one unpacked image, or with `unused` every one no Sandbox uses except the default | A named in-use image is `InUse` (`detail.count`); `unused` skips it. Naming the default removes it and the next create unpacks it again. `freed` is free bytes after minus before. `InvalidInput`, `NotFound`, `InUse`, `Unsupported` | +| `create(name, platform, …)` | Makes and starts an enabled VM (`apple-container`, `lima`) | `Unsupported` when the platform's tool is missing; `InvalidInput` for `native` or `docker`; `Unsupported` for the removed `firecracker`, `ssh` and `microsandbox`. `memory`, `disk`: bytes or text with `K`, `M`, `G`; `os_image` sets the VM's operating-system image; `on_progress` reports the start stages plus `{ phase: "create", machine }` | | `remove(name)` | Deletes a VM and its disk; a removed platform's record is only forgotten | `InUse` while it holds Stores or Sandboxes. While it runs, the record holds its `op` and every call that would start, update, remove or use the machine is `Busy` | -| `update(name, …)` | Sets `native`/`docker`'s image, Docker socket (`docker`) and WSL distribution (`wsl`) | `InUse` while Sandboxes run on it; `wsl` only on a Windows host; resizing a VM (`cpus`, `memory`) is `Unsupported`, not built | +| `update(name, …)` | Sets `native`/`docker`'s image, Docker socket (`docker`), WSL distribution (`wsl`) and the daemon path (`daemon`) | `InUse` while Sandboxes run on it; `wsl` only on a Windows host; resizing a VM (`cpus`, `memory`) is `Unsupported`, not built | An `Unknown` outcome of a create or remove keeps its `op`, marked `uncertain`, with the `error`. Once that process is gone, diff --git a/.agents/skills/layerfs-js-sdk/references/methods/project.md b/.agents/skills/layerfs-js-sdk/references/methods/project.md index bbd29a11b0..de14ea91ab 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/project.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/project.md @@ -12,8 +12,10 @@ always named with its Store: `/`, such as `shop/api`. A bare project.init(project, folder, { timeout, on_progress }) project.list(store?, { page_size, page_pos }) project.get(project) +project.stats(project, branch?, commit?) project.export(project, destination, { branch, on_progress }) -project.remove(project) +project.delete(project) +project.remove_file(project, path) ``` ## `project.init` @@ -119,23 +121,60 @@ The project's record: its Store, the Store's kind and machine. {"project":"shop/api","store":"shop","name":"api","kind":"sqlite","machine":"native","created":1793574061265} ``` -## `project.remove` +## `project.stats` -`project.remove(project)` +`project.stats(project, branch?, commit?)` -Removes one project from its Store. Its Branches, Commits and `main` go with -it; there is no undo. The live result: +Entry counts of one pinned saved tree: `main` unless `branch` is given, and +that Branch's saved head unless `commit` is. Uncommitted Workspace changes +are not counted. ```json -{"stack":"3104401f11fb…","packs_kept":true,"project":"shop/api"} +{"project":"shop/api","at":{"id":"12ad2061f04f…","kind":"Commit","branch":"work","root":"f55fa9660a73…"}, + "files":12,"directories":3,"symlinks":0,"others":0,"logical":18432,"complete":true} ``` -`packs_kept: true`: the project's records are gone, but its stored content -stays in the Store file. Removing a project from a PostgreSQL Store is -`Unsupported` in this build. While a Sandbox of the project is recorded it is -refused `InUse`; `detail.related` lists each as `{ kind: "sandbox", name }` -(close them first). To delete a whole Store with every project in it, use -`store.remove`. +The root directory is excluded, hard links count per path and symlinks are +not followed; `logical` is the pinned tree's recorded regular-file bytes. The +walk shares the saved-read budget of 10,000 entries: past it the answer is +partial with `complete: false` and a `reason`, never a silent cap. A bad +`branch`/`commit` is `NoSuchBranch`/`NoSuchCommit`. -An agent working on a project never needs `remove`; it is for whoever owns the -home. +## `project.delete` + +`project.delete(project)` + +Retires one project in its Store and removes its home entry (spec §4). The +project disappears from `project.list` and `project.get`; its Layers, +Branches, Commits and stored content stay until a later `store.compact`, and +the Store stays addressable. The live result: + +```json +{"kind":"Removed","project":"shop/api","deleted_ms":1793574061265,"home_entry_removed":true} +``` + +The Store marker is the commit point. If the home entry cannot be removed the +call still succeeds with `home_entry_removed: false` and keeps a durable +intent, so the same call retried from the same home reuses `deleted_ms` and +finishes the entry. A mount of the project is refused `MountsActive`, another +user or operation `Busy`/`Unresolved`. + +## `project.remove_file` + +`project.remove_file(project, path)` + +Hides one absolute project path from every node at or below its marker +frontier (spec §4.3). It writes a marker only; no bytes are freed. The path is +normalized (one leading `/` stripped) and must be a canonical absolute POSIX +path. The live result: + +```json +{"kind":"Removed","project":"shop/api","path":"docs/old.md","frontier_serial":42,"removed_ms":1793574061265} +``` + +`kind` is `UpToDate` when no project node is above this path's marker +frontier. Later reads and stats of the path are refused `Removed`; a Commit +after the frontier that writes the path again is visible. + +An agent working on a project never needs `delete` or `remove_file`; they are +for whoever owns the home. diff --git a/.agents/skills/layerfs-js-sdk/references/methods/sandbox.md b/.agents/skills/layerfs-js-sdk/references/methods/sandbox.md index 1908f7693a..0fa6d70fca 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/sandbox.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/sandbox.md @@ -7,10 +7,11 @@ > (`{"items":[]}`, seen live). ```js -sandbox.create(project, name, machine?, { image }) +sandbox.create(project, name, machine?, { image, on_progress }) sandbox.list(project?, { page_size, page_pos }) sandbox.get(name, { page_size, page_pos }) sandbox.close(name, { force }) +sandbox.files(sandbox, path?, { depth, cursor, limit, offset, length }) ``` A Sandbox is named (unique in the home), bound to one project when created, and @@ -168,6 +169,35 @@ of it runs. (`{ closed, was }`, `was` being `gone` or `unresolved`; for an unresolved create, `left` says a Sandbox may remain on its machine). +## `sandbox.files` + +`sandbox.files(sandbox, path?, { depth, cursor, limit, offset, length })` + +Reads what is in a Sandbox: its roots, one page of a folder, or one window of +a file, in its home and, for a Sandbox with a root of its own, in that root. +It is a read for a UI that shows a tree: no lock, no lease, no daemon call, +nothing written, no call-log line, and the MCP server does not list it. + +With no `path` it answers the roots that exist, each with a `role`: +`workspace` (one per mounted Workspace), `home`, `log` and, for a Sandbox with +its own root, `root` (`/`). `depth` is 1 (default) to 32; `limit` is 1 to +1000 (default 500); `cursor` is a page's `next`; `offset` and `length` take +a file window (default 0 and 256 KiB, at most 1 MiB). + +A folder answers `{ kind: "folder", entries, next, has_more, reason? }`; an +entry is `{ path, depth, kind, size, mode, mtime }`, `kind` one of `file`, +`folder`, `symlink`, `mount`, `restricted` or `other`. A file answers +`{ kind: "file", size, offset, content, binary, next }`; `binary` is true when +the window holds a NUL byte, and `content` is then empty. + +Errors: `InvalidInput` (an option out of range, a `cursor` with no `path`, a +path that is not absolute or holds `..`, a symlink or a private entry, a path +outside the home when the Sandbox has no root of its own); `NotFound`; +`NoSuchSandbox`; `Unavailable` (the machine is stopped, its worker does not +answer, or the root is read while its daemon is not running); `Busy` (two +inspection reads already run on that machine); `Unsupported` (a kernel too old +for `openat2`, or a non-root reader of the root). + ## Sandbox lifetime A Sandbox runs until `sandbox.close(name)`; `machine.stop` is refused diff --git a/.agents/skills/layerfs-js-sdk/references/methods/store.md b/.agents/skills/layerfs-js-sdk/references/methods/store.md index 0b1bead4f1..c053178544 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/store.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/store.md @@ -11,11 +11,15 @@ store.list() store.get(name) store.status(name) +store.stats(name) store.create(name) store.connect(name, url, { password, synchronous_commit }) +store.attach(name, machine, { url, password }) +store.detach(name, machine) store.push(name, machine, { on_progress }) store.pull(name, { on_progress }) store.remove(name) +store.compact(name) store.disconnect(name) ``` @@ -54,15 +58,35 @@ machine is stopped, or whose worker does not answer, is `state: "down"` with `down_reason` and `file.unavailable`. `state` is `idle`, `in_use`, `busy` (a live `op`), `down` or `unresolved` (a gone process's uncertain `op`). +`store.stats(name)` is the Store-wide counters, read in constant work with no +history, tree or object scan: + +```json +{"name":"shop","kind":"sqlite","history_logical":18432,"canonical":217088,"stored":217088, + "metadata":32768,"file":212992,"free":0, + "init":{"entries":3,"files":2,"bytes":18432,"canonical":18432,"stored":18432,"metadata":4096,"millis":12}} +``` + +`history_logical` sums saved-tree bytes over Commit and Layer records (each +publication counts its tree again); `canonical` and `stored` are allocated +immutable content, not reclaimed by `project.delete`; `metadata` is accounted +metadata plus database overhead; `file` is database pages, not the file +length; `free` is SQLite free-page bytes (`null` for PostgreSQL). A sqlite +Store on another machine is read through its worker; a stopped worker is +`Unavailable`. + ## Changes | Call | What it does | Notes | | --- | --- | --- | | `create(name)` | An empty sqlite Store on `native` | Live result: the record above, with `projects: []`. `Exists` for a taken name | | `connect(name, url, …)` | Attaches a PostgreSQL database | `url` is `postgresql://@:/[?sslmode=…]` without a password (one in the user part or in any query parameter whose name contains `password` is `InvalidInput`); `password` is a reference, `env:LAYERFS_` or `file:/path`. `synchronous_commit` defaults to `off`. The same database under a second name is `Exists` | +| `attach(name, machine, …)` | Lets a machine run Sandboxes of a PostgreSQL Store | Checks from that machine that it reaches the database and its role may write; `url` and `password` replace the Store's for that machine, either alone. `InvalidInput` for sqlite; `Unsupported` when the platform runs no PostgreSQL Sandboxes; `Unavailable` (not reached, machine stopped); `Denied` (login or privilege); `Conflict` while the Store is disconnected or reconnected | +| `detach(name, machine)` | Stops a machine running Sandboxes of a PostgreSQL Store | Dials nothing; detaching `native` is allowed. `InvalidInput` (sqlite, or not attached there); `InUse` while a Sandbox of the Store is recorded on the machine | | `push(name, machine, …)` | Sends a sqlite Store's file from `native` to `machine` | `InvalidInput` unless the Store is on `native` (pull it first). Progress `{ phase, done, total }` on `on_progress` / `--progress`: `done` and `total` are bytes during `send`, null in the other phases (`check`, `snapshot`, `verify`, `install`, `record`, `retire`). `InUse` while the Store is held (Sandbox records, processes with it open; `detail.related` lists them as `{ kind, name }`); `Capacity` when the target has less free disk than the Store plus a headroom of 1 GiB (or a tenth of the Store), checked before anything changes; `Unavailable` naming `layerfs machine start ` for a stopped machine; while it runs, Store calls and operations on it are `Busy`, from the caller's process too. An uncertain push whose process is gone (it ended during install or record, or a step's answer was lost) leaves every later call `Unresolved`, naming `store status` and the push that clears it: a new `push` (or `pull`) first deletes the copy on the machine the record does not name, then runs. A push whose `retire` did not finish leaves the Store in use on `machine` and `last_transfer.retired: false`; the next `pull` deletes the old copy first | | `pull(name, …)` | Brings a sqlite Store's file from its machine to `native` | `InvalidInput` when it is already on `native`; otherwise as `push`. Between two machines: `pull`, then `push` | | `remove(name)` | Deletes a sqlite Store with every project in it | Returns `{ removed }`. `NotFound` for an unknown name (live); `Unavailable` for a Store whose machine is unknown; `Busy` while a live process pushes, pulls or removes it. It clears an `Unresolved` Store. Backups stay | +| `compact(name)` | Rebuilds the Store from its live history, reclaiming dead bytes | The only byte-reclaiming call: no confirm, force or dry run. Returns `{ kind: "Compacted", path, generation, … }` with `dead_bytes_bound`, `dead_bytes_refined`, `logical_discarded`, `backend_reusable`, `physical_shrink`, `kept_waste`, `peak_temp_bytes`, `residue`, `estimate` and `zero_reasons` (a list of `{field, reason}`). A Store with under 10% dead packs takes the `none` path and only marks. A sqlite Store on another machine is `Unsupported` (local-only route). `MountsActive`, `Busy`, `Unresolved`, `StaleStore`, `Conflict`, `Capacity`, `Unknown` | | `disconnect(name)` | Forgets a PostgreSQL Store; its data stays | `InUse` while another process holds the Store (its calls hold it while it stays open) or a Sandbox of it runs; `detail.related` lists them as `{ kind: "lease", name: }` and `{ kind: "sandbox", name }` | Pushing or pulling a Store moves where its Sandboxes can run: a Sandbox of a sqlite Store diff --git a/.agents/skills/layerfs-js-sdk/references/methods/workspace.md b/.agents/skills/layerfs-js-sdk/references/methods/workspace.md index aeef1cc0bf..b72023492d 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/workspace.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/workspace.md @@ -9,7 +9,8 @@ ```js workspace.mount(sandbox, branch, { name }) -workspace.exec(ws, command, { message, stream, stdout, stderr }) +workspace.exec(ws, command, { message, stream, stdout, stderr, exec_id, on_progress }) +workspace.exec_cancel(ws, exec_id, { grace_ms }) workspace.commit(ws, { message, force }) workspace.unmount(ws, { discard, force }) workspace.status(ws) @@ -96,7 +97,7 @@ mounted; `ws` is then `:`. ## `workspace.exec` -`workspace.exec(ws, command, { message, stream, stdout, stderr })` +`workspace.exec(ws, command, { message, stream, stdout, stderr, exec_id, on_progress })` Runs one command with `bash -c` in the Workspace and waits for it to end. The working directory is the Workspace root, so use relative paths @@ -128,6 +129,11 @@ tests, `f67d5a9f6f`, `771823d728`, `94671c974c`, `37f1bbd2b9`): - Output over 1 MiB arrives only this way, 1 MiB per piece; without a function, `stream` or `--stream` those bytes are not kept. +`exec_id` (1–64 of `A-Z a-z 0-9 . _ -`, not starting with `.` or `-`) names +the running command; it defaults to `exec-<16 hex>` and must be unique among +the Workspace's running commands (`Exists` otherwise). `on_progress` receives +one event before the command starts: `{ exec: { id } }`. + **Limits** - No timeout: put `timeout 110` inside the command. @@ -154,9 +160,28 @@ const run = await layerfs.workspace.exec(ws, `printf '%s' ${b64} | base64 -d > s ``` **Refusals:** `NoSuchSandbox` (the Sandbox is gone: its Workspaces are gone -too; mount again elsewhere), `NotMounted`, `Unknown` (the exit code is not +too; mount again elsewhere), `NotMounted`, `Exists` (that `exec_id` is +running in this Workspace), `Unknown` (the exit code is not known: the command may or may not have run; stop). +## `workspace.exec_cancel` + +`workspace.exec_cancel(ws, exec_id, { grace_ms })` + +Ends a running command from any process: SIGTERM to everything it started, +then SIGKILL after `grace_ms` (default 5000, not negative; 0 sends SIGKILL +right after SIGTERM). One explicit attempt; there is no operation timeout. + +| Answer | Meaning | +| --- | --- | +| `{ outcome: "cancelled", signal }` | `SIGTERM` when the command was gone within the grace, else `SIGKILL`; a command not yet running is marked and answered `SIGKILL`, and its owner ends it once it runs | +| `{ outcome: "ended" }` | no record, or nothing left; with no record, a pending mark is kept for 30 s and cancels a command registered with that id in that time | +| `{ outcome: "unknown" }` | the worker was not reached or did not answer; call again once it answers | + +A cancelled `workspace.exec` answers `kind: "cancelled"` and its code is +128 + the signal. Error: `Failed` (a record that does not read, or the signal +could not be sent). + ## `workspace.commit` `workspace.commit(ws, { message, force })` diff --git a/.agents/skills/layerfs-sdk-observe/SKILL.md b/.agents/skills/layerfs-sdk-observe/SKILL.md index 33256ed601..8eaf6a9c43 100644 --- a/.agents/skills/layerfs-sdk-observe/SKILL.md +++ b/.agents/skills/layerfs-sdk-observe/SKILL.md @@ -15,7 +15,7 @@ description: Use when observing, measuring or troubleshooting a LayerFS 2.0 run > private home on 2026-11-02, unless marked "from source". Everything here is a read: it changes no project. 2.0 has no separate -diagnostic package; the reads are ordinary calls of the nine families, open to +diagnostic package; the reads are ordinary calls of the twelve families, open to every caller. | Question | 2.0 source | Page | diff --git a/.agents/skills/layerfs-sdk-observe/references/tracing.md b/.agents/skills/layerfs-sdk-observe/references/tracing.md index 316686abfa..5f3534511c 100644 --- a/.agents/skills/layerfs-sdk-observe/references/tracing.md +++ b/.agents/skills/layerfs-sdk-observe/references/tracing.md @@ -19,12 +19,16 @@ Python, direct `layerfs_api::call`) appends one line to and `LAYERFS_TRACE` / `LAYERFS_TRACE_STEP` do not affect it. ```json -{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","/index.ts","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} +{"v":1,"at":1793000004120,"ms":212,"client":"node","argv":["file","write","demo/app","main","<8 bytes>","<5321 bytes>"],"outcome":"ok","kind":"Committed","commit":"<66 hex>","tag":"dsh:s_4d2a:a91e"} ``` It keeps 1.1.0's redaction rule (values cut to 200 characters; contents, -JSON, commands, messages, edit strings, passwords and URLs as byte counts) -and drops the daemon telemetry: no `sizes`, `scope` or `daemon`. `argv` is +JSON, commands, messages, edit strings, passwords and URLs as byte counts; +a path argument as the byte count of its normalized form, one leading `/` +stripped, so `/a` and `a` are both `<1 bytes>`; a glob or grep `pattern` by +its own bytes) and drops the daemon telemetry: no `sizes`, `scope` or +`daemon`. A refusal is logged as its code only: its message and `detail` are +never traced, so a `Removed` path stays out of the line. `argv` is the call as CLI words (`argv[0]`/`argv[1]` are the family and call), `outcome` is `"ok"` or the refusal code, `kind` and `commit`/`layer` come from the result, `client` is `cli`/`node`/`python`/`rust`, and `tag` is the diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b26e05294..00672e1c65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,9 @@ on: - ".github/workflows/ci.yml" - "!**/*.md" pull_request: - # CI runs only for main: pull requests into any other branch run nothing (owner, 2026-11-16) + # Pull requests run only for main, except the store-ext4 lane, which also + # runs for layerfs_v2_0_0 (the 2.0 release base); any other branch runs + # nothing (owner, 2026-11-16) branches: [main] paths: - "core/**" diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b409a0b928..608e0ba55a 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,7 +12,9 @@ on: - "ruff.toml" - ".github/workflows/format.yml" pull_request: - # CI runs only for main: pull requests into any other branch run nothing (owner, 2026-11-16) + # Pull requests run only for main, except the store-ext4 lane, which also + # runs for layerfs_v2_0_0 (the 2.0 release base); any other branch runs + # nothing (owner, 2026-11-16) branches: [main] paths: - "tools/**/*.py" diff --git a/.github/workflows/store-ext4.yml b/.github/workflows/store-ext4.yml new file mode 100644 index 0000000000..88cf5a7492 --- /dev/null +++ b/.github/workflows/store-ext4.yml @@ -0,0 +1,205 @@ +# Store lane: the Store suites, on a filesystem that carries SQLite's locks. +# +# A Store opens only on a directory that is private, owned by the test user and +# on a local filesystem whose locks reach every opener: ext4, xfs or btrfs on +# Linux (engine/crates/layerfs-store-service/src/kernel_guard.rs). The Store +# suites keep their Stores in TMPDIR and in $CARGO_TARGET_DIR/tmp, so both must +# be on such a filesystem, and an overlay checkout or a tmpfs /tmp is refused. +# tools/ci/test_store_ext4.sh refuses any other directory before cargo starts +# and points TMPDIR, CARGO_TARGET_DIR and CARGO_HOME at the one it is given; +# this lane only provides that directory. See +# docs/architecture/engine/store/testing.md. +# +# It adds no test selection of its own: every target it runs is already in the +# push or nightly lane chosen by tools/ci_tests.py. It runs them under the +# environment contract their guard requires. This is the one lane that also +# runs for pull requests into layerfs_v2_0_0, the 2.0 release base. +# +# Reports only, like the other lanes (AGENTS.md §4): never a required check. +# The runner's /mnt is a separate ext4 data disk while /tmp is tmpfs, so the +# prepare step asserts the filesystem: a runner image whose /mnt is not ext4 +# fails there, with its own message, instead of as a guard refusal inside a +# suite. +name: store-ext4 + +on: + push: + branches: [main] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + pull_request: + # PRs into layerfs_v2_0_0 run this lane. Unlike the faster lanes it is not + # off for that base: 2.0 releases land there and need the Store suites. + branches: [main, layerfs_v2_0_0] + paths: + - "core/**" + - "engine/**" + - "control/**" + - "api/crates/**" + - "!api/crates/layerfs-node/**" + - "Cargo.*" + - ".config/**" + - "tools/ci/**" + - ".cargo/**" + - ".github/workflows/store-ext4.yml" + - "!**/*.md" + schedule: + # After nightly.yml's 04:23 UTC run; the two lanes never share runner time. + - cron: "47 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: store-ext4-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + # CI needs no debugger: smaller objects, much shorter links. + CARGO_PROFILE_TEST_DEBUG: "0" + # Link the test binaries with mold instead of GNU ld, as ci.yml's test job + # does. Scoped to this target, so the repository's aarch64 rustflags + # (.cargo/config.toml) stay. + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C link-arg=-fuse-ld=mold" + # The harness requires this and validates it before cargo starts. /mnt is the + # runner's ext4 data disk; never /tmp, which is tmpfs there. + LAYERFS_TEST_TMPDIR: /mnt/layerfs-store-tests + +jobs: + quick: + name: store ext4 (--quick) + # The push lane: a target subset from each of the five suites (the api suite has four). + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + # /mnt belongs to root on the runner image and the tests run as the + # runner, so hand the directory over before the harness probes it. + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # No Swatinem/rust-cache here: the harness moves CARGO_HOME and + # CARGO_TARGET_DIR onto LAYERFS_TEST_TMPDIR, so the checkout-scoped cache + # the other lanes restore is never read. The cold build is the price of + # running the suites where their guard admits them. + - name: Run the Store suites (--quick) + run: tools/ci/test_store_ext4.sh --quick + + full-postgresql: + name: store ext4 (--full, PostgreSQL 17) + # The scheduled and manual lane: every Store suite, with the PostgreSQL + # cases on. A manual dispatch runs this job only, which is the full sweep. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + # A libpq connection string, not a URL (testing.md). A service + # container's published port is 127.0.0.1 on the runner. The server is + # scratch -- each PostgreSQL test creates and drops its own schema -- and + # lives only for this job. The PostgreSQL engine is always built beside + # SQLite (#726), so setting this only enables the PostgreSQL cases. + LAYERFS_TEST_POSTGRES_URL: host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=disable + # The full-mode doctor calls amcheck's verify_heapam and bt_index_check; + # the step below creates it in this database. The physical doctor's two + # extra servers (an amcheck-free database and a physical standby) are + # not available in this job; see testing.md. Set so the diagnostic test + # asserts relation coverage is Pass instead of only printing it. + LAYERFS_TEST_EXPECT_AMCHECK: "1" + services: + postgres: + # PostgreSQL 17 is the supported version for these cases: the schema + # needs 16 or newer for STORAGE PLAIN, and the suites are qualified on + # 17.x (docs/architecture/engine/store/testing.md). + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + # Trust authentication: a throwaway server, reachable from this job's + # steps only, and the tests create and drop their own schemas. + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=postgres --dbname=postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - name: Install mold + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mold + mold --version + - name: Prepare the ext4 test directory + run: | + sudo mkdir -p "$LAYERFS_TEST_TMPDIR" + sudo chown "$(id -u):$(id -g)" "$LAYERFS_TEST_TMPDIR" + fs="$(findmnt -n -o FSTYPE --target "$LAYERFS_TEST_TMPDIR")" + echo "LAYERFS_TEST_TMPDIR=$LAYERFS_TEST_TMPDIR ($fs)" + df -h "$LAYERFS_TEST_TMPDIR" + case "$fs" in + ext4|xfs|btrfs) ;; + *) + echo "::error title=unsupported filesystem::$LAYERFS_TEST_TMPDIR is on $fs; the Store guard admits only ext4, xfs or btrfs" + exit 1 + ;; + esac + # The service health check gates this job on the server answering. The + # engine never installs extensions: a full-mode audit calls amcheck's + # verify_heapam and bt_index_check and reports every relation as + # Unknown when the extension is absent, while the compaction suite + # asserts a clean full audit. A fresh cluster therefore needs these + # contrib extensions before any suite connects (T-G14). The server is + # scratch, and this is the one database every suite reaches. + - name: Create the PostgreSQL extensions + run: | + set -euo pipefail + # The runner image does not ship psql; the distro client reaches the + # PostgreSQL 17 service container. + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends postgresql-client + psql --version + psql "$LAYERFS_TEST_POSTGRES_URL" -X -v ON_ERROR_STOP=1 <<'SQL' + CREATE EXTENSION IF NOT EXISTS amcheck; + CREATE EXTENSION IF NOT EXISTS pgstattuple; + SQL + # Fail here, with a readable list, rather than as five Unknown + # relation checks inside a compaction suite. + psql "$LAYERFS_TEST_POSTGRES_URL" -X -A -t -v ON_ERROR_STOP=1 \ + -c "SELECT extname FROM pg_catalog.pg_extension WHERE extname IN ('amcheck','pgstattuple') ORDER BY extname" + # The suites own their schemas; the harness reports the mode it took. + - name: Run the Store suites (--full) + run: tools/ci/test_store_ext4.sh --full diff --git a/AGENTS.md b/AGENTS.md index 2102515e82..3ad415877e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -289,11 +289,13 @@ The workflow's §5 keeps the cache-credit failure these rules came from. - **CI reports; it never gates** (owner, 2026-10-23). `.github/workflows/ci.yml` runs fmt, guards, Clippy and the fast tests after a push or pull request, `nightly.yml` the slow and FUSE mount tests, `format.yml` the Ruff check; - [tools/ci_tests.py](tools/ci_tests.py) picks the lanes. CI runs only for - `main`: pushes to it and pull requests into it (owner, 2026-11-16); - `layerfs_v2_0_0` and every other branch run no lane, stacked pull - requests included. Nothing runs before a push and nothing is required: do not wait - for CI or treat a green run as verification. Verify the changed scope with + [tools/ci_tests.py](tools/ci_tests.py) picks the lanes. CI runs for + `main`: pushes to it and pull requests into it (owner, 2026-11-16), plus the + report-only `store-ext4` Store lane for pull requests into `main` or + `layerfs_v2_0_0`, the 2.0 release base (owner, 2026-11-16); + `layerfs_v2_0_0` and every other branch run no other lane, stacked pull + requests included. Nothing runs before a push and nothing is required: do not + wait for CI or treat a green run as verification. Verify the changed scope with the [product checks](engine/AGENTS.md#checks-and-completion) and report checks and gaps. `tools/preflight.sh` stays retired; restore no equivalent wrapper. An empty guard scan proves no implementation. diff --git a/Cargo.lock b/Cargo.lock index e4e280a07f..726ba5ff55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,6 +1002,7 @@ dependencies = [ "layerfs-telemetry", "libc", "nix", + "postgres", ] [[package]] diff --git a/api/README.md b/api/README.md index 3310540a13..db6aff2bbe 100644 --- a/api/README.md +++ b/api/README.md @@ -10,21 +10,22 @@ CLI package `@layerfs/cli` (in [`cli/`](cli)), the Python package (in the private stdio MCP server [`mcp/`](mcp) (`@layerfs/mcp`: one generated tool per call, `work` and `admin` profiles). The design is in [docs/architecture/api](../docs/architecture/api/README.md). The package is the -library's call table as twelve families of functions, 82 calls in all: +library's call table as twelve families of functions, 85 calls in all: | Family | Calls | | --- | --- | | [`home`](../docs/architecture/api/sdk/home.md) | `init`, `get` | -| [`project`](../docs/architecture/api/sdk/project.md) | `init`, `list`, `get`, `stats`, `export`, `remove` | -| [`branch`](../docs/architecture/api/sdk/branch.md) | `list`, `get`, `commits`, `get_commit`, `fork`, `merge`, `publish`, `diff` | +| [`project`](../docs/architecture/api/sdk/project.md) | `init`, `list`, `get`, `stats`, `export`, `delete`, `remove_file` | +| [`branch`](../docs/architecture/api/sdk/branch.md) | `list`, `get`, `commits`, `get_commit`, `fork`, `merge`, `publish`, `diff`, `remove` | | [`commit`](../docs/architecture/api/sdk/commit.md) | `list`, `get`, `diff` | | [`file`](../docs/architecture/api/sdk/file.md) | `read`, `stat`, `list`, `glob`, `grep`, `revision`, `blame`, `diff`, `write`, `edit`, `rm`, `mkdir`, `mv`, `cp`, `apply` | -| [`sandbox`](../docs/architecture/api/sdk/sandbox.md) | `create`, `list`, `get`, `close` | +| [`sandbox`](../docs/architecture/api/sdk/sandbox.md) | `create`, `list`, `get`, `close`, `files` | | [`workspace`](../docs/architecture/api/sdk/workspace.md) | `mount`, `exec`, `exec_cancel`, `commit`, `unmount`, `status` | -| [`machine`](../docs/architecture/api/sdk/machine.md) | `list`, `platforms`, `get`, `status`, `start`, `stop`, `create`, `remove`, `update` | -| [`store`](../docs/architecture/api/sdk/store.md) | `list`, `get`, `status`, `stats`, `create`, `connect`, `attach`, `detach`, `push`, `pull`, `remove`, `disconnect` | +| [`machine`](../docs/architecture/api/sdk/machine.md) | `list`, `platforms`, `get`, `images`, `load`, `prune`, `status`, `stats`, `space`, `files`, `start`, `stop`, `create`, `remove`, `update` | +| [`store`](../docs/architecture/api/sdk/store.md) | `list`, `get`, `status`, `stats`, `create`, `connect`, `attach`, `detach`, `push`, `pull`, `remove`, `compact`, `disconnect` | | [`backup`](../docs/architecture/api/sdk/backup.md) | `connect`, `disconnect`, `list`, `create`, `restore` | | [`logs`](../docs/architecture/api/sdk/logs.md) | `list`, `reset`, `machine`, `sandbox` | +| [`doctor`](../docs/architecture/api/sdk/doctor.md) | `fix` | `layerfs.table` lists every call with its arguments and options; it is the source for the list above, for the `layerfs` command (`layerfs help`) and for diff --git a/api/crates/layerfs-api/src/branch/mod.rs b/api/crates/layerfs-api/src/branch/mod.rs index d4e4632dd6..bc4b6b5b7c 100644 --- a/api/crates/layerfs-api/src/branch/mod.rs +++ b/api/crates/layerfs-api/src/branch/mod.rs @@ -1,4 +1,4 @@ -//! The `branch` family: 8 calls, one file each. +//! The `branch` family: 9 calls, one file each. mod commits; mod diff; mod fork; @@ -7,6 +7,7 @@ mod get_commit; mod list; mod merge; mod publish; +mod remove; use crate::calls::Calls; @@ -21,5 +22,6 @@ pub(crate) fn calls() -> Calls { merge::CALL, publish::CALL, diff::CALL, + remove::CALL, ] } diff --git a/api/crates/layerfs-api/src/branch/remove.rs b/api/crates/layerfs-api/src/branch/remove.rs new file mode 100644 index 0000000000..e9279cbd84 --- /dev/null +++ b/api/crates/layerfs-api/src/branch/remove.rs @@ -0,0 +1,28 @@ +//! `branch.remove`: soft-remove one Branch through its Store marker. +use crate::calls::table::Kind::{Name, Project}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::branch::{remove, BranchRemove}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "branch", + "remove", + &[a("project", Project), a("branch", Name)], + &[], + "soft-remove one Branch; its name stays taken until compaction", + run, +); + +/// Runs `branch.remove`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = BranchRemove { + project: text("project").unwrap_or_default(), + branch: text("branch").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/calls/mod.rs b/api/crates/layerfs-api/src/calls/mod.rs index 708235f169..9d55ee006b 100644 --- a/api/crates/layerfs-api/src/calls/mod.rs +++ b/api/crates/layerfs-api/src/calls/mod.rs @@ -2,7 +2,7 @@ //! table's shape, argument checks, folder scope, result keys and the one //! dispatcher with its call log line (`render`, #659); moved back from //! `layerfs-control` by layerfs-project#107. -//! The 82 entries in twelve families live beside it, one folder per family and +//! The 85 entries in twelve families live beside it, one folder per family and //! one file per call; each function here that needs them takes that list. mod check; mod context; @@ -18,6 +18,7 @@ pub use check::size; pub use context::Context; pub use dispatch::{call, call_scoped, home, Client}; pub use layerfs_home::json; +pub use render::REDACTED; pub use scope::Scope; pub use table::{find, table_json, Arg, Call, CallRef, Calls, Families, Family, Kind, Run}; pub use text::call_json; diff --git a/api/crates/layerfs-api/src/calls/render.rs b/api/crates/layerfs-api/src/calls/render.rs index 87d904246a..3684a0c25d 100644 --- a/api/crates/layerfs-api/src/calls/render.rs +++ b/api/crates/layerfs-api/src/calls/render.rs @@ -9,9 +9,46 @@ use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::outcome::Outcome; /// The `Text` arguments logged only as ``: exec command text, -/// Commit messages, edit strings, Store passwords and connection URLs that -/// may embed `user:pass@`. A new call with such an argument adds its name. -pub const REDACTED: [&str; 6] = ["command", "message", "old", "new", "password", "url"]; +/// Commit messages, edit strings, Store passwords, connection URLs that may +/// embed `user:pass@`, every path argument (`path`, `left_path`, +/// `right_path`, `from_path`, `to_path`, `path_prefix`) and a glob or grep +/// `pattern`. A path is counted in its normalized form ([`PATHS`]), so a +/// `project.remove_file` or `file.*` path never enters the call log (#41 +/// §4.3). A new call with such an argument adds its name. +pub const REDACTED: [&str; 13] = [ + "command", + "message", + "old", + "new", + "password", + "url", + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", +]; + +/// The arguments that name a path: they log the length of the normalized +/// path, one leading `/` stripped, so `/a` and `a` count the same and a root +/// `/` counts as 0 (#41 §4.3). A `pattern` is not a path and keeps every +/// byte it has. +const PATHS: [&str; 6] = [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", +]; + +/// A path argument's logged byte length: the §4.3 normalization (strip one +/// leading `/`) applied before counting. +fn path_len(text: &str) -> usize { + text.strip_prefix('/').unwrap_or(text).len() +} /// The most characters a logged value keeps. const CUT: usize = 200; @@ -46,6 +83,7 @@ fn word(name: &str, kind: Kind, value: &SdkValue) -> Option { (Kind::Content, SdkValue::Text(text)) => bytes(text.len()), (Kind::Content, SdkValue::Bytes(raw)) => bytes(raw.len()), (Kind::Content | Kind::Json, value) => bytes(layerfs_home::json::print(value).len()), + (Kind::Text, SdkValue::Text(text)) if PATHS.contains(&name) => bytes(path_len(text)), (Kind::Text, SdkValue::Text(text)) if REDACTED.contains(&name) => bytes(text.len()), (_, SdkValue::Text(text)) => SdkValue::Text(cut(text)), (_, SdkValue::Int(n)) => SdkValue::Text(n.to_string()), diff --git a/api/crates/layerfs-api/src/calls/scope.rs b/api/crates/layerfs-api/src/calls/scope.rs index c7c81b6b2c..bdb139bf60 100644 --- a/api/crates/layerfs-api/src/calls/scope.rs +++ b/api/crates/layerfs-api/src/calls/scope.rs @@ -151,6 +151,11 @@ impl Scope { | ("file", "revision") => { return Err(out("history")); } + // GC retires or compacts a whole entity, never one folder: + // a scope refuses these instead of widening them (§10). + ("branch", "remove") | ("project", "delete" | "remove_file") | ("store", "compact") => { + return Err(out(name)); + } ("file", _) => { bound(&args)?; let paths: &[&str] = match call { diff --git a/api/crates/layerfs-api/src/lib.rs b/api/crates/layerfs-api/src/lib.rs index feeaf15ace..26a8d5518d 100644 --- a/api/crates/layerfs-api/src/lib.rs +++ b/api/crates/layerfs-api/src/lib.rs @@ -79,7 +79,7 @@ pub use layerfs_sandbox::{ BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi, }; -/// The twelve families, in the order help lists them: 82 calls, the same +/// The twelve families, in the order help lists them: 85 calls, the same /// from the `layerfs` binary, Node and Python. pub fn families() -> Families { &[ diff --git a/api/crates/layerfs-api/src/project/remove.rs b/api/crates/layerfs-api/src/project/delete.rs similarity index 68% rename from api/crates/layerfs-api/src/project/remove.rs rename to api/crates/layerfs-api/src/project/delete.rs index c464f1a441..9afba64e3f 100644 --- a/api/crates/layerfs-api/src/project/remove.rs +++ b/api/crates/layerfs-api/src/project/delete.rs @@ -1,4 +1,4 @@ -//! `project.remove`: remove one project from its Store. +//! `project.delete`: retire one project in its Store and remove its home entry. use crate::calls::table::Kind::Project; use crate::calls::table::{a, call, Call}; use crate::calls::Context; @@ -9,16 +9,16 @@ use layerfs_home::outcome::Outcome; /// Its table entry. pub(crate) const CALL: Call = call( "project", - "remove", + "delete", &[a("project", Project)], &[], - "remove one project from its Store", + "retire one project in its Store and remove its home entry", run, ); -/// Runs `project.remove`. +/// Runs `project.delete`. fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { let project = ProjectRef::parse(args.get_text("project").unwrap_or(""))?; let (home, events) = context.parts(); - layerfs_control::project::remove(home, events, &project) + layerfs_control::project::delete(home, events, &project) } diff --git a/api/crates/layerfs-api/src/project/mod.rs b/api/crates/layerfs-api/src/project/mod.rs index 03b79a3597..caee164cb7 100644 --- a/api/crates/layerfs-api/src/project/mod.rs +++ b/api/crates/layerfs-api/src/project/mod.rs @@ -1,9 +1,10 @@ -//! The `project` family: 6 calls, one file each. +//! The `project` family: 7 calls, one file each. +mod delete; mod export; mod get; mod init; mod list; -mod remove; +mod remove_file; mod stats; use crate::calls::Calls; @@ -16,6 +17,7 @@ pub(crate) fn calls() -> Calls { get::CALL, stats::CALL, export::CALL, - remove::CALL, + delete::CALL, + remove_file::CALL, ] } diff --git a/api/crates/layerfs-api/src/project/remove_file.rs b/api/crates/layerfs-api/src/project/remove_file.rs new file mode 100644 index 0000000000..3c8d4f2176 --- /dev/null +++ b/api/crates/layerfs-api/src/project/remove_file.rs @@ -0,0 +1,29 @@ +//! `project.remove_file`: hide one absolute path from every node at or below +//! its marker frontier. +use crate::calls::table::Kind::{Project, Text}; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::outcome::Outcome; +use layerfs_store::calls::project::{remove_file, ProjectRemoveFile}; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "project", + "remove_file", + &[a("project", Project), a("path", Text)], + &[], + "hide one absolute path from every node at or below its marker frontier", + run, +); + +/// Runs `project.remove_file`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let text = |name: &str| args.get_text(name).map(str::to_owned); + let request = ProjectRemoveFile { + project: text("project").unwrap_or_default(), + path: text("path").unwrap_or_default(), + }; + let (home, events) = context.parts(); + remove_file(home, events, &request) +} diff --git a/api/crates/layerfs-api/src/store/compact.rs b/api/crates/layerfs-api/src/store/compact.rs new file mode 100644 index 0000000000..dc1969db25 --- /dev/null +++ b/api/crates/layerfs-api/src/store/compact.rs @@ -0,0 +1,28 @@ +//! `store.compact`: reclaim dead bytes from one Store. +//! +//! A sqlite Store on another machine is refused by the control function as +//! local-only (spec S6 deviation D5, `core/docs/issues/project-41/gc-42-decisions.md`); +//! this entry parses the name and hands the call over. +use crate::calls::table::Kind::Name; +use crate::calls::table::{a, call, Call}; +use crate::calls::Context; +use layerfs_bridge::control::SdkValue; +use layerfs_home::names::StoreName; +use layerfs_home::outcome::Outcome; + +/// Its table entry. +pub(crate) const CALL: Call = call( + "store", + "compact", + &[a("name", Name)], + &[], + "reclaim dead bytes from one Store", + run, +); + +/// Runs `store.compact`. +fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { + let name = StoreName::parse(args.get_text("name").unwrap_or(""))?; + let (home, events) = context.parts(); + layerfs_control::store::compact(home, events, &name) +} diff --git a/api/crates/layerfs-api/src/store/mod.rs b/api/crates/layerfs-api/src/store/mod.rs index 8c0df7b57c..2142e041ea 100644 --- a/api/crates/layerfs-api/src/store/mod.rs +++ b/api/crates/layerfs-api/src/store/mod.rs @@ -1,6 +1,7 @@ -//! The `store` family: 12 calls, one file each (H1: push and pull replace +//! The `store` family: 13 calls, one file each (H1: push and pull replace //! move; attach and detach supersede #536 C4). mod attach; +mod compact; mod connect; mod create; mod detach; @@ -29,6 +30,7 @@ pub(crate) fn calls() -> Calls { push::CALL, pull::CALL, remove::CALL, + compact::CALL, disconnect::CALL, ] } diff --git a/api/crates/layerfs-api/tests/calls_scope.rs b/api/crates/layerfs-api/tests/calls_scope.rs index 11d9b26286..28d19e6e6d 100644 --- a/api/crates/layerfs-api/tests/calls_scope.rs +++ b/api/crates/layerfs-api/tests/calls_scope.rs @@ -114,7 +114,16 @@ fn a_scope_refuses_every_call_it_does_not_cover() { let uncovered: &[(&str, &[(&str, SdkValue)])] = &[ ("store.remove", &[("name", t("shop"))]), ("store.list", &[]), - ("project.remove", &[("project", t("shop/api"))]), + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("shop/api"))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", t("/web/a"))], + ), + ("store.compact", &[("name", t("shop"))]), ( "project.init", &[("project", t("shop/x")), ("folder", t("/tmp"))], diff --git a/api/crates/layerfs-api/tests/calls_table.rs b/api/crates/layerfs-api/tests/calls_table.rs index bf1e505904..a94f20ac97 100644 --- a/api/crates/layerfs-api/tests/calls_table.rs +++ b/api/crates/layerfs-api/tests/calls_table.rs @@ -1,4 +1,4 @@ -//! The call table and its dispatcher (#533 step 2): 82 calls in twelve +//! The call table and its dispatcher (#533 step 2): 85 calls in twelve //! families, arguments checked before any effect, one test per family that //! a valid call reaches its family function and a wrong one is refused. use layerfs_api::{families, json, keys, size, Call, Kind}; @@ -51,9 +51,9 @@ fn code(name: &str, args: SdkValue) -> String { } #[test] -fn the_table_has_twelve_families_and_82_calls() { +fn the_table_has_twelve_families_and_85_calls() { let all: Vec<&Call> = layerfs_api::calls(); - assert_eq!(all.len(), 82); + assert_eq!(all.len(), 85); // each family holds only its own calls, and has at least one for family in families() { assert!(!family.calls().is_empty(), "{}", family.name); @@ -66,7 +66,7 @@ fn the_table_has_twelve_families_and_82_calls() { let mut names: Vec<_> = all.iter().map(|c| c.name()).collect(); names.sort(); names.dedup(); - assert_eq!(names.len(), 82, "call names are unique"); + assert_eq!(names.len(), 85, "call names are unique"); // an optional positional is only ever followed by optional ones (D7) for c in &all { let first_optional = c.args.iter().position(|a| a.optional); @@ -75,7 +75,7 @@ fn the_table_has_twelve_families_and_82_calls() { } } let counts: Vec = families().iter().map(|f| f.calls().len()).collect(); - assert_eq!(counts, [2, 6, 8, 3, 15, 5, 6, 15, 12, 5, 4, 1]); + assert_eq!(counts, [2, 7, 9, 3, 15, 5, 6, 15, 13, 5, 4, 1]); } #[test] @@ -98,6 +98,17 @@ fn unknown_calls_and_arguments_are_refused_before_any_effect() { .starts_with("InvalidInput")); } +#[test] +fn project_remove_is_retired() { + // spec 4: the hard delete leaves the table; a caller gets `InvalidInput` + // naming it, never a silent hard delete + assert!(layerfs_api::find("project.remove").is_none()); + assert_eq!( + code("project.remove", map(&[("project", t("shop/api"))])), + "InvalidInput: no call named `project.remove`" + ); +} + #[test] fn project_family() { assert!(code("project.get", map(&[("project", t("api"))])) @@ -553,3 +564,28 @@ fn sandbox_images_are_named_by_the_image_kind() { assert!(prune.args[1].optional); assert_eq!(prune.options[0].name, "unused"); } + +#[test] +fn the_four_gc_calls_take_only_ids() { + // T11.1: the four GC calls, each one resource identifier and no option. + use layerfs_api::Kind; + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name}"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name}"); + assert!(spec.options.is_empty(), "{name} takes no option"); + } +} diff --git a/api/crates/layerfs-api/tests/gc_calls.rs b/api/crates/layerfs-api/tests/gc_calls.rs new file mode 100644 index 0000000000..ad390c8eab --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_calls.rs @@ -0,0 +1,351 @@ +//! T11.1: the four GC calls in the API call table — their ids-only arguments, +//! typed refusals before any effect, and the result shapes of one reachable +//! refusal per call plus the three removal results and the §11 report. +use layerfs_api::Kind; +use layerfs_bridge::control::SdkValue; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-calls-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call( + home: &Path, + name: &str, + fields: &[(&str, SdkValue)], +) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) + .map_err(|refusal| (refusal.code, refusal.message)) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> (String, String) { + call(home, name, fields).expect_err("the call is refused") +} + +/// A home with Store `shop` and project `shop/api` imported from a folder. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +#[test] +fn the_four_gc_calls_take_ids_only() { + let expected: &[(&str, &[(&str, Kind)])] = &[ + ( + "branch.remove", + &[("project", Kind::Project), ("branch", Kind::Name)], + ), + ("project.delete", &[("project", Kind::Project)]), + ( + "project.remove_file", + &[("project", Kind::Project), ("path", Kind::Text)], + ), + ("store.compact", &[("name", Kind::Name)]), + ]; + for &(name, args) in expected { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); + assert_eq!(given.as_slice(), args, "{name} arguments"); + assert!(spec.args.iter().all(|arg| !arg.optional), "{name} required"); + assert!(spec.options.is_empty(), "{name} has no options"); + } +} + +#[test] +fn wrong_kinds_and_unknown_options_are_refused_before_any_effect() { + let home = tempdir("types"); + layerfs_api::home::open(&home).unwrap(); + let wrong: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("shop/api")), ("branch", SdkValue::Int(1))], + ), + ("project.delete", &[("project", SdkValue::Int(1))]), + ( + "project.remove_file", + &[("project", t("shop/api")), ("path", SdkValue::Int(1))], + ), + ("store.compact", &[("name", SdkValue::Int(1))]), + ]; + for &(name, fields) in wrong { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.starts_with(name), "{name}: {message}"); + } + let unknown: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[ + ("project", t("shop/api")), + ("branch", t("main")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "project.delete", + &[("project", t("shop/api")), ("force", SdkValue::Bool(true))], + ), + ( + "project.remove_file", + &[ + ("project", t("shop/api")), + ("path", t("/base.txt")), + ("force", SdkValue::Bool(true)), + ], + ), + ( + "store.compact", + &[("name", t("shop")), ("force", SdkValue::Bool(true))], + ), + ]; + for &(name, fields) in unknown { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!( + message.contains("unknown argument `force`"), + "{name}: {message}" + ); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn branch_remove_refuses_reserved_and_absent_branches() { + let (home, folder) = project(); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("main"))]); + assert_eq!(code, "ReservedName", "{message}"); + // nothing ran: main is still there + ok(&home, "branch.get", &[p(), ("branch", t("main"))]); + let (code, message) = refused(&home, "branch.remove", &[p(), ("branch", t("ghost"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + // a live Branch is hidden by the marker and reported as `Removed` + ok( + &home, + "branch.fork", + &[p(), ("name", t("work")), ("from_branch", t("main"))], + ); + let removed = ok(&home, "branch.remove", &[p(), ("branch", t("work"))]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the project is the caller's /, not the Store's inner name + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert_eq!(removed.get_text("branch"), Some("work")); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + let (code, message) = refused(&home, "branch.get", &[p(), ("branch", t("work"))]); + assert_eq!(code, "NoSuchBranch", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_remove_file_checks_its_path_grammar_then_reports_markers() { + let (home, folder) = project(); + for bad in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "\\", + ] { + let (code, message) = refused(&home, "project.remove_file", &[p(), ("path", t(bad))]); + assert_eq!(code, "InvalidInput", "{bad:?}: {message}"); + } + // no effect: the file the bad calls named is still readable + let read = ok( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.get_text("text"), Some("hello\n")); + + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + // the same full project name the caller passed + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("frontier_serial").is_some()); + assert!(removed.get_int("removed_ms").is_some_and(|ms| ms >= 0)); + // a repeat at the same frontier is UpToDate + let repeat = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(repeat.get_text("kind"), Some("UpToDate")); + assert_eq!( + repeat.get_int("frontier_serial"), + removed.get_int("frontier_serial") + ); + // the marker hides it from reads + let (code, message) = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(code, "Removed", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn project_delete_reports_removal_and_retires_the_entry() { + let (home, folder) = project(); + let (code, message) = refused(&home, "project.delete", &[("project", t("shop/ghost"))]); + assert_eq!(code, "NoSuchProject", "{message}"); + let removed = ok(&home, "project.delete", &[p()]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!(removed.get_text("project"), Some("shop/api")); + assert!(removed.get_int("deleted_ms").is_some_and(|ms| ms >= 0)); + assert!(removed.get_bool("home_entry_removed")); + let (code, _) = refused(&home, "project.get", &[p()]); + assert_eq!(code, "NoSuchProject"); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +/// A project name a `project.delete` retired is still held: a second +/// `project.init` is `NameTaken` with the `deleted` holder, refused before +/// the folder is scanned. +#[test] +fn project_init_after_delete_is_nametaken_deleted() { + let (home, folder) = project(); + ok(&home, "project.delete", &[p()]); + let refusal = layerfs_api::call( + Some(&home), + "project.init", + map(&[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ]), + &mut |_| {}, + ) + .unwrap_err(); + assert_eq!(refusal.code, "NameTaken", "{}", refusal.message); + let holder = refusal.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("deleted")); + assert_eq!(refusal.detail.get_text("source"), Some("store")); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn store_compact_refuses_an_unknown_store_and_reports_a_run() { + let home = tempdir("compact-none"); + layerfs_api::home::open(&home).unwrap(); + let (code, message) = refused(&home, "store.compact", &[("name", t("ghost"))]); + assert_eq!(code, "NoSuchStore", "{message}"); + std::fs::remove_dir_all(home).unwrap(); + + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + let path = report.get_text("path").expect("path"); + assert!(matches!(path, "none" | "rebuild"), "{path}"); + let generation = report.get("generation").expect("generation"); + let before = generation.get_int("before").unwrap(); + let after = generation.get_int("after").unwrap(); + if path == "none" { + assert_eq!(before, after); + } else { + assert_eq!(after, before + 1); + } + for field in [ + "dead_bytes_bound", + "dead_bytes_refined", + "logical_discarded", + "backend_reusable", + "physical_shrink", + "peak_temp_bytes", + "estimate", + "zero_reasons", + "kept_waste", + "residue", + ] { + assert!(report.get(field).is_some(), "missing {field}"); + } + let kept = report.get("kept_waste").expect("kept_waste"); + for field in [ + "dead_in_retained_packs_bytes", + "delta_bases", + "inode_blocks_bytes", + ] { + assert!(kept.get(field).is_some(), "missing kept_waste.{field}"); + } + let residue = report.get("residue").expect("residue"); + assert!(residue.get("bytes").is_some()); + assert!(residue.get("artifacts").is_some()); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +#[test] +fn a_bare_project_name_is_refused_by_every_project_call() { + let home = tempdir("bare-names"); + layerfs_api::home::open(&home).unwrap(); + let bare: &[(&str, &[(&str, SdkValue)])] = &[ + ( + "branch.remove", + &[("project", t("api")), ("branch", t("main"))], + ), + ("project.delete", &[("project", t("api"))]), + ( + "project.remove_file", + &[("project", t("api")), ("path", t("/base.txt"))], + ), + ]; + for &(name, fields) in bare { + let (code, message) = refused(&home, name, fields); + assert_eq!(code, "InvalidInput", "{name}: {message}"); + assert!(message.contains("with its Store"), "{name}: {message}"); + } + // store.compact takes a Store name; a project path is not one + let (code, message) = refused(&home, "store.compact", &[("name", t("shop/api"))]); + assert_eq!(code, "InvalidInput", "{message}"); + assert!(message.contains("store name"), "{message}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/gc_trace.rs b/api/crates/layerfs-api/tests/gc_trace.rs new file mode 100644 index 0000000000..a3cb2144c2 --- /dev/null +++ b/api/crates/layerfs-api/tests/gc_trace.rs @@ -0,0 +1,309 @@ +//! T-G2: the call log redacts every path argument (#41 §4.3). A +//! `project.remove_file` or a hidden `file.read`/`stat`/`list`/`glob` path is +//! logged as `` of its normalized form (one leading `/` stripped), +//! the `Removed` refusal still echoes the request path, the salted trace key +//! changes with the salt, and a `store.compact` report's artifact names never +//! reach the log. +use layerfs_api::calls::REDACTED; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::path::{path_key, trace_key}; +use layerfs_history::LayerStackId; +use layerfs_home::home::paths; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard}; + +static ENV: Mutex<()> = Mutex::new(()); +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// The tests share the process environment, so each one holds this lock. +fn serial() -> MutexGuard<'static, ()> { + let guard = ENV.lock().unwrap_or_else(|poison| poison.into_inner()); + std::env::remove_var("LAYERFS_LOGS"); + std::env::remove_var("LAYERFS_LOGS_TAG"); + guard +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-gc-trace-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +fn call(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> Result { + layerfs_api::call(Some(home), name, map(fields), &mut |_| {}) +} + +fn ok(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, fields).unwrap_or_else(|refusal| panic!("{name}: {refusal:?}")) +} + +fn refused(home: &Path, name: &str, fields: &[(&str, SdkValue)]) -> SdkRefusal { + call(home, name, fields).expect_err("the call is refused") +} + +fn p() -> (&'static str, SdkValue) { + ("project", t("shop/api")) +} + +/// A home with Store `shop` and project `shop/api` imported from a folder +/// holding `base.txt`. +fn project() -> (PathBuf, PathBuf) { + let home = tempdir("home"); + layerfs_api::home::open(&home).unwrap(); + let folder = tempdir("folder"); + std::fs::write(folder.join("base.txt"), "hello\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + (home, folder) +} + +/// Every line of the call log, oldest first. +fn lines(home: &Path) -> Vec { + match std::fs::read_to_string(paths::calls_file(home)) { + Ok(text) => text + .lines() + .map(|line| layerfs_api::json::parse(line).unwrap()) + .collect(), + Err(_) => Vec::new(), + } +} + +fn last(home: &Path) -> SdkValue { + lines(home).pop().expect("a line") +} + +fn log_text(home: &Path) -> String { + std::fs::read_to_string(paths::calls_file(home)).unwrap_or_default() +} + +fn argv(line: &SdkValue) -> Vec { + match line.get("argv") { + Some(SdkValue::List(words)) => words + .iter() + .map(|word| match word { + SdkValue::Text(text) => text.clone(), + other => panic!("argv word {other:?}"), + }) + .collect(), + other => panic!("no argv: {other:?}"), + } +} + +fn bytes(n: usize) -> String { + format!("<{n} bytes>") +} + +/// The one path a `Removed` refusal details. +fn removed_path(refusal: &SdkRefusal) -> &str { + let Some(SdkValue::List(paths)) = refusal.detail.get("paths") else { + panic!("no paths detail: {:?}", refusal.detail); + }; + paths + .first() + .and_then(|path| path.get_text("path")) + .expect("one path") +} + +#[test] +fn path_arguments_are_byte_counts_in_the_call_log() { + let _env = serial(); + let (home, folder) = project(); + let removed = ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!( + argv(&last(&home)), + [ + "project".to_owned(), + "remove-file".into(), + "shop/api".into(), + bytes("base.txt".len()), + ] + ); + + // Hidden reads, stats, listings and globs: a path is a byte count. + let read = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(read.code, "Removed"); + assert_eq!( + argv(&last(&home)), + [ + "file".to_owned(), + "read".into(), + "shop/api".into(), + "main".into(), + bytes("base.txt".len()), + ] + ); + let stat = refused( + &home, + "file.stat", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(stat.code, "Removed"); + assert_eq!(argv(&last(&home))[4], bytes("base.txt".len())); + + let listed = ok( + &home, + "file.list", + &[p(), ("branch", t("main")), ("path", t("/"))], + ); + assert_eq!(listed.get_int("removed_count"), Some(1)); + assert_eq!(argv(&last(&home))[4], bytes(0)); + + let globbed = ok( + &home, + "file.glob", + &[ + p(), + ("branch", t("main")), + ("path", t("/")), + ("pattern", t("*.txt")), + ], + ); + assert_eq!(globbed.get_int("removed_count"), Some(1)); + let words = argv(&last(&home)); + assert_eq!(words[4], bytes(0)); + assert_eq!(words[5], bytes("*.txt".len())); + + let text = log_text(&home); + assert!(!text.contains("/base.txt"), "raw path in the log:\n{text}"); + assert!(!text.contains("base.txt"), "raw path in the log:\n{text}"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn a_removed_refusal_echoes_the_path_the_log_does_not() { + let _env = serial(); + let (home, folder) = project(); + ok( + &home, + "project.remove_file", + &[p(), ("path", t("/base.txt"))], + ); + let refusal = refused( + &home, + "file.read", + &[p(), ("branch", t("main")), ("path", t("/base.txt"))], + ); + assert_eq!(refusal.code, "Removed"); + assert_eq!(removed_path(&refusal), "/base.txt"); + let line = last(&home); + assert_eq!(line.get_text("outcome"), Some("Removed")); + assert!(line.get("detail").is_none(), "a detail reached the log"); + let text = log_text(&home); + assert!(!text.contains("/base.txt")); + assert!(!text.contains("base.txt")); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} + +#[test] +fn the_redacted_set_covers_every_path_bearing_call() { + for name in [ + "path", + "left_path", + "right_path", + "from_path", + "to_path", + "path_prefix", + "pattern", + ] { + assert!(REDACTED.contains(&name), "{name} is not redacted"); + } + for (name, args) in [ + ("project.remove_file", &["path"][..]), + ("file.read", &["path"][..]), + ("file.stat", &["path"][..]), + ("file.list", &["path"][..]), + ("file.glob", &["path", "pattern"][..]), + ("file.diff", &["left_path", "right_path"][..]), + ("file.cp", &["from_path", "to_path"][..]), + ("file.mv", &["from_path", "to_path"][..]), + ("branch.diff", &["path_prefix"][..]), + ] { + let spec = layerfs_api::find(name).unwrap_or_else(|| panic!("{name} is missing")); + for arg in args { + assert!(spec.arg(arg).is_some(), "{name} has no {arg}"); + assert!(REDACTED.contains(arg), "{name}'s {arg} is not redacted"); + } + } + // text that is not a path stays visible + for name in ["project", "branch", "name", "commit", "content", "changes"] { + assert!(!REDACTED.contains(&name), "{name} is redacted"); + } +} + +#[test] +fn the_trace_key_is_salted_by_the_store_policy_salt() { + let _env = serial(); + let stack = LayerStackId::from_authority([7; 16]); + let path = b"a/b"; + let first = trace_key(&[1; 32], stack, path); + assert_eq!(first, trace_key(&[1; 32], stack, path), "stable salt"); + let second = trace_key(&[2; 32], stack, path); + assert_ne!(first, second, "a different salt changes the keyed hash"); + assert_eq!(first.len(), 32); + // the keyed hash is not the marker's plain BLAKE3(normalized path) + assert_ne!(first, path_key(path)); + // the stack is part of the key + let other = LayerStackId::from_authority([9; 16]); + assert_ne!(first, trace_key(&[1; 32], other, path)); +} + +#[test] +fn store_compact_artifact_names_never_reach_the_log() { + let _env = serial(); + let (home, folder) = project(); + let report = ok(&home, "store.compact", &[("name", t("shop"))]); + assert_eq!(report.get_text("kind"), Some("Compacted")); + assert_eq!( + argv(&last(&home)), + ["store".to_owned(), "compact".into(), "shop".into()] + ); + let text = log_text(&home); + assert!( + !text.contains(".compact"), + "an artifact name reached the log:\n{text}" + ); + let store = paths::store_file(&home, "shop"); + let name = store.file_name().unwrap().to_string_lossy().into_owned(); + assert!(!text.contains(&name), "the Store file name reached the log"); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(folder).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/home_layout_3.rs b/api/crates/layerfs-api/tests/home_layout_3.rs index 63eade8633..b9ff16fe24 100644 --- a/api/crates/layerfs-api/tests/home_layout_3.rs +++ b/api/crates/layerfs-api/tests/home_layout_3.rs @@ -87,11 +87,20 @@ fn home_with(tag: &str, extra: &[(&str, SdkValue)]) -> PathBuf { let home = tempdir(tag); home::open(&home).unwrap(); call(&home, "store.create", &[("name", t("shop"))]).unwrap(); - write_record( - &paths::project_record(&home, "shop", "app"), - &SdkValue::map([("name", t("app")), ("created", SdkValue::Int(1))]), + // a real project, so the §4.2 home↔Store intersection lists and gets it + // across the conversion + let source = tempdir(&format!("{tag}-source")); + std::fs::write(source.join("README.md"), "app\n").unwrap(); + call( + &home, + "project.init", + &[ + ("project", t("shop/app")), + ("folder", t(&source.display().to_string())), + ], ) .unwrap(); + let _ = std::fs::remove_dir_all(source); write_record( &paths::machine_record(&home, "dev"), &SdkValue::map([ diff --git a/api/crates/layerfs-api/tests/logs.rs b/api/crates/layerfs-api/tests/logs.rs index 5fdbf68dbf..cbb0f9cc44 100644 --- a/api/crates/layerfs-api/tests/logs.rs +++ b/api/crates/layerfs-api/tests/logs.rs @@ -181,21 +181,19 @@ fn argv_has_positionals_options_flags_and_the_scope_before_it_rewrites() { ("page_size", SdkValue::Int(5)), ], ); - assert_eq!( - argv(&last(&home)), - [ - "file", - "diff", - "shop/api", - "main", - "/README.md", - "main", - "/README.md", - "--allow-missing", - "--page-size", - "5" - ] - ); + let want: Vec = vec![ + "file".into(), + "diff".into(), + "shop/api".into(), + "main".into(), + bytes("README.md".len()), + "main".into(), + bytes("README.md".len()), + "--allow-missing".into(), + "--page-size".into(), + "5".into(), + ]; + assert_eq!(argv(&last(&home)), want); // optional positionals stop at the first absent one let _ = call(&home, "backup.list", &[]); assert_eq!(argv(&last(&home)), ["backup", "list"]); @@ -266,7 +264,7 @@ fn free_text_credentials_content_and_json_are_byte_counts() { "write".into(), "shop/api".into(), "dev".into(), - "/a.ts".into(), + bytes("a.ts".len()), bytes(5321), "--message".into(), bytes(5) @@ -283,16 +281,22 @@ fn free_text_credentials_content_and_json_are_byte_counts() { ], ); assert_eq!(argv(&last(&home))[5], bytes(7)); - // text is cut at 200 characters + // a path argument is logged as its byte length, never its text let long = format!("/{}", "é".repeat(300)); let _ = call( &home, "file.stat", &[p(), ("branch", t("main")), ("path", t(&long))], ); - let path = &argv(&last(&home))[4]; - assert_eq!(path.chars().count(), 201); - assert!(path.ends_with('…')); + // a path logs the length of its normalized form: one leading `/` is + // stripped, so this path counts 600 bytes, not 601 + assert_eq!(argv(&last(&home))[4], bytes(long.len() - 1)); + // text that is not redacted is cut at 200 characters + let long = "n".repeat(300); + let _ = call(&home, "store.get", &[("name", t(&long))]); + let name = &argv(&last(&home))[2]; + assert_eq!(name.chars().count(), 201); + assert!(name.ends_with('…')); // each named text argument let _ = call( &home, diff --git a/api/crates/layerfs-api/tests/project_export.rs b/api/crates/layerfs-api/tests/project_export.rs index 85ef103334..d509e39b17 100644 --- a/api/crates/layerfs-api/tests/project_export.rs +++ b/api/crates/layerfs-api/tests/project_export.rs @@ -1283,7 +1283,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { // 2. During an in-flight `project.export`: // - same-Store `file.read` and a second concurrent `project.export` succeed; // - exclusive Store operations (`store.remove`, `store.push`) are refused `InUse`; - // - `project.remove` drops the catalog stack while packs stay (`packs_kept: true`), + // - `project.delete` retires the Stack (its marker keeps packs/objects), // leaving the in-flight pinned export intact while refusing later exports. let first_dest = area.join("first"); let second_dest = area.join("second"); @@ -1336,10 +1336,14 @@ fn shared_store_lease_and_concurrent_remove_admission() { .unwrap_err(); assert_eq!(push_code, "InUse", "{push_msg}"); - // `project.remove` drops the LayerStack while keeping packs/objects; - // the already-pinned export continues without corruption. - let removed = ok(&home, "project.remove", &[("project", t("shop/app"))]); - assert_eq!(removed.get("packs_kept"), Some(&SdkValue::Bool(true))); + // `project.delete` retires the LayerStack and removes the home + // entry; the already-pinned export continues without corruption. + let removed = ok(&home, "project.delete", &[("project", t("shop/app"))]); + assert_eq!(removed.get_text("kind"), Some("Removed")); + assert_eq!( + removed.get("home_entry_removed"), + Some(&SdkValue::Bool(true)) + ); } }, ) @@ -1350,7 +1354,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { assert_same_tree(&source, &second_dest); let _ = std::fs::remove_file(&other_lease); - // After `project.remove`, a new export is refused `NoSuchProject` + // After `project.delete`, a new export is refused `NoSuchProject` let after_rm_dest = area.join("after-rm"); let (code, _) = call( &home, @@ -1364,12 +1368,13 @@ fn shared_store_lease_and_concurrent_remove_admission() { assert_eq!(code, "NoSuchProject"); assert!(!after_rm_dest.exists()); - // Re-init `shop/app` and verify Store-level `Busy` and `Unresolved` admission + // Re-init a new project (`shop/app` stays a retired name) and verify + // Store-level `Busy` and `Unresolved` admission. ok( &home, "project.init", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("folder", t(&source.display().to_string())), ], ); @@ -1394,7 +1399,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { &home, "project.export", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("destination", t(&busy_dest.display().to_string())), ], ) @@ -1419,7 +1424,7 @@ fn shared_store_lease_and_concurrent_remove_admission() { &home, "project.export", &[ - ("project", t("shop/app")), + ("project", t("shop/app2")), ("destination", t(&busy_dest.display().to_string())), ], ) @@ -1600,3 +1605,70 @@ fn export_progress_events_follow_time_not_entry_count() { let _ = std::fs::remove_dir_all(d); } } + +#[test] +fn export_omits_a_path_hidden_by_remove_file() { + let home = private_home("removed-home"); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir("removed-src"); + std::fs::write(source.join("a.txt"), "a\n").unwrap(); + std::fs::write(source.join("b.txt"), "b\n").unwrap(); + std::fs::create_dir(source.join("dir")).unwrap(); + std::fs::write(source.join("dir/c.txt"), "c\n").unwrap(); + ok( + &home, + "project.init", + &[ + ("project", t("shop/app")), + ("folder", t(&source.display().to_string())), + ], + ); + + // The marker is written through the Store engine, the route the fs calls + // use; `project.remove_file` is not on this crate's call table yet. + layerfs_store::service::with_service( + &layerfs_store::open_args::OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + ..Default::default() + }, + |service| { + let call = layerfs_bridge::control::SdkCall { + method: "project.remove_file".to_owned(), + args: SdkValue::map([("project", t("app")), ("path", t("/a.txt"))]), + }; + layerfs_store_service::methods::call(service, &call) + }, + ) + .unwrap(); + + let out_parent = tempdir("removed-out"); + let dest = out_parent.join("exported"); + let res = ok( + &home, + "project.export", + &[ + ("project", t("shop/app")), + ("destination", t(&dest.display().to_string())), + ], + ); + + // §4.3: a hidden entry is omitted from the exported folder and counts. + assert!( + !dest.join("a.txt").exists(), + "a hidden path must not be exported" + ); + assert_eq!(std::fs::read_to_string(dest.join("b.txt")).unwrap(), "b\n"); + assert_eq!( + std::fs::read_to_string(dest.join("dir/c.txt")).unwrap(), + "c\n" + ); + assert_eq!(res.get_int("files"), Some(2)); + assert_eq!(res.get_int("directories"), Some(1)); + assert_eq!(res.get_int("symlinks"), Some(0)); + + for d in [home, source, out_parent] { + let _ = std::fs::remove_dir_all(d); + } +} diff --git a/api/crates/layerfs-api/tests/project_family.rs b/api/crates/layerfs-api/tests/project_family.rs index fc4ba41bc0..69a105c8c9 100644 --- a/api/crates/layerfs-api/tests/project_family.rs +++ b/api/crates/layerfs-api/tests/project_family.rs @@ -3,7 +3,7 @@ //! it, the Store pushed to a worker machine and read there, then the project //! removed. use layerfs_api::home::{self, paths, write_record}; -use layerfs_bridge::control::SdkValue; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::net::TcpListener; use std::path::{Path, PathBuf}; @@ -147,6 +147,10 @@ fn init_files_branches_move_and_remove() { assert_eq!(items.len(), 1); ok(&home, "project.get", &[("project", t("shop/api"))]); + // A named Store the home does not record is `NoSuchStore`, the same code + // every other project call reports (spec §4.2). + let refused = call(&home, "project.list", &[("store", t("nope"))]).unwrap_err(); + assert_eq!(refused.0, "NoSuchStore", "{refused:?}"); // project.list pages with `has_more` (#611) let web = tempdir("web"); std::fs::write(web.join("index.html"), "

shop

\n").unwrap(); @@ -335,7 +339,15 @@ fn init_files_branches_move_and_remove() { ); assert_eq!(content(&read), "newer\n", "read through the worker"); - ok(&home, "project.remove", &[p()]); + // `project.delete` retires the project in the Store under `box` and + // removes both entries; its history stays until a compaction. + let deleted = ok(&home, "project.delete", &[p()]); + assert_eq!(deleted.get_text("kind"), Some("Removed")); + assert_eq!(deleted.get_text("project"), Some("shop/api")); + assert_eq!( + deleted.get("home_entry_removed"), + Some(&SdkValue::Bool(true)) + ); assert_eq!( call(&home, "project.get", &[p()]).unwrap_err().0, "NoSuchProject" @@ -389,3 +401,178 @@ fn a_private_home_may_be_passable_and_nothing_more() { std::fs::remove_dir_all(home).unwrap(); let _ = std::fs::remove_dir_all(folder); } + +fn refused(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkRefusal { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).expect_err("the call is refused") +} + +fn private_home(tag: &str) -> PathBuf { + let home = tempdir(tag); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + home +} + +/// A `project.init` name collision is the typed `NameTaken`: the home record +/// first, and the Store's own holder when the home has no record for it. +#[test] +fn project_init_name_collision_reports_holder_and_source() { + let home = private_home("name-taken"); + let folder = tempdir("name-taken-folder"); + std::fs::write(folder.join("README"), b"held\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + + let home_taken = refused( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + assert_eq!(home_taken.code, "NameTaken", "{}", home_taken.message); + let holder = home_taken.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project")); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(home_taken.detail.get_text("source"), Some("home")); + + // A home that no longer records the project meets the Store's own holder. + std::fs::remove_dir_all(paths::project_dir(&home, "shop", "api")).unwrap(); + let store_taken = refused( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + assert_eq!(store_taken.code, "NameTaken", "{}", store_taken.message); + let holder = store_taken.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project")); + assert_eq!(holder.get_text("name"), Some("api")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(store_taken.detail.get_text("source"), Some("store")); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + +/// A Layer keeps the name of the Branch it was published from after that +/// Branch is retired: provenance rendering is not a caller-facing lookup. +#[test] +fn a_retired_source_branch_still_renders_on_its_layers() { + let home = private_home("retired-source"); + let folder = tempdir("retired-source-folder"); + std::fs::write(folder.join("base.txt"), b"base\n").unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + ok( + &home, + "project.init", + &[ + ("project", t("shop/api")), + ("folder", t(&folder.display().to_string())), + ], + ); + ok( + &home, + "branch.fork", + &[ + ("project", t("shop/api")), + ("name", t("work")), + ("from_branch", t("main")), + ], + ); + ok( + &home, + "file.write", + &[ + ("project", t("shop/api")), + ("branch", t("work")), + ("path", t("/new.txt")), + ("content", t("new\n")), + ("message", t("new")), + ], + ); + ok( + &home, + "branch.publish", + &[("project", t("shop/api")), ("branch", t("work"))], + ); + + let main = ok( + &home, + "commit.list", + &[("project", t("shop/api")), ("branch", t("main"))], + ); + let SdkValue::List(layers) = main.get("items").expect("items") else { + panic!("{main:?}") + }; + let id = layers[0].get_text("id").expect("layer id").to_owned(); + assert_eq!( + layers[0].get_text("source_branch"), + Some("work"), + "{main:?}" + ); + + ok( + &home, + "branch.remove", + &[("project", t("shop/api")), ("branch", t("work"))], + ); + + // `layer.list` still renders the provenance ... + let main = ok( + &home, + "commit.list", + &[("project", t("shop/api")), ("branch", t("main"))], + ); + let SdkValue::List(layers) = main.get("items").expect("items") else { + panic!("{main:?}") + }; + assert_eq!( + layers[0].get_text("source_branch"), + Some("work"), + "layer.list: {main:?}" + ); + // ... and so does `layer.get` ... + let got = ok( + &home, + "commit.get", + &[ + ("project", t("shop/api")), + ("branch", t("main")), + ("commit", t(&id)), + ], + ); + assert_eq!( + got.get_text("source_branch"), + Some("work"), + "layer.get: {got:?}" + ); + // ... while a caller-facing lookup of the retired Branch stays refused. + let (code, message) = call( + &home, + "branch.get", + &[("project", t("shop/api")), ("branch", t("work"))], + ) + .unwrap_err(); + assert_eq!(code, "NoSuchBranch", "{message}"); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} diff --git a/api/crates/layerfs-api/tests/project_init_worker.rs b/api/crates/layerfs-api/tests/project_init_worker.rs index d0f3900c4b..07b805b32d 100644 --- a/api/crates/layerfs-api/tests/project_init_worker.rs +++ b/api/crates/layerfs-api/tests/project_init_worker.rs @@ -231,14 +231,19 @@ fn init_into_a_worker_s_store_copies_the_caller_s_folder() { let stat = store_call(&home, "file.stat", "shop/web", "/link"); assert_eq!(stat.get_text("type"), Some("symlink"), "{stat:?}"); - let again = call( - &home, + let again = layerfs_api::call( + Some(&home), "project.init", init_args("shop/web", &folder), &mut |_| {}, ) .unwrap_err(); - assert_eq!(again.0, "Exists", "{}", again.1); + assert_eq!(again.code, "NameTaken", "{}", again.message); + let holder = again.detail.get("holder").expect("holder detail"); + assert_eq!(holder.get_text("kind"), Some("project"), "{again:?}"); + assert_eq!(holder.get_text("name"), Some("web")); + assert_eq!(holder.get_text("state"), Some("live")); + assert_eq!(again.detail.get_text("source"), Some("home")); open_up(&folder); let (missing, plain) = (missing.parent().unwrap(), plain.parent().unwrap()); diff --git a/api/crates/layerfs-api/tests/project_list_intersection.rs b/api/crates/layerfs-api/tests/project_list_intersection.rs new file mode 100644 index 0000000000..8e7a431f5c --- /dev/null +++ b/api/crates/layerfs-api/tests/project_list_intersection.rs @@ -0,0 +1,144 @@ +//! project.list and project.get intersect the home's project entries with +//! the Store's filtered listing (spec §4.2): a home entry whose Store +//! project is retired (hidden) or absent is not listed, and project.get +//! answers it `NoSuchProject`; a live project still lists and gets. +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::LayerStackId; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::record::ProjectRecord; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store_service::history::deletion::delete_stack; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-project-intersection-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +fn ok(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, args).unwrap_or_else(|e| panic!("{name}: {e:?}")) +} + +/// Marks Store project `name` hidden, as the project.delete write path will +/// (T4.5b); the home entry stays, exactly the stale case §4.2 hides. +fn retire(home: &Path, store: &str, name: &str, stack: &str) { + let record = StoreRecord::read(home, store).unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: Some(name.to_owned()), + store_path: Some(paths::store_file(home, store).display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = LayerStackId::from_slice(&unhex(stack).unwrap()).unwrap(); + with_service(&open, |service| { + delete_stack(&service.store, stack, 1234).unwrap(); + Ok(SdkValue::Null) + }) + .unwrap(); +} + +#[test] +fn a_retired_or_absent_store_project_is_not_listed_or_got() { + let home = tempdir("home"); + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir("folder"); + std::fs::write(source.join("README.md"), "live\n").unwrap(); + let init = |name: &str| { + ok( + &home, + "project.init", + &[ + ("project", t(&format!("shop/{name}"))), + ("folder", t(&source.display().to_string())), + ], + ) + }; + init("live"); + init("retired"); + let retired = ProjectRecord::read(&home, "shop", "retired") + .unwrap() + .expect("the retired entry"); + let retired_stack = hex(&retired.stack().expect("the entry's manifest stack")); + retire(&home, "shop", "retired", &retired_stack); + + // A home entry whose Store project is absent: a second home's stale + // record, or a failed home-entry removal. Its stack is not in the Store. + let absent = LayerStackId::from_authority([9; 16]).to_bytes(); + write_record( + &paths::project_record(&home, "shop", "absent"), + &SdkValue::map([ + ("name", t("absent")), + ("stack", t(&hex(&absent))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + + // Only the live project lists; the retired and absent entries do not. + let list = ok(&home, "project.list", &[]); + let SdkValue::List(items) = list.get("items").unwrap() else { + panic!("{list:?}") + }; + let names: Vec<&str> = items.iter().filter_map(|i| i.get_text("name")).collect(); + assert_eq!(names, vec!["live"]); + // the named listing intersects the same way + let one = ok(&home, "project.list", &[("store", t("shop"))]); + let SdkValue::List(items) = one.get("items").unwrap() else { + panic!("{one:?}") + }; + let names: Vec<&str> = items.iter().filter_map(|i| i.get_text("name")).collect(); + assert_eq!(names, vec!["live"]); + // both stale home entries are still there; the Store, not the home, hides them + for name in ["retired", "absent"] { + assert!( + paths::project_record(&home, "shop", name).exists(), + "{name}" + ); + } + + // project.get refuses a hidden project even with its home entry present, + // and an entry the Store does not hold. + for name in ["retired", "absent"] { + let (code, message) = call( + &home, + "project.get", + &[("project", t(&format!("shop/{name}")))], + ) + .unwrap_err(); + assert_eq!(code, "NoSuchProject", "{name}: {message}"); + } + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(source); +} diff --git a/api/crates/layerfs-api/tests/project_list_worker.rs b/api/crates/layerfs-api/tests/project_list_worker.rs new file mode 100644 index 0000000000..950c068f41 --- /dev/null +++ b/api/crates/layerfs-api/tests/project_list_worker.rs @@ -0,0 +1,437 @@ +//! project.list and project.get on a worker-backed Store (T4.5c): the +//! listing is the Store's own filtered one, read in the machine's worker +//! (the same `layerfs_project::store::projects` a local Store uses), so a +//! project retired there is not listed and is `NoSuchProject` from +//! project.get; a live project still lists and gets, and a worker's refusal +//! is the call's. +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::record::ProjectRecord; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store_service::history::deletion::delete_stack; +use std::net::TcpListener; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "lfs-list-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +fn ok(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkValue { + call(home, name, args).unwrap_or_else(|e| panic!("{name}: {e:?}")) +} + +/// The whole refusal, so a test can read its `detail`. +fn refusal(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> SdkRefusal { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).expect_err("the call is refused") +} + +fn names(list: &SdkValue) -> Vec { + let SdkValue::List(items) = list.get("items").unwrap() else { + panic!("{list:?}") + }; + items + .iter() + .filter_map(|item| item.get_text("name")) + .map(str::to_owned) + .collect() +} + +/// Marks Store project `stack` retired (the `project.delete` write path, +/// T4.5b): the Store hides it, its home entry stays. +fn retire(home: &Path, store: &str, stack: &[u8]) { + let record = StoreRecord::read(home, store).unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: None, + store_path: Some(paths::store_file(home, store).display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = LayerStackId::from_slice(stack).unwrap(); + with_service(&open, |service| { + delete_stack(&service.store, stack, 1234).unwrap(); + Ok(SdkValue::Null) + }) + .unwrap(); +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + let record = home::with_field(record, "machine", t(machine)); + write_record(&record_path, &record).unwrap(); +} + +/// A home whose Store `shop` holds `live` and `retired`, the latter retired +/// in the Store, and whose machine `box` record names `listener` as its +/// worker; `shop`'s calls run on `box`. With `root`, the Store's file is +/// moved under the worker's root. +fn shop_on_box(tag: &str, listener: &TcpListener, root: Option<&Path>) -> PathBuf { + let home = tempdir(tag); + std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700)).unwrap(); + home::open(&home).unwrap(); + ok(&home, "store.create", &[("name", t("shop"))]); + let source = tempdir(&format!("{tag}-folder")); + std::fs::write(source.join("README.md"), "live\n").unwrap(); + for name in ["live", "retired"] { + ok( + &home, + "project.init", + &[ + ("project", t(&format!("shop/{name}"))), + ("folder", t(&source.display().to_string())), + ], + ); + } + let retired = ProjectRecord::read(&home, "shop", "retired") + .unwrap() + .expect("the retired entry"); + retire( + &home, + "shop", + &retired.stack().expect("the entry's manifest stack"), + ); + write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", t("box")), ("platform", t("lima"))]), + ) + .unwrap(); + let address = listener.local_addr().unwrap().to_string(); + write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", t(&address)), ("secret", t("k"))]), + ) + .unwrap(); + if let Some(root) = root { + move_store_file(&home, root, "shop"); + } + place_store(&home, "shop", "box"); + home +} + +type Seen = Arc>>; + +/// A fake worker that completes the greeting and reads the request, then +/// drops the connection without an answer: the request may have run and the +/// caller cannot know the outcome. +fn fake_lost_worker(listener: TcpListener) -> Seen { + let seen: Seen = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + if let Some(request) = read_frame(&mut stream).ok().flatten() { + requests.lock().unwrap().push(request); + } + // No answer: the caller's request may have run. + } + }); + seen +} + +/// A fake worker: answers its secret, reads each request and answers +/// `reply`'s frame; the requests it read are shared back. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Seen { + let seen: Seen = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + let _ = write_frame(&mut stream, &answer); + } + }); + seen +} + +#[test] +fn a_worker_s_filtered_listing_hides_a_retired_project() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-list", &listener, None); + let live = ProjectRecord::read(&home, "shop", "live") + .unwrap() + .expect("the live entry"); + let live_stack = hex(&live.stack().expect("the live entry's stack")); + let seen = fake_worker(listener, move |_| { + // The worker's answer envelope is `{ value }`; its value is the + // `project.list` reply. + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![SdkValue::map([ + ("name", t("live")), + ("stack_id", t(&live_stack)), + ])]), + ), + ]), + )]) + }); + + // The worker's listing names only `live`: the retired entry is hidden. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + assert_eq!( + names(&ok(&home, "project.list", &[("store", t("shop"))])), + vec!["live"] + ); + // project.get refuses the retired entry and gets the live one. + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + // Both home entries are still there; the Store, not the home, hides one. + for name in ["live", "retired"] { + assert!( + paths::project_record(&home, "shop", name).exists(), + "{name}" + ); + } + + // Every read went to the worker, as the Store listing's own frame. + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 4, "{requests:?}"); + for request in requests.iter() { + assert_eq!(request.get_text("op"), Some("project.list")); + let args = request.get("args").expect("args"); + assert_eq!(args.get_text("store"), Some("shop")); + assert!(args.get("store_record").is_some()); + } + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_refusal_is_the_calls_refusal() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-refusal", &listener, None); + let _seen = fake_worker(listener, |_| { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", t("Busy")), + ("message", t("the Store is held")), + ("detail", SdkValue::Null), + ]), + )]) + }); + + // project.get propagates the refusal, never the unfiltered record. + let (code, message) = call(&home, "project.get", &[("project", t("shop/live"))]).unwrap_err(); + assert_eq!(code, "Busy"); + assert_eq!(message, "the Store is held"); + // The named listing reports it; the all-Stores one omits the Store. + let (code, _) = call(&home, "project.list", &[("store", t("shop"))]).unwrap_err(); + assert_eq!(code, "Busy"); + let list = ok(&home, "project.list", &[]); + assert!(names(&list).is_empty(), "{list:?}"); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_worker_store_lists_only_its_live_projects() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let root = tempdir("worker-root"); + let home = shop_on_box("real-list", &listener, Some(&root)); + let guest = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), guest) + }); + + // The Store under the worker's root hides its retired project; the + // caller's home entry for it stays but is not listed. + assert_eq!(names(&ok(&home, "project.list", &[])), vec!["live"]); + let (code, message) = + call(&home, "project.get", &[("project", t("shop/retired"))]).unwrap_err(); + assert_eq!(code, "NoSuchProject", "{message}"); + let live = ok(&home, "project.get", &[("project", t("shop/live"))]); + assert_eq!(live.get_text("project"), Some("shop/live")); + assert!(paths::project_record(&home, "shop", "retired").exists()); + let _ = std::fs::remove_dir_all(home); + let _ = std::fs::remove_dir_all(root); +} + +/// F43R3-1: a stale worker's lost listing answer is `Unknown`, and the call +/// names the Store it listed: `{entity: {kind, name}, phase, op}` (spec +/// §10 A6), not a null detail. +#[test] +fn a_lost_worker_answer_names_the_store_in_the_unknown_detail() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-lost", &listener, None); + let seen = fake_lost_worker(listener); + + for (method, args) in [ + ("project.get", vec![("project", t("shop/live"))]), + ("project.list", vec![("store", t("shop"))]), + ] { + let refused = refusal(&home, method, &args); + assert_eq!(refused.code, "Unknown", "{method}: {refused:?}"); + let entity = refused.detail.get("entity").expect("entity detail"); + assert_eq!( + entity.get_text("kind"), + Some("store"), + "{method}: {entity:?}" + ); + assert_eq!( + entity.get_text("name"), + Some("shop"), + "{method}: {entity:?}" + ); + assert!( + refused.detail.get("phase").expect("phase") == &SdkValue::Null, + "{method}: a lost listing has no phase" + ); + assert!( + refused.detail.get("op").expect("op") == &SdkValue::Null, + "{method}: a listing holds no durable operation" + ); + } + + // The all-Stores listing omits a Store it cannot list. + let list = ok(&home, "project.list", &[]); + assert!(names(&list).is_empty(), "{list:?}"); + + // Every read went to the worker as the Store listing's own frame. + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 3, "{requests:?}"); + for request in requests.iter() { + assert_eq!(request.get_text("op"), Some("project.list")); + } + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// F43R4-1: a worker that refuses with the bare empty-map `Unknown` detail (a +/// Store listing that could not be read) still names the Store in the +/// `Unknown` detail, exactly as a lost answer does. +#[test] +fn a_worker_unknown_with_the_empty_map_detail_names_the_store() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let home = shop_on_box("fake-empty-unknown", &listener, None); + let _seen = fake_worker(listener, |_| { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", t("Unknown")), + ("message", t("the Store's listing did not answer")), + ("detail", SdkValue::map::<0>([])), + ]), + )]) + }); + + for (method, args) in [ + ("project.get", vec![("project", t("shop/live"))]), + ("project.list", vec![("store", t("shop"))]), + ] { + let refused = refusal(&home, method, &args); + assert_eq!(refused.code, "Unknown", "{method}: {refused:?}"); + let entity = refused.detail.get("entity").expect("entity detail"); + assert_eq!( + entity.get_text("kind"), + Some("store"), + "{method}: {entity:?}" + ); + assert_eq!( + entity.get_text("name"), + Some("shop"), + "{method}: {entity:?}" + ); + assert!( + refused.detail.get("phase").expect("phase") == &SdkValue::Null, + "{method}: a listing has no phase" + ); + assert!( + refused.detail.get("op").expect("op") == &SdkValue::Null, + "{method}: a listing holds no durable operation" + ); + } + + // The all-Stores listing omits a Store it cannot list. + let list = ok(&home, "project.list", &[]); + assert!(names(&list).is_empty(), "{list:?}"); + let _ = std::fs::remove_dir_all(home); +} diff --git a/api/crates/layerfs-api/tests/project_stats.rs b/api/crates/layerfs-api/tests/project_stats.rs index bbaf0da941..53ccaab49d 100644 --- a/api/crates/layerfs-api/tests/project_stats.rs +++ b/api/crates/layerfs-api/tests/project_stats.rs @@ -202,3 +202,51 @@ fn budget_exhaustion_returns_partial_counts_with_a_reason() { std::fs::remove_dir_all(home).unwrap(); std::fs::remove_dir_all(source).unwrap(); } + +#[test] +fn hidden_paths_are_not_counted() { + let home = folder("hidden-home"); + call(&home, "store.create", &[("name", t("trees"))]); + let source = folder("hidden-source"); + std::fs::write(source.join("keep"), "aa").unwrap(); + std::fs::write(source.join("hide"), "bbb").unwrap(); + std::fs::create_dir(source.join("sub")).unwrap(); + std::fs::write(source.join("sub/inside"), "cccc").unwrap(); + call( + &home, + "project.init", + &[ + ("project", t("trees/p")), + ("folder", t(source.to_str().unwrap())), + ], + ); + let before = call(&home, "project.stats", &[("project", t("trees/p"))]); + assert_eq!(before.get_int("files"), Some(3), "{before:?}"); + assert_eq!(before.get_int("directories"), Some(1), "{before:?}"); + let logical = before.get_int("logical").unwrap(); + call( + &home, + "project.remove_file", + &[("project", t("trees/p")), ("path", t("/hide"))], + ); + let after = call(&home, "project.stats", &[("project", t("trees/p"))]); + assert_eq!(after.get_int("files"), Some(2), "{after:?}"); + assert_eq!(after.get_int("directories"), Some(1), "{after:?}"); + assert_eq!( + after.get_int("logical"), + Some(logical), + "the pinned publication's logical size is unchanged" + ); + // A removed directory hides its whole subtree. + call( + &home, + "project.remove_file", + &[("project", t("trees/p")), ("path", t("/sub"))], + ); + let nested = call(&home, "project.stats", &[("project", t("trees/p"))]); + assert_eq!(nested.get_int("files"), Some(1), "{nested:?}"); + assert_eq!(nested.get_int("directories"), Some(0), "{nested:?}"); + layerfs_api::close_idle(std::time::Duration::ZERO); + std::fs::remove_dir_all(home).unwrap(); + std::fs::remove_dir_all(source).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/scope_references.rs b/api/crates/layerfs-api/tests/scope_references.rs index 86aa60c205..02ba058dd6 100644 --- a/api/crates/layerfs-api/tests/scope_references.rs +++ b/api/crates/layerfs-api/tests/scope_references.rs @@ -193,9 +193,15 @@ fn fork_refuses_every_reference_outside_the_namespace_before_any_effect() { let mut given = vec![p()]; given.extend(args.iter().cloned()); let refused = call(&home, Some(&scoped), "branch.fork", &given).expect_err(what); - assert_eq!(refused.0, "InvalidInput", "{what}: {refused:?}"); - // the two composed Branch names (cases 0 and 3) name no Branch and - // are refused by the Store; the others never reach it + // The two composed Branch names (cases 0 and 3) are inside the scope's + // namespace and name no Branch, so the Store refuses them as + // `NoSuchBranch`; the others are refused by the scope before dispatch. + let expected = if n == 0 || n == 3 { + "NoSuchBranch" + } else { + "InvalidInput" + }; + assert_eq!(refused.0, expected, "{what}: {refused:?}"); if n != 0 && n != 3 { assert!( refused.1.contains("out of scope: branch"), diff --git a/api/crates/layerfs-api/tests/store_family.rs b/api/crates/layerfs-api/tests/store_family.rs index 36e5100aae..88f3be4293 100644 --- a/api/crates/layerfs-api/tests/store_family.rs +++ b/api/crates/layerfs-api/tests/store_family.rs @@ -496,3 +496,43 @@ fn snapshot_checkpoints_an_unheld_leftover_wal_before_pushing_and_pulling() { let _ = std::fs::remove_dir_all(root); let _ = std::fs::remove_dir_all(folder); } + +#[test] +fn compact_of_a_remote_sqlite_store_is_a_documented_local_only_refusal() { + let home = tempdir("compact-home"); + home::open(&home).unwrap(); + write_record( + &paths::store_record(&home, "shop"), + &SdkValue::map([ + ("name", t("shop")), + ("kind", t("sqlite")), + ("binding", SdkValue::Bytes(vec![7; 16])), + ("cursor", SdkValue::Bytes(vec![8; 32])), + ("profile", t("disposable")), + ("machine", t("box")), + ]), + ) + .unwrap(); + write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", t("box")), ("platform", t("lima"))]), + ) + .unwrap(); + // No worker is recorded for `box`: a call that dialled the machine would + // be `Unavailable`. `Unsupported` proves `store.compact` never takes the + // worker route: it is the local-only S6 deviation D5 recorded in + // `core/docs/issues/project-41/gc-42-decisions.md`. + let (code, message) = call(&home, "store.compact", &[("name", t("shop"))]).unwrap_err(); + assert_eq!(code, "Unsupported", "{message}"); + assert!(message.contains("local-only"), "{message}"); + assert!(message.contains("#42 D5"), "{message}"); + assert!( + message.contains("run it on that Store's machine"), + "{message}" + ); + assert!( + !paths::store_file(&home, "shop").exists(), + "the refusal opened no Store" + ); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/api/crates/layerfs-api/tests/store_projects.rs b/api/crates/layerfs-api/tests/store_projects.rs index 8961aaf5f8..9f3875af36 100644 --- a/api/crates/layerfs-api/tests/store_projects.rs +++ b/api/crates/layerfs-api/tests/store_projects.rs @@ -7,7 +7,10 @@ use layerfs_content::{ file::FileView, filesystem::{FilesystemRead, FilesystemRootId, LogicalPath}, }; -use layerfs_history::{BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId}; +use layerfs_history::{ + error::{HolderKind, HolderState, NameSource}, + BranchId, HistoryCatalog, HistoryCatalogConfig, HistoryName, LayerStackId, +}; use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; @@ -124,7 +127,13 @@ fn an_empty_store_holds_two_projects_and_refuses_a_taken_name() { assert_ne!(shop.initialized.root, api.initialized.root); assert_eq!(shop.branch.effective_root, shop.initialized.root); let taken = add(&path, source(&directory, "again", b"x"), "shop", 30); - assert!(matches!(taken, Err(InitError::Exists)), "{taken:?}"); + let Err(InitError::NameTaken(state)) = taken else { + panic!("a held project name is NameTaken: {taken:?}"); + }; + assert_eq!(state.holder.kind, HolderKind::Project); + assert_eq!(state.holder.name, "shop"); + assert_eq!(state.holder.state, HolderState::Live); + assert_eq!(state.source, NameSource::Store); let handles = SqliteHandles::open_writable(store(&path), BINDING, CURSOR).unwrap(); let names: Vec<_> = projects(&handles.history) diff --git a/api/crates/layerfs-api/tests/store_stats.rs b/api/crates/layerfs-api/tests/store_stats.rs index bd78e37bc6..dfca7542ff 100644 --- a/api/crates/layerfs-api/tests/store_stats.rs +++ b/api/crates/layerfs-api/tests/store_stats.rs @@ -26,28 +26,31 @@ fn call(home: &Path, method: &str, fields: &[(&str, SdkValue)]) -> SdkValue { layerfs_api::call(Some(home), method, args, &mut |_| {}) .unwrap_or_else(|r| panic!("{method}: {r:?}")) } -fn check(home: &Path, kind: &str) { +fn check(home: &Path, kind: &str, test: &str) { let stats = || call(home, "store.stats", &[("name", t("stats"))]); let empty = stats(); assert_eq!(empty.get_text("kind"), Some(kind)); - assert_eq!(empty.get_int("history_logical"), Some(0)); + // a previous project's bytes stay after its soft delete, so later runs + // measure from the Store's current history + let base = empty.get_int("history_logical").unwrap(); assert_eq!( empty.get("free") == Some(&SdkValue::Null), kind == "postgresql" ); assert!(empty.get("dedupe").is_none()); - let source = folder(&format!("input-{kind}")); + let name = format!("stats/{test}"); + let source = folder(&format!("input-{kind}-{test}")); std::fs::write(source.join("file"), "hello").unwrap(); call( home, "project.init", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("folder", t(source.to_str().unwrap())), ], ); let initial = stats(); - assert_eq!(initial.get_int("history_logical"), Some(5)); + assert_eq!(initial.get_int("history_logical"), Some(base + 5)); assert!(initial.get_int("canonical").unwrap() > 0); assert!(initial.get_int("stored").unwrap() > 0); assert!(initial.get_int("file").unwrap() > 0); @@ -55,7 +58,7 @@ fn check(home: &Path, kind: &str) { home, "branch.fork", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("name", t("dev")), ("from_branch", t("main")), ], @@ -64,7 +67,7 @@ fn check(home: &Path, kind: &str) { home, "file.write", &[ - ("project", t("stats/p")), + ("project", t(&name)), ("branch", t("dev")), ("path", t("/file")), ("content", t("abcdef")), @@ -73,30 +76,31 @@ fn check(home: &Path, kind: &str) { call( home, "branch.publish", - &[("project", t("stats/p")), ("branch", t("dev"))], + &[("project", t(&name)), ("branch", t("dev"))], ); let published = stats(); assert_eq!( published.get_int("history_logical"), - Some(17), + Some(base + 17), "Commit and Layer count their snapshots" ); - if kind == "postgresql" { - let refused = layerfs_api::call( - Some(home), - "project.remove", - SdkValue::map([("project", t("stats/p"))]), - &mut |_| {}, - ) - .unwrap_err(); - assert_eq!(refused.code, "Unsupported"); - } else { - call(home, "project.remove", &[("project", t("stats/p"))]); - let removed = stats(); - assert_eq!(removed.get_int("history_logical"), Some(0)); - assert_eq!(removed.get("canonical"), published.get("canonical")); - assert_eq!(removed.get("stored"), published.get("stored")); - } + // `project.delete` retires the Stack and removes the home entry; the + // marker is the commit point, so the history bytes stay until a + // `store.compact` collects them and the counters do not drop. + call(home, "project.delete", &[("project", t(&name))]); + let removed = stats(); + assert_eq!(removed.get_int("history_logical"), Some(base + 17)); + assert_eq!(removed.get("canonical"), published.get("canonical")); + assert_eq!(removed.get("stored"), published.get("stored")); + let gone = layerfs_api::call( + Some(home), + "project.get", + SdkValue::map([("project", t(&name))]), + &mut |_| {}, + ) + .unwrap_err(); + assert_eq!(gone.code, "NoSuchProject"); + assert!(!paths::project_record(home, "stats", test).exists()); layerfs_api::close_idle(std::time::Duration::ZERO); std::fs::remove_dir_all(source).unwrap(); } @@ -105,7 +109,7 @@ fn check(home: &Path, kind: &str) { fn native_sqlite_and_worker_statistics_use_the_same_counters() { let home = folder("sqlite-home"); call(&home, "store.create", &[("name", t("stats"))]); - check(&home, "sqlite"); + check(&home, "sqlite", "p"); let root = folder("worker-root"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap().to_string(); @@ -126,7 +130,7 @@ fn native_sqlite_and_worker_statistics_use_the_same_counters() { "store.push", &[("name", t("stats")), ("machine", t("box"))], ); - check(&home, "sqlite"); + check(&home, "sqlite", "w"); // A stopped worker refuses rather than starting anything. layerfs_api::close_idle(std::time::Duration::ZERO); std::fs::remove_file(paths::worker_record(&home, "box")).unwrap(); @@ -158,6 +162,6 @@ fn postgresql_statistics_have_null_free_and_count_publications() { ("synchronous_commit", t("off")), ], ); - check(&home, "postgresql"); + check(&home, "postgresql", "p"); std::fs::remove_dir_all(home).unwrap(); } diff --git a/api/crates/layerfs-cli/tests/common/mod.rs b/api/crates/layerfs-cli/tests/common/mod.rs index dec059a6ef..bd8de97c34 100755 --- a/api/crates/layerfs-cli/tests/common/mod.rs +++ b/api/crates/layerfs-cli/tests/common/mod.rs @@ -22,7 +22,10 @@ const DEADLINE: Duration = Duration::from_secs(60); /// A fresh private folder, to be used as a home or a scratch directory. pub fn tempdir(tag: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!("lfs-cli-{tag}-{}", std::process::id())); + let base = std::env::var_os("LAYERFS_TEST_TMPDIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + let dir = base.join(format!("lfs-cli-{tag}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); diff --git a/api/crates/layerfs-cli/tests/gc.rs b/api/crates/layerfs-cli/tests/gc.rs new file mode 100644 index 0000000000..0b2096f0d6 --- /dev/null +++ b/api/crates/layerfs-cli/tests/gc.rs @@ -0,0 +1,180 @@ +//! T11.2: the four GC calls on the shell. Each has one command, the call +//! that it names and the arguments the table gives it: a wrong arity or an +//! option the call does not take is exit 2 before any call, a well-formed +//! command line reaches the library, and the help is the table's. +mod common; +use common::{layerfs, run, tempdir, text}; +use std::path::Path; + +fn run_code(home: &Path, words: &[&str]) -> (Option, String, String) { + let out = run(&mut layerfs(home, words)); + (out.status.code(), text(&out.stdout), text(&out.stderr)) +} + +/// The help line of each call, as the table renders it: remove-file is the +/// shell spelling of remove_file. +const CALLS: &[(&[&str], &str, &str)] = &[ + ( + &["branch", "remove"], + "layerfs branch remove ", + "soft-remove one Branch", + ), + ( + &["project", "delete"], + "layerfs project delete ", + "retire one project", + ), + ( + &["project", "remove-file"], + "layerfs project remove-file ", + "hide one absolute path", + ), + ( + &["store", "compact"], + "layerfs store compact ", + "reclaim dead bytes", + ), +]; + +#[test] +fn project_remove_is_a_usage_error() { + let home = tempdir("gc-remove"); + let (code, out, err) = run_code(&home, &["project", "remove", "shop/api"]); + assert_eq!(code, Some(2), "{out} {err}"); + assert!(err.contains("no call `project remove`"), "{err}"); + assert!(out.is_empty(), "{out}"); + assert!(!home.join("store").exists(), "no call reached a home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn help_lists_each_gc_call_with_its_usage() { + let home = tempdir("gc-help"); + let (code, out, err) = run_code(&home, &["help"]); + assert_eq!(code, Some(0), "{err}"); + assert!(err.is_empty(), "{err}"); + let line = |family: &str| { + out.lines() + .find(|line| line.split_whitespace().next() == Some(family)) + .unwrap_or_else(|| panic!("no {family} line in {out}")) + }; + assert!(line("branch").contains("remove"), "{}", line("branch")); + assert!( + line("project").contains("delete") && line("project").contains("remove-file"), + "{}", + line("project") + ); + assert!(line("store").contains("compact"), "{}", line("store")); + + for (words, usage, help) in CALLS { + let topic: Vec<&str> = std::iter::once("help") + .chain(words.iter().copied()) + .collect(); + let (code, out, err) = run_code(&home, &topic); + assert_eq!(code, Some(0), "{words:?}: {err}"); + assert!(err.is_empty(), "{words:?}: {err}"); + assert!(out.starts_with(usage), "{words:?}: {out}"); + assert!(out.contains(help), "{words:?}: {out}"); + // the same help after the call, through -h and --help + for flag in ["-h", "--help"] { + let with_flag: Vec<&str> = words.iter().copied().chain([flag]).collect(); + let (code, more, err) = run_code(&home, &with_flag); + assert_eq!(code, Some(0), "{with_flag:?}: {err}"); + assert_eq!(more, out, "{with_flag:?}"); + } + } + assert!(!home.join("store").exists(), "help made no home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn wrong_arity_and_options_are_usage_errors() { + let home = tempdir("gc-usage"); + let wrong: &[&[&str]] = &[ + &["branch", "remove"], + &["branch", "remove", "shop/api"], + &["branch", "remove", "shop/api", "work", "extra"], + &["branch", "remove", "shop/api", "work", "--force"], + &["project", "delete"], + &["project", "delete", "shop/api", "extra"], + &["project", "delete", "shop/api", "--force"], + &["project", "remove-file", "shop/api"], + &["project", "remove-file", "shop/api", "/a", "extra"], + &["project", "remove-file", "shop/api", "/a", "--force"], + &["store", "compact"], + &["store", "compact", "shop", "extra"], + &["store", "compact", "shop", "--force"], + ]; + for words in wrong { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(2), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("layerfs: "), "{words:?}: {err}"); + } + // no positional was called: a usage error makes no home + assert!(!home.join("store").exists(), "a usage error made a home"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn well_formed_lines_reach_the_library() { + let home = tempdir("gc-reach"); + let well_formed: &[&[&str]] = &[ + &["branch", "remove", "shop/api", "work"], + &["project", "delete", "shop/api"], + &["project", "remove-file", "shop/api", "/a"], + &["store", "compact", "shop"], + ]; + for words in well_formed { + let (code, out, err) = run_code(&home, words); + // no Store in the home: the library refused after reading the line + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!(err.contains("\"code\":\"NoSuchStore\""), "{words:?}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_bare_project_is_refused_and_the_four_are_out_of_scope() { + let home = tempdir("gc-names"); + // the library, not the parser, refuses a project without its Store + for words in [ + &["branch", "remove", "api", "main"][..], + &["project", "delete", "api"], + &["project", "remove-file", "api", "/a"], + ] { + let (code, out, err) = run_code(&home, words); + assert_eq!(code, Some(125), "{words:?}: {err}"); + assert!(out.is_empty(), "{words:?}: {out}"); + assert!(err.starts_with("{"), "{words:?}: {err}"); + assert!( + err.contains("\"code\":\"InvalidInput\""), + "{words:?}: {err}" + ); + assert!(err.contains("with its Store"), "{words:?}: {err}"); + } + + // a folder scope guards one folder and one Branch: none of the four is a + // folder operation, so every one is refused, never widened + let scoped: &[(&[&str], &str)] = &[ + (&["branch", "remove", "shop/api", "main"], "branch.remove"), + (&["project", "delete", "shop/api"], "project.delete"), + ( + &["project", "remove-file", "shop/api", "/web/a"], + "project.remove_file", + ), + (&["store", "compact", "shop"], "store.compact"), + ]; + for (words, call) in scoped { + let mut command: Vec<&str> = words.to_vec(); + command.extend(["--scope", "/web", "--scope-project", "shop/api"]); + let (code, out, err) = run_code(&home, &command); + assert_eq!(code, Some(125), "{call}: {err}"); + assert!(out.is_empty(), "{call}: {out}"); + assert!(err.contains("out of scope: "), "{call}: {err}"); + assert!(err.contains(call), "{call}: {err}"); + } + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs new file mode 100644 index 0000000000..9e1012b9ab --- /dev/null +++ b/api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs @@ -0,0 +1,428 @@ +//! AC-01 (§12 crash injection, §9): `kill -9` the **real process** that +//! carries `store.compact`, not a re-execution of this test binary. +//! +//! The in-process matrix (`engine/crates/layerfs-persistence/tests/ +//! sqlite_compact_crash.rs`) stops at a failpoint inside the test process, and +//! its `real_process` arm re-executes the *test binary*, so the killed process +//! is still a test child. This test starts the built `layerfs` command with +//! `store compact`, waits through the SQLite engine's Unix-socket gate until +//! that process stands inside the run, and `SIGKILL`s it there. Per #352 +//! F-020 no daemon serves Store calls: the caller's process performs the +//! compaction, so this is the process the §9 crash describes. (It is still not +//! a long-lived `layerfs worker` serving a machine's Store.) +//! +//! Every arm reopens the Store through the CLI and asserts the §9 crash state: +//! wholly old at `G` or wholly new at `G + 1`, the exact artifact set, the +//! sidecars a killed session leaves, the `store.status` projection (phase, +//! `uncertain`, `unresolved`), the next `store.compact` settling the leftover +//! (an installed `G + 1` answers the recorded `Rebuild` report, F41-1), the +//! retained layer still readable, and a clean `doctor`. The `G` baseline +//! is the §3 `none` run before the dead pack exists, so the crash arms start +//! from a Store with exactly one wholly dead pack. +//! +//! Linux only: the arm asserts `SIGKILL` delivery through +//! `ExitStatusExt::signal`. A macOS build has no gate in the engine, so the +//! hook is inert there by construction. +#![cfg(target_os = "linux")] +mod common; + +use common::{layerfs, run, run_with_stdin, tempdir, text, LAYERFS}; +use layerfs_api::json; +use layerfs_bridge::control::SdkValue; +use std::io::Read; +use std::os::unix::net::UnixListener; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +/// The engine's gate: a run that reaches the named failpoint connects to this +/// socket and blocks there. Both variables are read by the production +/// `store.compact` path only when the gate is present. +const GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +const FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// A short socket path: the sandbox's own root is longer than `SUN_LEN`. +fn gate_socket() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + PathBuf::from(format!( + "/tmp/layerfs-g65-gate-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) +} + +/// One crash point: the failpoint, the phase `store.status` must project, +/// whether the rebuild is already installed at `G + 1`, and which staged +/// artifacts the kill leaves beside the Store. +#[derive(Clone, Copy, Debug)] +struct Arm { + failpoint: &'static str, + phase: &'static str, + installed: bool, + target: bool, + visited: bool, + /// A kill before the seal ends the process with the writer session's WAL + /// on disk (`-wal`, `-shm`); every later point has closed it. + live_sidecars: bool, +} + +/// `layerfs ` on `home`, parsing the one-line JSON result. +fn call(home: &Path, words: &[&str]) -> SdkValue { + let out = run(&mut layerfs(home, words)); + assert_eq!( + out.status.code(), + Some(0), + "{words:?}: {}", + text(&out.stderr) + ); + json::parse(text(&out.stdout).trim()).unwrap_or_else(|error| { + panic!("{words:?}: {error:?}: {}", text(&out.stdout)); + }) +} + +/// `store.compact s`, the call whose process the crash arms kill. +fn compact(home: &Path) -> SdkValue { + call(home, &["store", "compact", "s"]) +} + +fn generations(report: &SdkValue) -> (u64, u64) { + let generation = report.get("generation").expect("the report's generation"); + let before = generation.get_int("before").expect("generation.before"); + let after = generation.get_int("after").expect("generation.after"); + (before as u64, after as u64) +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +/// Kills `store.compact` for real: waits until the process is stopped inside +/// the run at `failpoint`, then `SIGKILL`s it. The gate keeps the wait from +/// being a sleep, and a process that ends early (a refusal) fails the arm with +/// its own stderr. +fn crash_and_kill(home: &Path, failpoint: &'static str) { + let gate = gate_socket(); + let _ = std::fs::remove_file(&gate); + let listener = UnixListener::bind(&gate).expect("the crash gate socket"); + listener + .set_nonblocking(true) + .expect("a nonblocking crash gate"); + let mut child = Command::new(LAYERFS) + .args(["store", "compact", "s"]) + .env("LAYERFS_HOME", home) + .env(GATE_ENV, &gate) + .env(FAILPOINT_ENV, failpoint) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("the real store.compact process"); + + let deadline = Instant::now() + Duration::from_secs(60); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait().expect("the child's status") { + let mut stderr = String::new(); + if let Some(mut pipe) = child.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!("store.compact ended before the gate ({failpoint}): {status}: {stderr}"); + } + assert!( + Instant::now() < deadline, + "store.compact never reached the gate ({failpoint})" + ); + std::thread::yield_now(); + } + Err(error) => panic!("crash gate accept: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(60))) + .expect("a bounded gate read"); + let mut ready = [0u8; 1]; + let mut reader = &stream; + reader + .read_exact(&mut ready) + .expect("the child reached the crash gate"); + assert_eq!(ready[0], b'R', "the child stopped at the crash gate"); + + // The real §9 crash: SIGKILL the process that performs the compaction. + child.kill().expect("SIGKILL the store.compact process"); + let status = child.wait().expect("the killed process exits"); + assert_eq!( + status.signal(), + Some(9), + "the store.compact process was SIGKILLed: {status:?}" + ); + drop(stream); + let _ = std::fs::remove_file(&gate); +} + +/// The fixture: a Store, one project, a Branch whose commit writes a large +/// blob, and then a `§3 none` baseline that names `G` without moving it. +fn fixture(tag: &str) -> (PathBuf, PathBuf, u64) { + let home = tempdir(&format!("g65-home-{tag}")); + let source = tempdir(&format!("g65-src-{tag}")); + std::fs::write(source.join("a.txt"), b"the retained bytes\n").unwrap(); + call(&home, &["store", "create", "s"]); + call(&home, &["project", "init", "s/p", source.to_str().unwrap()]); + call(&home, &["branch", "fork", "s/p", "dev", "main"]); + // A large blob so its pack is wholly dead once `dev` is soft-removed, and + // large enough that the rebuild threshold (`dead >= 10% of total`) holds. + let big: Vec = (0..256 * 1024).map(|at| (at % 251) as u8).collect(); + let written = run_with_stdin( + &mut layerfs(&home, &["file", "write", "s/p", "dev", "/big"]), + &big, + ); + assert!(written.status.success(), "{}", text(&written.stderr)); + + // Nothing is dead yet: §3 `none`, so the generation does not move. This + // names `G`, the generation every crash arm starts from. + let baseline = compact(&home); + assert_eq!(baseline.get_text("path"), Some("none"), "{baseline:?}"); + let (before, after) = generations(&baseline); + assert_eq!(before, after, "a `none` run moves no generation"); + + // One wholly dead pack: the Branch that wrote `/big` is soft-removed and + // its objects become unreachable; the next run must reclaim them. + call(&home, &["branch", "remove", "s/p", "dev"]); + (home, source, before) +} + +fn exercise(arm: Arm) { + let (home, source, generation) = fixture(&arm.failpoint.to_lowercase()); + let store = home.join("store/s/store.sqlite"); + assert!(store.is_file(), "{}", store.display()); + + crash_and_kill(&home, arm.failpoint); + + // §9 crash state, read before any handle opens the Store: the exact + // artifact set beside the live file, and the sidecars the kill leaves. + assert_eq!( + with_suffix(&store, ".compact").exists(), + arm.target, + "staged Store at {}", + arm.failpoint + ); + assert_eq!( + with_suffix(&store, ".compact-visited").exists(), + arm.visited, + "visited scratch at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, ".compact-op").exists(), + "op record at {}", + arm.failpoint + ); + if arm.live_sidecars { + assert!( + with_suffix(&store, "-wal").exists(), + "the killed session's WAL at {}", + arm.failpoint + ); + assert!( + with_suffix(&store, "-shm").exists(), + "the killed session's shared memory at {}", + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + if arm.live_sidecars && suffix != "-journal" { + continue; + } + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} beside the Store at {}", + arm.failpoint + ); + assert!( + !with_suffix(&with_suffix(&store, ".compact"), suffix).exists(), + "sidecar {suffix} beside the staged Store at {}", + arm.failpoint + ); + } + + // `store.status` (§11) projects the recorded run: the phase it stood in, + // an uncertain outcome, and an unresolved state because its owner died. + let observed = call(&home, &["store", "status", "s"]); + let op = observed.get("op").expect("the op projection"); + assert!(op.get("id").is_some(), "{observed:?}"); + assert_eq!(op.get_text("phase"), Some(arm.phase), "{observed:?}"); + assert_eq!( + op.get("uncertain"), + Some(&SdkValue::Bool(true)), + "{observed:?}" + ); + assert_eq!( + observed.get_text("state"), + Some("unresolved"), + "{observed:?}" + ); + + // The next run judges the leftover: a rebuild from `G` before install, and + // the recorded `Rebuild` G -> G + 1 report after it (F41-1: an installed + // `G + 1` answers that report instead of re-deciding to `none` on the + // already rebuilt Store), and no residue either way. + let recovered = compact(&home); + assert_eq!(recovered.get_text("path"), Some("rebuild"), "{recovered:?}"); + assert_eq!( + generations(&recovered), + (generation, generation + 1), + "{recovered:?}" + ); + if arm.installed { + assert!( + recovered.get_int("dead_bytes_bound").unwrap_or(0) > 0, + "the adopted report keeps the collected bytes at {}: {recovered:?}", + arm.failpoint + ); + } + let residue = recovered.get("residue").expect("the residue"); + assert_eq!(residue.get_int("bytes"), Some(0), "{recovered:?}"); + assert!( + matches!(residue.get("artifacts"), Some(SdkValue::List(items)) if items.is_empty()), + "{recovered:?}" + ); + + // Nothing is left beside the Store, and the op record is settled. + for path in [ + with_suffix(&store, ".compact"), + with_suffix(&store, ".compact-visited"), + with_suffix(&store, ".compact-op"), + ] { + assert!( + !path.exists(), + "{} left at {}", + path.display(), + arm.failpoint + ); + } + for suffix in ["-wal", "-shm", "-journal"] { + assert!( + !with_suffix(&store, suffix).exists(), + "sidecar {suffix} left at {}", + arm.failpoint + ); + } + let settled = call(&home, &["store", "status", "s"]); + assert!( + settled.get("op").and_then(|op| op.get("id")).is_none(), + "{settled:?}" + ); + assert_ne!(settled.get_text("state"), Some("unresolved"), "{settled:?}"); + + // The retained layer still reads, and the home passes the doctor: the + // dead pack is collected while the live one is intact. + let stats = call(&home, &["project", "stats", "s/p"]); + assert!(stats.get_int("files").unwrap_or(0) >= 1, "{stats:?}"); + call(&home, &["file", "stat", "s/p", "main", "/a.txt"]); + let doctor = call(&home, &["doctor"]); + assert_eq!(doctor.get("ok"), Some(&SdkValue::Bool(true)), "{doctor:?}"); + + std::fs::remove_dir_all(&home).unwrap(); + std::fs::remove_dir_all(&source).unwrap(); +} + +macro_rules! sigkill_at { + ($name:ident, $failpoint:literal, $phase:literal, $installed:literal, $target:literal, $visited:literal, $live_sidecars:literal) => { + #[test] + fn $name() { + exercise(Arm { + failpoint: $failpoint, + phase: $phase, + installed: $installed, + target: $target, + visited: $visited, + live_sidecars: $live_sidecars, + }); + } + }; +} + +// The mark phase: the open session and admission intent at `BeforeSeal` (the +// kill leaves the writer's WAL sidecars), the sealed Store at `AfterSeal`. +sigkill_at!( + sigkill_before_seal, + "BeforeSeal", + "mark", + false, + false, + true, + true +); +sigkill_at!( + sigkill_after_seal, + "AfterSeal", + "mark", + false, + false, + true, + false +); +// The build: immediately before the fresh file's first write, and after the +// staged Store is built. +sigkill_at!( + sigkill_before_load, + "BeforeLoad", + "build", + false, + false, + true, + false +); +sigkill_at!( + sigkill_after_build, + "AfterBuild", + "build", + false, + true, + true, + false +); +// The verification and the install boundary. +sigkill_at!( + sigkill_during_verify, + "DuringVerify", + "verify", + false, + true, + true, + false +); +sigkill_at!( + sigkill_before_install, + "BeforeInstall", + "verify", + false, + true, + true, + false +); +// Installed at `G + 1`: after the rename, and partway through the cleanup +// settle (the phase §11 records for it). +sigkill_at!( + sigkill_after_install, + "AfterInstall", + "install", + true, + false, + true, + false +); +sigkill_at!( + sigkill_during_settle, + "DuringSettle", + "cleanup", + true, + false, + false, + false +); diff --git a/api/mcp/package-lock.json b/api/mcp/package-lock.json index 03c11274a5..9dd94f8566 100644 --- a/api/mcp/package-lock.json +++ b/api/mcp/package-lock.json @@ -20,9 +20,18 @@ } }, "../node": { - "name": "layerfs", - "version": "2.0.0", + "name": "@layerfs/node", + "version": "2.0.0-rc.1", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], "license": "MIT", + "os": [ + "linux" + ], "engines": { "node": ">=22" } diff --git a/api/mcp/src/profiles.js b/api/mcp/src/profiles.js index 1e26a7b019..58f4c8b591 100644 --- a/api/mcp/src/profiles.js +++ b/api/mcp/src/profiles.js @@ -6,9 +6,9 @@ // Assignment rule (each call lands in exactly one bucket): // work the families an agent works in (branch, commit, file, workspace) // plus the read-only project and Sandbox calls named below -// admin every other call an agent may make: project create/remove, -// Sandbox create/close, and the home, machine, store, backup and -// logs families +// admin every other call an agent may make: project create/delete/ +// remove_file, Branch remove, Sandbox create/close, and the home, +// machine, store, backup and logs families // ui the monitoring calls a UI polls (layerfs-project#178 D2, #164): in no // profile, so no agent is offered them // none the doctor family: like `layerfs doctor` itself, which is no @@ -25,8 +25,8 @@ const WORK_FAMILIES = new Set(["branch", "commit", "file", "workspace"]); /** Read-only calls of families that are otherwise administration. */ const WORK_CALLS = new Set(["project.list", "project.get", "project.stats", "project.export", "sandbox.list", "sandbox.get"]); -/** Administrative calls of the project and sandbox families. */ -const ADMIN_CALLS = new Set(["project.init", "project.remove", "sandbox.create", "sandbox.close"]); +/** Administrative calls outside the admin families. */ +const ADMIN_CALLS = new Set(["project.init", "project.delete", "project.remove_file", "branch.remove", "sandbox.create", "sandbox.close"]); /** Monitoring calls for a UI, which no profile lists (layerfs-project#178 D2, #164). */ const UI_CALLS = new Set(["machine.stats", "machine.space", "machine.files", "logs.machine", "logs.sandbox", "sandbox.files"]); @@ -40,7 +40,7 @@ const NO_TOOL_FAMILIES = new Set(["doctor"]); /** Why a family's calls sit where they do; one line per family, in the doc. */ const WHY = { home: "set the home up and read what this build would use with it; administration", - project: "read a project's record or its entry counts, or export one (work); importing and removing a project are administration", + project: "read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration", branch: "work on Branches: list, read, fork, merge, publish and diff", commit: "read a Branch's Commit history", file: "read and change files in a Branch; each write is one explicit Commit", @@ -62,10 +62,13 @@ const nameOf = (spec) => `${spec.family}.${spec.call}`; */ export function bucketOf(spec) { const name = nameOf(spec); + // The admin exception list is checked before the work families: a retired + // entity in a work family (`branch.remove`) is still administration (§10), + // and a monitoring call a UI polls is offered to no agent. if (UI_CALLS.has(name)) return "ui"; + if (ADMIN_CALLS.has(name)) return "admin"; if (WORK_CALLS.has(name)) return "work"; if (WORK_FAMILIES.has(spec.family)) return "work"; - if (ADMIN_CALLS.has(name)) return "admin"; if (ADMIN_FAMILIES.has(spec.family)) return "admin"; if (NO_TOOL_FAMILIES.has(spec.family)) return "none"; return null; @@ -75,7 +78,10 @@ export function bucketOf(spec) { export function whyOf(spec) { if (UI_CALLS.has(nameOf(spec))) return "a monitoring read a UI polls (a machine's stats, disk space, files or worker log, a Sandbox's daemon log or files); offered to no agent"; if (spec.family === "project" && spec.call === "init") return "import a host folder into a Store; administration"; - if (spec.family === "project" && spec.call === "remove") return "remove a project from its Store; administration"; + if (spec.family === "project" && spec.call === "delete") return "retire a project in its Store and remove its home entry; administration; a folder scope refuses it"; + if (spec.family === "project" && spec.call === "remove_file") return "hide one absolute path from every node at or below its marker frontier; administration; a folder scope refuses it"; + if (spec.family === "branch" && spec.call === "remove") return "retire one Branch in its Store; administration; a folder scope refuses it"; + if (spec.family === "store" && spec.call === "compact") return "reclaim one Store's dead bytes; administration; a folder scope refuses it"; if (spec.family === "sandbox" && spec.call === "create") return "start a Sandbox process; administration"; if (spec.family === "sandbox" && spec.call === "close") return "stop a Sandbox; administration"; if (spec.family === "project" && spec.call === "export") return "export a Branch snapshot to a local folder under a launch-time --export-root; a folder scope refuses it"; diff --git a/api/mcp/src/server.js b/api/mcp/src/server.js index 25650c447c..440c6eed82 100755 --- a/api/mcp/src/server.js +++ b/api/mcp/src/server.js @@ -36,8 +36,9 @@ const USAGE = `usage: layerfs-mcp [--profile work|admin] [--home DIR] [--help] [--version] --profile work the calls an agent works with (default) - --profile admin every call, including machines, Stores, backups and - project create/remove + --profile admin every call, including machines, Stores, backups and the + four GC calls project.delete, project.remove_file, + branch.remove and store.compact --home DIR the LayerFS home (default: LAYERFS_HOME, else ~/.layerfs) --scope-project the folder scope's project, with --scope-folder --scope-folder the folder scope; the library checks every scoped call diff --git a/api/mcp/src/tools.js b/api/mcp/src/tools.js index c8fc065b52..8f7635a9b0 100644 --- a/api/mcp/src/tools.js +++ b/api/mcp/src/tools.js @@ -31,6 +31,11 @@ const COMMON = { description: "a whole number; send a decimal string to keep one beyond ±(2^53-1) exact", }, bool: { type: "boolean" }, + image: { + type: "string", + minLength: 1, + description: "a sha256: image digest, a Docker image id, or a Docker reference such as postgres:17-bookworm", + }, size: { oneOf: [{ type: "integer" }, { type: "string", minLength: 1 }], description: "a size in bytes, or text such as \"512M\" or \"8G\"", diff --git a/api/mcp/test/profiles.test.mjs b/api/mcp/test/profiles.test.mjs index ab4b9dd08a..de0a8c02e9 100644 --- a/api/mcp/test/profiles.test.mjs +++ b/api/mcp/test/profiles.test.mjs @@ -14,10 +14,10 @@ const nameOf = (spec) => `${spec.family}.${spec.call}`; test("every call is in exactly one bucket, and admin adds the work bucket", () => { assert.deepEqual(check(table), []); - assert.equal(table.length, 82); + assert.equal(table.length, 85); const buckets = table.map(bucketOf); assert.equal(buckets.filter((bucket) => bucket === "work").length, 38); - assert.equal(buckets.filter((bucket) => bucket === "admin").length, 37); + assert.equal(buckets.filter((bucket) => bucket === "admin").length, 40); assert.ok(buckets.every((bucket) => bucket === "work" || bucket === "admin" || bucket === "ui" || bucket === "none")); // `doctor.fix`, like `layerfs doctor` itself, is no tool in either profile assert.deepEqual(table.filter((spec) => bucketOf(spec) === "none").map(nameOf), ["doctor.fix"]); @@ -32,12 +32,20 @@ test("every call is in exactly one bucket, and admin adds the work bucket", () = const work = tools(table, "work").map(nameOf); const admin = tools(table, "admin").map(nameOf); assert.equal(work.length, 38); - assert.equal(admin.length, 75); + assert.equal(admin.length, 78); for (const name of work) assert.ok(admin.includes(name), `${name} is in work but not admin`); assert.ok(!admin.some((name) => name.startsWith("doctor.")), "a doctor call is a tool"); assert.throws(() => tools(table, "root"), /unknown profile/); }); +test("the admin exception list is checked before the work families", () => { + // `branch` is a work family, but retiring one is administration (§10) + const retire = { family: "branch", call: "remove" }; + assert.equal(bucketOf(retire), "admin"); + assert.ok(!tools([...table, retire], "work").includes(retire)); + assert.ok(tools([...table, retire], "admin").includes(retire)); +}); + test("every generated tool is a valid MCP tool with the table's arguments and options", () => { const generated = toolsOf(table); assert.equal(generated.length, table.length); diff --git a/api/mcp/test/workflow.test.mjs b/api/mcp/test/workflow.test.mjs index 31525aeca2..2b8b9fb54a 100644 --- a/api/mcp/test/workflow.test.mjs +++ b/api/mcp/test/workflow.test.mjs @@ -22,10 +22,11 @@ test("a real workflow, refusals, progress, scope and the two profiles", async () const listed = await server.client.listTools(); assert.equal(listed.tools.length, calls("admin")); - // every call but the doctor family's one, which is no tool - assert.equal(listed.tools.length, table.length - 1); + // the admin profile lists work + admin only: the six monitoring calls a UI + // polls and the doctor family's one are no tool in any profile + assert.equal(listed.tools.length, table.length - 1 - 6); assert.ok(!listed.tools.some((tool) => tool.name.startsWith("doctor_"))); - assert.equal(new Set(listed.tools.map((tool) => tool.name)).size, table.length - 1); + assert.equal(new Set(listed.tools.map((tool) => tool.name)).size, calls("admin")); for (const tool of listed.tools) ToolSchema.parse(tool); const run = async (name, args, options) => value((await call(server.client, name, args, options)).result); @@ -94,6 +95,11 @@ test("a real workflow, refusals, progress, scope and the two profiles", async () server.client.callTool({ name: "not_a_tool", arguments: {} }), (error) => error.code === -32602 && /unknown tool/.test(error.message), ); + // retired in 2.0 (spec 4): the old tool name does not exist + await assert.rejects( + server.client.callTool({ name: "project_remove", arguments: { project: "shop/api" } }), + (error) => error.code === -32602 && /unknown tool/.test(error.message), + ); await assert.rejects( server.client.callTool({ name: "store_create", arguments: { name: 5 } }), (error) => error.code === -32602 && /name.*must be text/.test(error.message), @@ -124,7 +130,7 @@ test("a real workflow, refusals, progress, scope and the two profiles", async () for (const name of names("admin").filter((name) => !names("work").includes(name))) { assert.ok(!workNames.has(name), `${name} is an admin tool in the work profile`); } - for (const prefix of ["store_", "machine_", "backup_", "project_init", "project_remove", "sandbox_create", "sandbox_close"]) { + for (const prefix of ["store_", "machine_", "backup_", "project_init", "project_delete", "project_remove_file", "branch_remove", "sandbox_create", "sandbox_close"]) { assert.ok(![...workNames].some((name) => name.startsWith(prefix)), `${prefix} in the work profile`); } const scopedRead = await call(work.client, "file_read", { project: "shop/api", branch: "feat", path: "/sub/b.txt" }); diff --git a/api/node/README.md b/api/node/README.md index a269925022..8e666010bd 100644 --- a/api/node/README.md +++ b/api/node/README.md @@ -2,7 +2,7 @@ The LayerFS SDK for Node: twelve families — `home`, `project`, `branch`, `commit`, `file`, `sandbox`, `workspace`, `machine`, `store`, `backup`, `logs`, `doctor` — over one call -table of 82 calls into one native library. Each function makes one call; it +table of 85 calls into one native library. Each function makes one call; it does not retry, replay or commit on its own. Results are plain objects with snake_case keys, whole numbers are exact (`bigint` beyond `Number.MAX_SAFE_INTEGER`), bytes are `Buffer`s, and the TypeScript typings diff --git a/api/node/src/index.d.ts b/api/node/src/index.d.ts index 7de8195dee..d404dabcfa 100644 --- a/api/node/src/index.d.ts +++ b/api/node/src/index.d.ts @@ -474,8 +474,10 @@ export interface ProjectFamily { stats(project: string, branch?: string | null, commit?: string | null, options?: CallOptions): Promise; /** export a project's snapshot from its Store to a local folder */ export(project: string, destination: string, options?: ProjectExportOptions): Promise; - /** remove one project from its Store */ - remove(project: string, options?: CallOptions): Promise; + /** retire one project in its Store and remove its home entry */ + delete(project: string, options?: CallOptions): Promise; + /** hide one absolute path from every node at or below its marker frontier */ + remove_file(project: string, path: string, options?: CallOptions): Promise; } export interface BranchFamily { @@ -497,6 +499,8 @@ export interface BranchFamily { publish(project: string, branch: string, commit?: string | null, options?: BranchPublishOptions): Promise; /** files that differ between two Branches' heads */ diff(project: string, from_branch: string, to_branch: string, options?: BranchDiffOptions): Promise; + /** soft-remove one Branch; its name stays taken until compaction */ + remove(project: string, branch: string, options?: CallOptions): Promise; } export interface CommitFamily { @@ -636,6 +640,8 @@ export interface StoreFamily { pull(name: string, options?: StorePullOptions): Promise; /** delete a sqlite Store with every project in it */ remove(name: string, options?: CallOptions): Promise; + /** reclaim dead bytes from one Store */ + compact(name: string, options?: CallOptions): Promise; /** forget a PostgreSQL Store; its data stays */ disconnect(name: string, options?: CallOptions): Promise; } diff --git a/api/node/test/branch-file/inventory.mjs b/api/node/test/branch-file/inventory.mjs index 9d626237bb..90897de44c 100644 --- a/api/node/test/branch-file/inventory.mjs +++ b/api/node/test/branch-file/inventory.mjs @@ -11,8 +11,8 @@ const root = fileURLToPath(new URL("../../../../", import.meta.url)); const command = process.env.LAYERFS_CLI ?? join(root, "target/debug/layerfs"); const docs = join(root, "docs/architecture/api/sdk"); -test("the SDK pages name all 82 current calls, one `### family.call` heading each", () => { - assert.equal(table.length, 82); +test("the SDK pages name all 85 current calls, one `### family.call` heading each", () => { + assert.equal(table.length, 85); const families = [...new Set(table.map((spec) => spec.family))]; assert.equal(families.length, 12); const index = readFileSync(join(docs, "README.md"), "utf8"); diff --git a/api/node/test/calls.mjs b/api/node/test/calls.mjs index 8f478b713b..e208e041de 100644 --- a/api/node/test/calls.mjs +++ b/api/node/test/calls.mjs @@ -14,9 +14,9 @@ const temp = (tag) => { return dir; }; -test("the table: twelve families, 82 calls, each a function", () => { +test("the table: twelve families, 85 calls, each a function", () => { assert.deepEqual(families, ["home", "project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs", "doctor"]); - assert.equal(table.length, 82); + assert.equal(table.length, 85); for (const spec of table) assert.equal(typeof layerfs[spec.family][spec.call], "function", `${spec.family}.${spec.call}`); }); diff --git a/api/node/test/coverage.mjs b/api/node/test/coverage.mjs index 59bc6cc68b..6d1889bbb8 100644 --- a/api/node/test/coverage.mjs +++ b/api/node/test/coverage.mjs @@ -26,7 +26,9 @@ test("coverage.md is what the front ends' own call lists give", { skip }, () => test("a call missing from a front end, or one that differs, is a problem and changes the block", { skip }, () => { const { block } = render(view); - const nodeSpec = view.node.table[0]; + // A call with an argument to rename: the table's first entry, `home.init`, + // takes none. + const nodeSpec = view.node.table.find((spec) => spec.args.length > 0); const name = `${nodeSpec.family}.${nodeSpec.call}`; const noCli = structuredClone(view); @@ -39,7 +41,8 @@ test("a call missing from a front end, or one that differs, is a problem and cha assert.match(render(noAio).problems.join("\n"), new RegExp(`${name}: no aio coroutine`)); const otherArgument = structuredClone(view); - otherArgument.python.table[0].args[0].name = "renamed"; + otherArgument.python.table.find((spec) => `${spec.family}.${spec.call}` === name).args[0].name = + "renamed"; assert.match(render(otherArgument).problems.join("\n"), new RegExp(`${name}: the Python table differs`)); const otherFlag = structuredClone(view); diff --git a/api/python/src/layerfs/_client.pyi b/api/python/src/layerfs/_client.pyi index 307a2c2a9b..07dc1e7d60 100644 --- a/api/python/src/layerfs/_client.pyi +++ b/api/python/src/layerfs/_client.pyi @@ -55,8 +55,10 @@ class ProjectFamily: log_tag: str | None = None, ) -> Result: """export a project's snapshot from its Store to a local folder""" - def remove(self, project: str, *, log_tag: str | None = None) -> Result: - """remove one project from its Store""" + def delete(self, project: str, *, log_tag: str | None = None) -> Result: + """retire one project in its Store and remove its home entry""" + def remove_file(self, project: str, path: str, *, log_tag: str | None = None) -> Result: + """hide one absolute path from every node at or below its marker frontier""" class BranchFamily: def list( @@ -138,6 +140,8 @@ class BranchFamily: log_tag: str | None = None, ) -> Result: """files that differ between two Branches' heads""" + def remove(self, project: str, branch: str, *, log_tag: str | None = None) -> Result: + """soft-remove one Branch; its name stays taken until compaction""" class CommitFamily: def list( @@ -570,6 +574,8 @@ class StoreFamily: """bring a sqlite Store from its machine to native""" def remove(self, name: str, *, log_tag: str | None = None) -> Result: """delete a sqlite Store with every project in it""" + def compact(self, name: str, *, log_tag: str | None = None) -> Result: + """reclaim dead bytes from one Store""" def disconnect(self, name: str, *, log_tag: str | None = None) -> Result: """forget a PostgreSQL Store; its data stays""" diff --git a/api/python/src/layerfs/aio.pyi b/api/python/src/layerfs/aio.pyi index 51923671ae..2e5dbc6fd5 100644 --- a/api/python/src/layerfs/aio.pyi +++ b/api/python/src/layerfs/aio.pyi @@ -53,8 +53,10 @@ class AsyncProjectFamily: log_tag: str | None = None, ) -> Result: """export a project's snapshot from its Store to a local folder""" - async def remove(self, project: str, *, log_tag: str | None = None) -> Result: - """remove one project from its Store""" + async def delete(self, project: str, *, log_tag: str | None = None) -> Result: + """retire one project in its Store and remove its home entry""" + async def remove_file(self, project: str, path: str, *, log_tag: str | None = None) -> Result: + """hide one absolute path from every node at or below its marker frontier""" class AsyncBranchFamily: async def list( @@ -136,6 +138,8 @@ class AsyncBranchFamily: log_tag: str | None = None, ) -> Result: """files that differ between two Branches' heads""" + async def remove(self, project: str, branch: str, *, log_tag: str | None = None) -> Result: + """soft-remove one Branch; its name stays taken until compaction""" class AsyncCommitFamily: async def list( @@ -581,6 +585,8 @@ class AsyncStoreFamily: """bring a sqlite Store from its machine to native""" async def remove(self, name: str, *, log_tag: str | None = None) -> Result: """delete a sqlite Store with every project in it""" + async def compact(self, name: str, *, log_tag: str | None = None) -> Result: + """reclaim dead bytes from one Store""" async def disconnect(self, name: str, *, log_tag: str | None = None) -> Result: """forget a PostgreSQL Store; its data stays""" diff --git a/api/python/test/test_calls.py b/api/python/test/test_calls.py index a0c2009c50..a5670d1b3f 100644 --- a/api/python/test/test_calls.py +++ b/api/python/test/test_calls.py @@ -35,7 +35,7 @@ # Load the real aio module with private import stubs, without pretending a # native binding is available or exercising native calls against a mock. layerfs = None - families = ["project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs"] + families = ["project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs", "doctor"] package = types.ModuleType("_layerfs_aio_unit") package.__path__ = [] dependencies = { @@ -350,7 +350,7 @@ def test_the_table(self): layerfs.FAMILIES, ["home", "project", "branch", "commit", "file", "sandbox", "workspace", "machine", "store", "backup", "logs", "doctor"], ) - self.assertEqual(len(layerfs.TABLE), 82) + self.assertEqual(len(layerfs.TABLE), 85) for spec in layerfs.TABLE: self.assertTrue(callable(getattr(getattr(layerfs, spec["family"]), spec["call"]))) # the options the Publish change added are in the table the Python diff --git a/api/scripts/gen-coverage.mjs b/api/scripts/gen-coverage.mjs index 1d61c69129..a0cb916784 100644 --- a/api/scripts/gen-coverage.mjs +++ b/api/scripts/gen-coverage.mjs @@ -107,9 +107,12 @@ function usage(words) { /** What the command line of `spec` must be, by the table: a content or JSON positional is read from * stdin or `--from`, `on_progress` is `--progress`, the output functions are `--stream`. */ function expectedCli(spec) { + const options = spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)); + // the command frames a call with a stdout sink as events: `--events`, last + if (spec.options.some((o) => o.name === "stdout")) options.push("events"); return { positionals: spec.args.filter((a) => !piped(a)).map((a) => ({ name: a.name, optional: a.optional })), - options: spec.options.filter((o) => o.kind !== FUNCTION || o.name === "on_progress").map((o) => (o.name === "on_progress" ? "progress" : o.name)), + options, from: spec.args.some(piped), }; } diff --git a/control/crates/layerfs-control/src/home.rs b/control/crates/layerfs-control/src/home.rs index b806304e2a..78c2004c63 100644 --- a/control/crates/layerfs-control/src/home.rs +++ b/control/crates/layerfs-control/src/home.rs @@ -339,7 +339,7 @@ fn daemon_value(home: &Path, build: Build<'_>, warnings: &mut Vec) -> } #[cfg(not(target_os = "macos"))] -fn daemon_value(_home: &Path, _build: Build<'_>, _warnings: &mut Vec) -> SdkValue { +fn daemon_value(_home: &Path, _build: Build<'_>, _warnings: &mut [SdkValue]) -> SdkValue { SdkValue::Null } diff --git a/control/crates/layerfs-control/src/inventory.rs b/control/crates/layerfs-control/src/inventory.rs index d84ff2afdb..9e88a7097b 100644 --- a/control/crates/layerfs-control/src/inventory.rs +++ b/control/crates/layerfs-control/src/inventory.rs @@ -191,6 +191,37 @@ pub fn can_change_machine(home: &Path, name: &str) -> Result<()> { )) } +/// `project.delete`: refused `Busy` while a Sandbox of `project` +/// (`/`) is recorded, so a live Sandbox cannot outlive its +/// project's Store rows. The `detail` is spec §10's `{entity: {kind: +/// "project", name}, phase: null, op: null}`, and `related` lists the +/// Sandboxes as `{kind, name}`; an unreachable Sandbox is still a live user +/// and keeps holding. +pub fn can_remove_project(home: &Path, project: &str) -> Result<()> { + let sandboxes = sandboxes_of_project(home, project)?; + if sandboxes.is_empty() { + return Ok(()); + } + let message = match sandboxes.as_slice() { + [sandbox] => format!("Sandbox {sandbox} of {project} is running"), + many => format!("Sandboxes {} of {project} are running", many.join(", ")), + }; + let mut refusal = refused("Busy", message); + refusal.detail = SdkValue::map([ + ( + "entity", + SdkValue::map([ + ("kind", SdkValue::text("project")), + ("name", SdkValue::text(project)), + ]), + ), + ("phase", SdkValue::Null), + ("op", SdkValue::Null), + ("related", SdkValue::List(related("sandbox", &sandboxes))), + ]); + Err(refusal) +} + /// `store.push`, `store.pull`, `store.remove`, `store.disconnect` and /// `backup.restore`: refused `InUse` while /// another live process holds Store `store` (its lease) or a Sandbox of one @@ -216,15 +247,3 @@ pub fn store_idle(home: &Path, store: &str) -> Result<()> { held.extend(related("sandbox", &sandboxes)); Err(in_use(why.join("; "), held)) } - -/// `project.remove`: refused `InUse` while a Sandbox of `project` is -/// recorded; `detail.related` lists them as `{kind: "sandbox", name}`. -pub fn can_remove_project(home: &Path, project: &str) -> Result<()> { - let sandboxes = sandboxes_of_project(home, project)?; - let message = match sandboxes.as_slice() { - [] => return Ok(()), - [sandbox] => format!("Sandbox {sandbox} of {project} is running"), - many => format!("Sandboxes {} of {project} are running", many.join(", ")), - }; - Err(in_use(message, related("sandbox", &sandboxes))) -} diff --git a/control/crates/layerfs-control/src/project.rs b/control/crates/layerfs-control/src/project.rs index c3505c5d13..b87a0f1e53 100644 --- a/control/crates/layerfs-control/src/project.rs +++ b/control/crates/layerfs-control/src/project.rs @@ -1,13 +1,20 @@ -//! The `project` call that checks who uses a project, `project.remove` -//! (layerfs-project#105); the other `project` calls are `layerfs-store`'s -//! own. +//! The `project` call that retires a project in its Store and removes the +//! caller's home entry (`project.delete`, spec §4); the other `project` +//! calls are `layerfs-store`'s own. +use crate::inventory; use layerfs_home::events::Events; use layerfs_home::names::ProjectRef; use layerfs_home::outcome::Outcome; use std::path::Path; -/// `project.remove`: a project's history dropped from its Store, once its -/// Sandboxes are closed. -pub fn remove(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { - layerfs_store::project::family::remove(home, events, project, crate::store::RELATED) +/// The project-scope `Busy` checks `project.delete` takes from this layer. +pub const PROJECT_RELATED: layerfs_store::Related = layerfs_store::Related { + store_idle: inventory::store_idle, + can_remove_project: inventory::can_remove_project, +}; + +/// `project.delete(project)`: retires the project in its Store and removes +/// the caller home entry (spec §4). +pub fn delete(home: &Path, events: &mut Events<'_>, project: &ProjectRef) -> Outcome { + layerfs_store::project::delete::delete(home, events, project, PROJECT_RELATED) } diff --git a/control/crates/layerfs-control/src/store.rs b/control/crates/layerfs-control/src/store.rs index d67e04a111..a193ee2147 100644 --- a/control/crates/layerfs-control/src/store.rs +++ b/control/crates/layerfs-control/src/store.rs @@ -18,6 +18,12 @@ use std::path::Path; pub use layerfs_store::close_idle; +/// `store.compact`: reclaim one Store's dead bytes, answering the +/// §11 report. +pub fn compact(home: &Path, events: &mut Events<'_>, name: &StoreName) -> Outcome { + layerfs_store::compact::compact(home, events, name.as_str()) +} + /// Who uses a Store or a project, read from the Sandbox records. pub const RELATED: Related = Related { store_idle: inventory::store_idle, diff --git a/control/crates/layerfs-control/src/worker_server.rs b/control/crates/layerfs-control/src/worker_server.rs index be7bc8fe4a..6acda1ffdc 100644 --- a/control/crates/layerfs-control/src/worker_server.rs +++ b/control/crates/layerfs-control/src/worker_server.rs @@ -13,10 +13,13 @@ //! `inspect` reads this machine or one of its Sandboxes (`layerfs-sandbox` //! `inspect`: at most two at once, each on a helper thread with deadlines); //! `store.call`, `store.file`, `store.reach`, `sandbox.host`, `project.stage`, -//! `project.init`, `project.export`, `project.remove` and `doctor.fix` run -//! one operation on this machine (`ops`), +//! `project.init`, `project.export`, `project.delete`, `project.list` and +//! `doctor.fix` run one operation on this machine (`ops`), //! with `home` set to the worker's own root (the machine's data disk), -//! never a path the caller sends. Answers are +//! never a path the caller sends. There is no `store.compact` operation: +//! compaction is local-only (spec S6 deviation D5, +//! `core/docs/issues/project-41/gc-42-decisions.md`), and the caller refuses a +//! remote sqlite Store before it dials this worker. Answers are //! `{ value }` or `{ refused }`, after any `{ event }` frames. //! //! A request may carry `home_id`, the caller's home id: the root is not a diff --git a/control/crates/layerfs-control/src/worker_server/ops.rs b/control/crates/layerfs-control/src/worker_server/ops.rs index add5009397..425ded31ca 100644 --- a/control/crates/layerfs-control/src/worker_server/ops.rs +++ b/control/crates/layerfs-control/src/worker_server/ops.rs @@ -3,10 +3,17 @@ use super::Server; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::outcome::{refused, Outcome}; +use layerfs_store::project::delete::{ + admit_project, delete_project, release_project, remove_project_entry, +}; use layerfs_store::project::discovery::home_of; use layerfs_store::project::export::in_worker as export_in_worker; -use layerfs_store::project::frame::{ExportFrame, InitFrame, RemoveFrame, StageFrame}; -use layerfs_store::project::{init::in_worker, remove::remove_project, staged}; +use layerfs_store::project::frame::{ + DeleteAdmitFrame, DeleteEntryFrame, DeleteFrame, DeleteReleaseFrame, ExportFrame, InitFrame, + ListingFrame, StageFrame, +}; +use layerfs_store::project::listing::list_projects; +use layerfs_store::project::{init::in_worker, staged}; use std::path::Path; /// The operations that change nothing on the machine. Only these answer @@ -14,13 +21,14 @@ use std::path::Path; /// any other operation's panic closes the connection, so its caller /// reports `Unknown` (layerfs-project#179 D7). A new read-only operation /// is added here. -const READ_ONLY: [&str; 6] = [ +const READ_ONLY: [&str; 7] = [ "worker.ping", layerfs_sandbox::inspect::OP, "machine.images", "store.reach", "doctor.sandbox", "store.diagnose", + "project.list", ]; /// Whether `op` is one of the read-only operations. @@ -155,12 +163,36 @@ pub(super) fn run( }, layerfs_sandbox::workspace::dispatch::call, ), - // drops one project from the Store under this worker's root (2.0): - // `{ store, project, store_record }` - "project.remove" => { + // retires one project in the Store under this worker root and + // removes the worker own entry (2.0, T4.5b): `{ store, stack_id, + // deleted_ms?, project?, op_id?, pending_retry?, store_record? }` + "project.delete" => { let root = home_of(Some(root))?; - remove_project(&root, &RemoveFrame::decode(&args)?) + delete_project(&root, &DeleteFrame::decode(&args)?) } + // registers the caller's project-scope admission before its sidecar + // and marker (2.0, T4.6): `{ store, stack_id, op_id, store_record? }` + "project.delete.admit" => { + let root = home_of(Some(root))?; + admit_project(&root, &DeleteAdmitFrame::decode(&args)?) + } + // releases the caller's project-scope admission after the marker + // (2.0, T4.6): `{ store, op_id, store_record? }` + "project.delete.release" => { + let root = home_of(Some(root))?; + release_project(&root, &DeleteReleaseFrame::decode(&args)?) + } + // removes only the worker's own entry for a `marked` sidecar whose + // marker is already durable (2.0, T4.5b): `{ store, stack_id, + // project? }` + "project.delete.entry" => { + let root = home_of(Some(root))?; + remove_project_entry(&root, &DeleteEntryFrame::decode(&args)?) + } + // reads the Store's own filtered project listing under this worker + // root, the same `layerfs_project::store::projects` a local Store + // uses (2.0, T4.5c): `{ store, store_record? }` + "project.list" => list_projects(root, &ListingFrame::decode(&args)?), // writes or discards one part of a caller's folder copy under this // worker's root (#536 H11a) "project.stage" => staged::run(root, &StageFrame::decode(args)?), diff --git a/control/crates/layerfs-control/tests/in_use_detail.rs b/control/crates/layerfs-control/tests/in_use_detail.rs index e83e63c43c..be250e38ae 100644 --- a/control/crates/layerfs-control/tests/in_use_detail.rs +++ b/control/crates/layerfs-control/tests/in_use_detail.rs @@ -1,9 +1,9 @@ -//! `InUse` from `project.remove` and `store.disconnect` fills -//! `detail.related` with `{kind, name}` entries (#536 §10b refusal detail): -//! the project's Sandboxes, and another process's lease on the Store. +//! `InUse` from `store.disconnect` fills `detail.related` with +//! `{kind, name}` entries (#536 §10b refusal detail): another process's +//! lease on the Store. use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::events::Events; -use layerfs_home::home::{paths, private_dir, write_entity, write_record}; +use layerfs_home::home::{paths, private_dir, write_entity}; use std::path::PathBuf; fn t(s: &str) -> SdkValue { @@ -34,36 +34,6 @@ fn refused(run: impl FnOnce(&mut Events<'_>) -> layerfs_home::outcome::Outcome) run(&mut Events::new(&mut sink)).unwrap_err() } -#[test] -fn project_remove_names_the_project_s_sandboxes() { - let home = home("project"); - let store = SdkValue::map([ - ("name", t("shop")), - ("kind", t("sqlite")), - ("machine", t("native")), - ]); - write_entity(&paths::store_record(&home, "shop"), &store).unwrap(); - let project = SdkValue::map([("name", t("api"))]); - write_entity(&paths::project_record(&home, "shop", "api"), &project).unwrap(); - for name in ["sa", "sb"] { - let sandbox = SdkValue::map([ - ("v", SdkValue::Int(1)), - ("name", t(name)), - ("project", t("shop/api")), - ("machine", t("native")), - ]); - write_record(&paths::sandbox_record(&home, name), &sandbox).unwrap(); - } - let project = layerfs_home::names::ProjectRef::parse("shop/api").unwrap(); - let refusal = refused(|events| layerfs_control::project::remove(&home, events, &project)); - assert_eq!(refusal.code, "InUse", "{refusal:?}"); - let sandbox = |n: &str| ("sandbox".to_owned(), n.to_owned()); - assert_eq!(related(&refusal), [sandbox("sa"), sandbox("sb")]); - // nothing was removed - assert!(paths::project_record(&home, "shop", "api").is_file()); - let _ = std::fs::remove_dir_all(&home); -} - #[test] fn store_disconnect_names_another_process_s_lease() { let home = home("disconnect"); diff --git a/control/crates/layerfs-control/tests/inventory.rs b/control/crates/layerfs-control/tests/inventory.rs index d72f18ba2a..badc1b0282 100644 --- a/control/crates/layerfs-control/tests/inventory.rs +++ b/control/crates/layerfs-control/tests/inventory.rs @@ -1,6 +1,6 @@ //! The relationships no record stores (#536 §5, §9.2): which Stores and //! Sandboxes a machine holds, which Sandboxes a Store or a project has, and -//! the `InUse` refusals built from them, with their texts. Layout 3: a +//! the `InUse`/`Busy` refusals built from them, with their texts. Layout 3: a //! Sandbox is read from `sandbox//` alone; a layout 2 folder under a //! machine is never read. use layerfs_bridge::control::SdkValue; @@ -118,6 +118,27 @@ fn relationships_are_read_from_the_records() { assert!(inventory::sandboxes_of_project(&home, "docs/api") .unwrap() .is_empty()); + // §5: an open Sandbox of the project refuses `project.delete` with + // `Busy`, even with nothing mounted. + let refusal = inventory::can_remove_project(&home, "docs/web").unwrap_err(); + assert_eq!(refusal.code, "Busy"); + assert_eq!(refusal.message, "Sandbox b of docs/web is running"); + // #536 §10: the detail names the entity and phase/op, not only `related` + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project")); + assert_eq!(entity.get_text("name"), Some("docs/web")); + assert_eq!(refusal.detail.get("phase"), Some(&SdkValue::Null)); + assert_eq!(refusal.detail.get("op"), Some(&SdkValue::Null)); + assert_eq!( + refusal.detail.get("related"), + Some(&SdkValue::List(vec![SdkValue::map([ + ("kind", SdkValue::text("sandbox")), + ("name", SdkValue::text("b")) + ])])) + ); + let refusal = inventory::can_remove_project(&home, "shop/api").unwrap_err(); + assert_eq!(refusal.code, "Busy"); + assert!(inventory::can_remove_project(&home, "docs/api").is_ok()); let _ = std::fs::remove_dir_all(&home); } @@ -216,13 +237,6 @@ fn the_checks_refuse_in_use_with_the_same_texts() { }; assert_eq!(names(refusal.detail.get("related").unwrap()), ["a", "c"]); assert!(inventory::store_idle(&home, "pg").is_ok()); - let refusal = inventory::can_remove_project(&home, "docs/web").unwrap_err(); - assert_eq!(refusal.message, "Sandbox b of docs/web is running"); - assert_eq!(names(refusal.detail.get("related").unwrap()), ["b"]); - let refusal = inventory::can_remove_project(&home, "shop/api").unwrap_err(); - assert_eq!(refusal.message, "Sandboxes a, c of shop/api are running"); - assert_eq!(names(refusal.detail.get("related").unwrap()), ["a", "c"]); - assert!(inventory::can_remove_project(&home, "shop/web").is_ok()); let _ = std::fs::remove_dir_all(&home); } diff --git a/control/crates/layerfs-control/tests/project_delete_call.rs b/control/crates/layerfs-control/tests/project_delete_call.rs new file mode 100644 index 0000000000..cc88aabe6b --- /dev/null +++ b/control/crates/layerfs-control/tests/project_delete_call.rs @@ -0,0 +1,17 @@ +//! `project.delete` is reachable at the control layer behind the call +//! dispatch T11 will add (the API/MCP call table itself is unchanged here). +use layerfs_home::events::Events; +use layerfs_home::names::ProjectRef; + +#[test] +fn project_delete_is_reachable_from_control_and_refuses_an_unknown_store() { + let home = std::env::temp_dir().join(format!("layerfs-control-delete-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&home); + layerfs_home::home::layout::create(&home).unwrap(); + let project = ProjectRef::parse("shop/app").unwrap(); + let mut sink = |_| {}; + let refused = + layerfs_control::project::delete(&home, &mut Events::new(&mut sink), &project).unwrap_err(); + assert_eq!(refused.code, "NoSuchStore"); + let _ = std::fs::remove_dir_all(&home); +} diff --git a/control/crates/layerfs-control/tests/project_delete_worker.rs b/control/crates/layerfs-control/tests/project_delete_worker.rs new file mode 100644 index 0000000000..e2cd63c3e0 --- /dev/null +++ b/control/crates/layerfs-control/tests/project_delete_worker.rs @@ -0,0 +1,900 @@ +//! `project.delete` on a worker-backed Store (T4.6): the caller writes its +//! sidecar naming the admission, then admits the project scope through the +//! worker and sends the marker. A live mount in the worker's Store refuses +//! the call with `MountsActive` (holders reported) and a live operation with +//! `Busy`, keeping the pending sidecar, writing no marker and leaking no +//! admission row; a pending retry then redos the marker and keeps the +//! caller and worker entries, and the next call finishes them with only the +//! entry-removal frame. The successful path releases the admission after the +//! marker. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{self, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_machine::worker::{self, read_frame, read_frame_within, write_frame, Hello, MAX_HELLO}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store_service::history::lookup; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layerfs_home::home::layout::create(&directory).unwrap(); + directory +} + +fn text(value: &str) -> SdkValue { + SdkValue::text(value) +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-worker-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// Moves Store `store`'s file from `from`'s Store folder to `to`'s. +fn move_store_file(from: &Path, to: &Path, store: &str) { + let source = paths::store_dir(from, store); + let target = paths::store_dir(to, store); + home::private_dir(&target).unwrap(); + for entry in std::fs::read_dir(&source).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("store.sqlite") { + std::fs::rename(entry.path(), target.join(&name)).unwrap(); + } + } +} + +/// The Store record of `store` places it on machine `machine`. +fn place_store(home: &Path, store: &str, machine: &str) { + let record_path = paths::store_record(home, store); + let record = home::read_record(&record_path).unwrap().unwrap(); + write_record( + &record_path, + &home::with_field(record, "machine", text(machine)), + ); +} + +fn write_record(path: &Path, value: &SdkValue) { + home::write_record(path, value).unwrap(); +} + +/// A worker's `OpenArgs` for Store `shop` under `root`, with the caller's +/// Store record (the same context every `project.delete*` frame carries). +fn worker_open(root: &Path, record: &StoreRecord) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(root, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + } +} + +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +fn project_names(open: &OpenArgs) -> Vec { + with_service(open, |service| { + let listed = service + .store + .published(layerfs_project::store::projects) + .unwrap(); + Ok(listed + .into_iter() + .map(|stack| stack.name.as_str().to_owned()) + .collect()) + }) + .unwrap() +} + +fn remove(home: &Path) -> Result { + let mut sink = |_| {}; + layerfs_control::project::delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + ) +} + +#[test] +fn the_worker_path_writes_the_sidecar_then_admits_and_releases_after_the_marker() { + // a home with Store `shop` and project `app`, created locally. A + // sandbox filesystem cannot hold a directly opened Store; the fake-home + // cases in the `layerfs-store` tests cover the same home path, and this + // case runs where the Store's filesystem carries SQLite locks. + let home = tempdir("home"); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) + if refusal.code == "Unsupported" + && refusal.message.contains("does not carry SQLite locks") => + { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir("folder"); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: Some(60_000), + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let caller_open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let (stack, branch) = with_service(&caller_open, |service| { + let stack = lookup::project(&service.store, "app").unwrap(); + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + let NamedSnapshot::Found(snapshot) = service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + else { + panic!("no main Branch"); + }; + Ok((stack.id, snapshot.branch.id)) + }) + .unwrap(); + + // the Store belongs to machine `box`; its file is under the worker's root + let root = tempdir("root"); + move_store_file(&home, &root, "shop"); + home::write_record( + &paths::machine_record(&home, "box"), + &SdkValue::map([("name", text("box")), ("platform", text("lima"))]), + ) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(&address)), ("secret", text("k"))]), + ) + .unwrap(); + place_store(&home, "shop", "box"); + let served = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), served) + }); + + let worker = worker_open(&root, &record); + + // a live mount in the worker's Store: `MountsActive`, a pending sidecar, + // no marker + let mount = GcIntent::shared( + [9; 16], + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(&worker, |service| { + service.store.history().gc_register(&mount).unwrap(); + Ok(()) + }) + .unwrap(); + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + assert!( + intents(&worker, stack).is_empty(), + "no project-scope row yet" + ); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + let pending = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar precedes the admission"); + assert_eq!( + pending.state, + layerfs_store::project::delete::IntentState::Pending + ); + let sidecar_op = pending.op.expect("the sidecar names its admission"); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert_eq!(project_names(&worker), vec!["app".to_owned()], "no marker"); + assert!( + intents(&worker, stack).is_empty(), + "the refused admission left no project-scope row" + ); + + // the mount ends; a live removal at project scope is `Busy` + with_service(&worker, |service| { + service.store.history().gc_release(&[9; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + // A `project.delete` whose reply was lost left its project-scope + // admission live. The control's own frame names the project, so the row + // does too and the retry's `Busy` reports the project rather than the hex + // LayerStack id. + let lost_admit = layerfs_store::project::frame::DeleteAdmitFrame { + store: StoreName::parse("shop").unwrap(), + stack, + op: [7; 16], + project: Some("app".to_owned()), + store_record: Some(record.to_record().clone()), + }; + layerfs_store::project::delete::admit_project(&root, &lost_admit).unwrap(); + let held = intents(&worker, stack); + assert_eq!(held.len(), 1); + assert_eq!( + held[0].project.as_deref(), + Some("app"), + "the control's admission names the project" + ); + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project")); + assert_eq!(entity.get_text("name"), Some("app"), "{refusal:?}"); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays") + .op, + Some(sidecar_op), + "the retry reuses the same admission identity" + ); + assert_eq!( + intents(&worker, stack).len(), + 1, + "only the test's own holder" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + + // the worker's own copy of the project entry: the deferred call must + // keep it, and the finishing call removes it. + home::write_record( + &paths::project_record(&root, "shop", "app"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", text("app")), + ("stack", text(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + std::fs::create_dir_all(paths::project_dir(&root, "shop", "app")).unwrap(); + + // the holder ends. The sidecar is still pending, so this call retries + // without a confirmed marker: it writes the marker, settles the sidecar + // and keeps the caller and worker entries. + with_service(&worker, |service| { + service.store.history().gc_release(&[7; 16]).unwrap(); + Ok(()) + }) + .unwrap(); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert!( + !outcome.get_bool("home_entry_removed"), + "a pending retry settles without finishing the entry" + ); + assert_eq!(outcome.get_int("deleted_ms"), Some(pending.deleted_ms)); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert!( + paths::project_dir(&root, "shop", "app").is_dir(), + "the worker keeps its own entry" + ); + let settled = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the sidecar is marked"); + assert_eq!( + settled.state, + layerfs_store::project::delete::IntentState::Marked + ); + assert_eq!(settled.deleted_ms, pending.deleted_ms); + assert!( + intents(&worker, stack).is_empty(), + "the admission was released" + ); + assert!(project_names(&worker).is_empty(), "the project is retired"); + + // The marker is confirmed now: the next call removes both entries and + // unlinks the sidecar. + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert!(!paths::project_dir(&root, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + assert!( + intents(&worker, stack).is_empty(), + "the admission was released" + ); + let _ = std::fs::remove_dir_all(&home); + let _ = std::fs::remove_dir_all(&root); +} + +/// A fake worker on `listener`: answers its secret, answers each request with +/// `reply`, and shares back the requests it read. A `None` answer drops the +/// connection without a reply, as a lost answer does. +fn fake_worker_with( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> Option + Send + 'static, +) -> Arc>> { + let seen: Arc>> = Arc::default(); + let requests = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + return; + }; + let Some(_hello) = read_frame_within(&mut stream, MAX_HELLO).ok().flatten() else { + continue; + }; + let ready = Hello { + protocol: Some(worker::PROTOCOL), + bundle: None, + }; + if write_frame(&mut stream, &ready.to_frame()).is_err() { + continue; + } + let Some(request) = read_frame(&mut stream).ok().flatten() else { + continue; + }; + let answer = reply(&request); + requests.lock().unwrap().push(request); + if let Some(answer) = answer { + let _ = write_frame(&mut stream, &answer); + } + } + }); + seen +} + +/// [`fake_worker_with`] of a worker that always answers. +fn fake_worker( + listener: TcpListener, + reply: impl Fn(&SdkValue) -> SdkValue + Send + 'static, +) -> Arc>> { + fake_worker_with(listener, move |request| Some(reply(request))) +} + +/// The `op` of one request the fake worker read. +fn op_of(request: &SdkValue) -> &str { + request.get_text("op").unwrap_or("") +} + +/// A frame's `args`. +fn args_of(request: &SdkValue) -> &SdkValue { + request.get("args").expect("args") +} + +/// The `op_id` a request's args carry. +fn sent_op(request: &SdkValue) -> String { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") + .to_owned() +} + +fn admitted_for(stack: LayerStackId) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Admitted")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ]), + )]) +} + +fn removed_for(stack: LayerStackId, deleted_ms: i64, retired: bool) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Removed")), + ("project", text("app")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(deleted_ms)), + ("retired", SdkValue::Bool(retired)), + ]), + )]) +} + +fn entry_removed_for(stack: LayerStackId) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Removed")), + ("project", text("app")), + ("stack_id", text(&hex(&stack.to_bytes()))), + ]), + )]) +} + +fn released_for(request: &SdkValue) -> SdkValue { + SdkValue::map([( + "value", + SdkValue::map([ + ("kind", text("Released")), + ("op_id", text(&sent_op(request))), + ]), + )]) +} + +fn refused(code: &str, detail: SdkValue) -> SdkValue { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", text(code)), + ("message", text("the fake worker refused")), + ("detail", detail), + ]), + )]) +} + +/// A caller's home naming Store `shop` on machine `box`, whose project `app` +/// records `stack`, with the fake worker's address. +fn caller_home(tag: &str, address: &str, stack: LayerStackId) -> PathBuf { + let home = tempdir(tag); + home::write_record( + &paths::store_record(&home, "shop"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("kind", text("sqlite")), + ("name", text("shop")), + ("machine", text("box")), + ]), + ) + .unwrap(); + home::write_record( + &paths::project_record(&home, "shop", "app"), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", text("app")), + ("stack", text(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); + home::write_record( + &paths::worker_record(&home, "box"), + &SdkValue::map([("address", text(address)), ("secret", text("k"))]), + ) + .unwrap(); + home +} + +#[test] +fn a_fake_workers_mounts_active_refusal_keeps_a_pending_sidecar_and_writes_no_marker() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([3; 16]); + let home = caller_home("fake-mount", &address, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", text("mount")), + ("entity", text("main")), + ])]), + )]); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => refused("MountsActive", holders.clone()), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("the fake worker's Store has a live mount"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + // The caller wrote its sidecar before admitting and sent no marker; the + // refused admission registered nothing, so the caller settles the + // identity it wrote and keeps the pending sidecar for a retry. + let pending = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar precedes the admission"); + assert_eq!( + pending.state, + layerfs_store::project::delete::IntentState::Pending + ); + assert!(pending.op.is_some(), "the sidecar names its admission"); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + ["project.delete.admit", "project.delete.release"], + "{requests:?}" + ); + assert_eq!(sent_op(&requests[0]), sent_op(&requests[1])); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_marker_refusal_releases_the_admission_it_took() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([4; 16]); + let home = caller_home("fake-busy", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => refused("Busy", SdkValue::Null), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("a live operation holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + // the admission was taken, named in the sidecar, used by the marker and + // released after the marker's refusal + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + drop(requests); + let body: [u8; 16] = hex_decode(&op).try_into().unwrap(); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays for a retry") + .op, + Some(body) + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_success_removes_the_entry_and_releases_the_admission() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([5; 16]); + let home = caller_home("fake-ok", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => removed_for(stack, 42, false), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let op = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), op); + assert_eq!(sent_op(&requests[2]), op); + assert!( + !args_of(&requests[1]).get_bool("pending_retry"), + "a fresh delete is no retry" + ); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The bytes of a lowercase hex string. +fn hex_decode(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) + .collect() +} + +#[test] +fn a_fake_workers_pending_retry_settles_the_marker_and_then_finishes_the_entry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([8; 16]); + let home = caller_home("fake-settle", &address, stack); + // The sidecar written by the crashed attempt, before its admission. + let op = [0x3c; 16]; + let pending = layerfs_store::project::delete::Intent { + store: "shop".to_owned(), + project: "app".to_owned(), + stack, + deleted_ms: 77, + attempt_ms: 1, + state: layerfs_store::project::delete::IntentState::Pending, + op: Some(op), + }; + layerfs_store::project::delete::Intent::write(&home, &pending).unwrap(); + + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => removed_for(stack, 77, false), + "project.delete.entry" => entry_removed_for(stack), + "project.delete.release" => released_for(request), + other => panic!("unexpected op {other}"), + }); + + // The first call is the pending retry: it redos the marker with the + // sidecar's deleted_ms, keeps the entry and leaves the sidecar marked. + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert!(!outcome.get_bool("home_entry_removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(77)); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let settled = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the sidecar is marked"); + assert_eq!( + settled.state, + layerfs_store::project::delete::IntentState::Marked + ); + { + let requests = seen.lock().unwrap(); + let marker = requests + .iter() + .find(|request| op_of(request) == "project.delete") + .expect("the marker frame"); + assert_eq!(sent_op(marker), hex(&op)); + assert!(args_of(marker).get_bool("pending_retry")); + } + + // The marker is confirmed: the next call sends only the entry-removal + // frame, takes no admission and writes no marker. + let outcome = remove(&home).unwrap(); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release", + "project.delete.entry" + ], + "{requests:?}" + ); + let entry = requests.last().expect("the entry frame"); + assert_eq!(args_of(entry).get_text("project"), Some("app")); + let expected = hex(&stack.to_bytes()); + assert_eq!(args_of(entry).get_text("stack_id"), Some(expected.as_str())); + assert!( + args_of(entry).get("op_id").is_none(), + "the entry frame takes no admission" + ); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_fake_workers_marked_sidecar_finishes_with_only_the_entry_frame() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([9; 16]); + let home = caller_home("fake-marked", &address, stack); + // The sidecar of an attempt whose marker committed but whose entry + // removal did not: only the entry removal is left. + let marked = layerfs_store::project::delete::Intent { + store: "shop".to_owned(), + project: "app".to_owned(), + stack, + deleted_ms: 55, + attempt_ms: 1, + state: layerfs_store::project::delete::IntentState::Marked, + op: Some([0x4d; 16]), + }; + layerfs_store::project::delete::Intent::write(&home, &marked).unwrap(); + + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.entry" => entry_removed_for(stack), + other => panic!("unexpected op {other}"), + }); + + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(55)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!(ops, ["project.delete.entry"], "{requests:?}"); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +#[test] +fn a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([6; 16]); + let home = caller_home("fake-lost", &address, stack); + // The first marker attempt loses its answer; the second (the retry) + // retires the Stack with the sidecar's `deleted_ms`. + let attempts = Arc::new(AtomicU32::new(0)); + let count = attempts.clone(); + let seen = fake_worker_with(listener, move |request| match op_of(request) { + "project.delete.admit" => Some(admitted_for(stack)), + "project.delete" => { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + None + } else { + Some(removed_for(stack, 42, true)) + } + } + "project.delete.release" => Some(released_for(request)), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("the marker answer was lost"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + // The sidecar stays `pending`, naming the admission, and the entry stays + // for the retry: an unknown outcome is never reported clean. + let intent = layerfs_store::project::delete::Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays"); + assert_eq!( + intent.state, + layerfs_store::project::delete::IntentState::Pending + ); + let op = intent.op.expect("the sidecar names its admission"); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let first = sent_op(&requests[0]); + assert_eq!(sent_op(&requests[1]), first); + assert_eq!(sent_op(&requests[2]), first); + assert_eq!(hex_decode(&first), op.to_vec()); + drop(requests); + + // The retry re-registers the same identity and finishes the entry. + let outcome = remove(&home).expect("the retry settles the unknown outcome"); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(42)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!( + layerfs_store::project::delete::Intent::read(&home, "shop", "app").unwrap(), + None, + "the retry unlinked the sidecar" + ); + let requests = seen.lock().unwrap(); + let ops: Vec<&str> = requests.iter().map(op_of).collect(); + assert_eq!( + ops, + [ + "project.delete.admit", + "project.delete", + "project.delete.release", + "project.delete.admit", + "project.delete", + "project.delete.release" + ], + "{requests:?}" + ); + let same: Vec<&str> = requests[3..].iter().map(sent_op_ref).collect(); + assert_eq!(same, [first.as_str(), first.as_str(), first.as_str()]); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} + +/// The `op_id` a request's args carry, borrowed. +fn sent_op_ref(request: &SdkValue) -> &str { + args_of(request) + .get_text("op_id") + .expect("the frame names its admission") +} + +/// F43R4-1: the worker's `project.delete.release` can answer `Unknown` with +/// the bare empty-map detail (a quarantined session); the caller still names +/// the project and the admission it held (spec §10 A6). +#[test] +fn a_fake_workers_empty_unknown_release_names_the_project_and_op() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let stack = LayerStackId::from_authority([0x1a; 16]); + let home = caller_home("fake-release-empty-unknown", &address, stack); + let seen = fake_worker(listener, move |request| match op_of(request) { + "project.delete.admit" => admitted_for(stack), + "project.delete" => removed_for(stack, 42, false), + "project.delete.release" => refused("Unknown", SdkValue::map::<0>([])), + other => panic!("unexpected op {other}"), + }); + + let refusal = remove(&home).expect_err("the release answer was lost"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity detail"); + assert_eq!(entity.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(entity.get_text("name"), Some("app"), "{refusal:?}"); + assert!( + refusal.detail.get("phase").expect("phase") == &SdkValue::Null, + "a lost release has no phase" + ); + let op = refusal.detail.get_text("op").expect("the admission"); + let requests = seen.lock().unwrap(); + assert_eq!(sent_op(&requests[2]), op, "{requests:?}"); + drop(requests); + let _ = std::fs::remove_dir_all(home); +} diff --git a/control/crates/layerfs-control/tests/worker_root.rs b/control/crates/layerfs-control/tests/worker_root.rs index be15497030..def8098656 100644 --- a/control/crates/layerfs-control/tests/worker_root.rs +++ b/control/crates/layerfs-control/tests/worker_root.rs @@ -127,3 +127,31 @@ fn the_retired_host_ops_are_refused() { } std::fs::remove_dir_all(root).unwrap(); } + +/// `store.compact` has no worker route (spec S6 deviation D5): a caller from +/// a home whose sqlite Store lives on another machine is refused before any +/// operation runs, and the worker names the unknown operation. +#[test] +fn store_compact_is_not_a_worker_operation() { + let root = tempdir("no-compact"); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = Endpoint { + address: listener.local_addr().unwrap(), + secret: "k".to_owned(), + }; + let guest = root.clone(); + std::thread::spawn(move || { + layerfs_control::worker_server::serve(listener, "k".to_owned(), guest) + }); + for _ in 0..200 { + if worker::ping(&endpoint).is_ok() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let args = SdkValue::map([("name", SdkValue::text("shop"))]); + let refusal = worker::call(&endpoint, "store.compact", args, &mut |_| {}).unwrap_err(); + assert_eq!(refusal.code, "InvalidInput", "{}", refusal.message); + assert_eq!(refusal.message, "the worker has no operation store.compact"); + std::fs::remove_dir_all(root).unwrap(); +} diff --git a/control/crates/layerfs-home/src/process.rs b/control/crates/layerfs-home/src/process.rs index ce63bbda16..c5a705a3b6 100644 --- a/control/crates/layerfs-home/src/process.rs +++ b/control/crates/layerfs-home/src/process.rs @@ -55,6 +55,43 @@ pub fn alive(pid: u32) -> bool { } } +/// This process's PID-namespace inode, when the platform names one. A pid is +/// meaningful only inside the namespace that recorded it. +fn pid_namespace() -> Option { + #[cfg(target_os = "linux")] + { + let target = std::fs::read_link("/proc/self/ns/pid").ok()?; + let text = target.to_str()?; + text.strip_prefix("pid:[")?.strip_suffix(']')?.parse().ok() + } + #[cfg(not(target_os = "linux"))] + { + None + } +} + +/// Whether a compaction's recorded owner (`pid`, kernel start time, boot id +/// and PID namespace) is provably a process this kernel still runs. Every fact +/// must be recorded and still match: a record from another boot, another PID +/// namespace or with a missing fact is never *provably* live, so the engine +/// keeps its dead-owner adoption path. +pub fn recorded_owner_alive( + pid: Option, + start: Option, + boot: Option, + namespace: Option, +) -> bool { + let (Some(pid), Some(start), Some(boot)) = (pid, start, boot) else { + return false; + }; + if let (Some(recorded), Some(here)) = (namespace, pid_namespace()) { + if recorded != here { + return false; + } + } + alive_as(pid, Some(start), Some(boot)) +} + /// Whether `pid` is alive and, when its start time and boot were recorded, /// still the same process. pub fn alive_as(pid: u32, start: Option, boot: Option) -> bool { diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index 49c161ef81..4aad4a659e 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -291,6 +291,19 @@ fn matches_reply(request: &Request, reply: &Reply) -> bool { Reply::Bound { token, binding }, ) => token.workspace == *workspace && binding.branch.name.as_str() == branch, (Request::MountNamed { .. }, Reply::SdkRefused(_)) => true, + ( + Request::MountPlaced { + workspace, branch, .. + }, + Reply::Bound { token, binding }, + ) => token.workspace == *workspace && binding.branch.id == *branch, + ( + Request::MountNamedPlaced { + workspace, branch, .. + }, + Reply::Bound { token, binding }, + ) => token.workspace == *workspace && binding.branch.name.as_str() == branch, + (Request::MountNamedPlaced { .. }, Reply::SdkRefused(_)) => true, ( Request::Commit(_), Reply::Committed(_) | Reply::MergeCommitted(_) | Reply::Conflict(_), diff --git a/control/crates/layerfs-sandbox/src/daemon/internal.rs b/control/crates/layerfs-sandbox/src/daemon/internal.rs index 3188e6d448..58b6a0ace2 100644 --- a/control/crates/layerfs-sandbox/src/daemon/internal.rs +++ b/control/crates/layerfs-sandbox/src/daemon/internal.rs @@ -2,7 +2,7 @@ pub use super::channel::{Control, ControlCause, ControlFailure, ControlPhase}; use crate::{BoundWorkspace, OperationFailure, WorkspaceApi}; use layerfs_bridge::{ - control::{CleanupObservation, ReadyMount, WorkspaceStatus, WorkspaceToken}, + control::{CleanupObservation, MountPlace, ReadyMount, WorkspaceStatus, WorkspaceToken}, duplex::Connection, }; use layerfs_history::{BranchId, WorkspaceId}; @@ -16,6 +16,7 @@ pub fn workspace(control: &mut Control) -> InternalWorkspaceApi<'_> { InternalWorkspaceApi(WorkspaceApi { control, observation_scope: None, + place: MountPlace::default(), }) } /// Applies explicit observation scope only to this borrow's filesystem calls. @@ -26,6 +27,7 @@ pub fn with_observations<'a>( InternalWorkspaceApi(WorkspaceApi { control, observation_scope: Some(scope), + place: MountPlace::default(), }) } impl<'a> Deref for InternalWorkspaceApi<'a> { @@ -40,6 +42,13 @@ impl DerefMut for InternalWorkspaceApi<'_> { } } impl InternalWorkspaceApi<'_> { + /// Names the Sandbox and machine a mount through this channel belongs + /// to; a mount then registers them beside its Workspace. + #[must_use] + pub fn placed(mut self, sandbox: Option<&str>, machine: Option<&str>) -> Self { + self.0.place = MountPlace::new(sandbox, machine); + self + } /// Original logical bind for component proofs; kernel mount still uses both steps. pub fn bind( &mut self, diff --git a/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs b/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs index 038cf2245f..9c5e86e90f 100644 --- a/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs +++ b/control/crates/layerfs-sandbox/src/daemon/workspace/mount.rs @@ -1,6 +1,6 @@ //! Logical binding and kernel mount acknowledgements. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::control::{ReadyMount, Reply, Request, WorkspaceToken}; +use layerfs_bridge::control::{MountPlace, ReadyMount, Reply, Request, WorkspaceToken}; use layerfs_history::{BranchId, BranchSnapshot, WorkspaceId}; /// Explicit filesystem operations on one caller-owned sequential control channel. @@ -8,6 +8,10 @@ use layerfs_history::{BranchId, BranchSnapshot, WorkspaceId}; pub struct WorkspaceApi<'a> { pub control: &'a mut Control, pub observation_scope: Option<&'a [u8; 32]>, + /// The Sandbox and machine a mount through this channel belongs to, + /// when its caller knows them: the daemon registers the holder with + /// them. Both absent when the caller knows neither. + pub place: MountPlace, } impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. @@ -21,8 +25,16 @@ impl<'a> WorkspaceApi<'a> { Self { control, observation_scope: None, + place: MountPlace::default(), } } + /// Names the Sandbox and machine a mount through this channel belongs + /// to; a mount then registers them beside its Workspace. + #[must_use] + pub fn placed(mut self, sandbox: Option<&str>, machine: Option<&str>) -> Self { + self.place = MountPlace::new(sandbox, machine); + self + } pub fn exchange(&mut self, request: Request) -> Result> { let request = match self.observation_scope { Some(scope) => Request::Observed { diff --git a/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs b/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs index 1c6836649b..25ab47ebce 100644 --- a/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs +++ b/control/crates/layerfs-sandbox/src/daemon/workspace/recovery.rs @@ -12,7 +12,15 @@ impl WorkspaceApi<'_> { workspace: WorkspaceId, branch: BranchId, ) -> Result> { - let request = Request::Mount { workspace, branch }; + let request = if self.place.known() { + Request::MountPlaced { + workspace, + branch, + place: self.place.clone(), + } + } else { + Request::Mount { workspace, branch } + }; match self.exchange(request.clone())? { Reply::Bound { token, binding } => Ok(BoundWorkspace { token, binding }), reply => Err(OperationFailure::unexpected(request, reply)), @@ -27,10 +35,19 @@ impl WorkspaceApi<'_> { project: &str, branch: &str, ) -> Result> { - let request = Request::MountNamed { - workspace, - project: project.to_owned(), - branch: branch.to_owned(), + let request = if self.place.known() { + Request::MountNamedPlaced { + workspace, + project: project.to_owned(), + branch: branch.to_owned(), + place: self.place.clone(), + } + } else { + Request::MountNamed { + workspace, + project: project.to_owned(), + branch: branch.to_owned(), + } }; match self.exchange(request.clone())? { Reply::Bound { token, binding } => Ok(BoundWorkspace { token, binding }), diff --git a/control/crates/layerfs-sandbox/src/lifecycle.rs b/control/crates/layerfs-sandbox/src/lifecycle.rs index 069d3b4c58..6f12402283 100644 --- a/control/crates/layerfs-sandbox/src/lifecycle.rs +++ b/control/crates/layerfs-sandbox/src/lifecycle.rs @@ -342,6 +342,14 @@ pub fn close(host: &Host, force: bool, held: &[String]) -> Outcome { )]), }); } + // §5.1: the daemon unmount transaction releases each mount intent; a + // force close drains the Workspaces it is about to kill first, so a + // stopped Sandbox leaves no live intent that would refuse a later GC + // call. A Workspace the daemon cannot drain is reported as unknown below. + drop(_guard); + for path in &mounted { + let _ = crate::workspace::serve::unmount(host, Some(path), true, true); + } let value = crate::files::started(&directory)? .ok_or_else(|| refused("NoSuchSandbox", format!("Sandbox {id} was closed")))?; let mut owner = crate::reopen::recorded(host, &id, &value)? diff --git a/control/crates/layerfs-sandbox/src/registry.rs b/control/crates/layerfs-sandbox/src/registry.rs index 2143093eee..1c152261d1 100644 --- a/control/crates/layerfs-sandbox/src/registry.rs +++ b/control/crates/layerfs-sandbox/src/registry.rs @@ -1,6 +1,6 @@ //! What this process owns: its running Inits and its live Sandboxes. A -//! project itself is only its home entry, so a project removed there can be -//! initialized again without a restart. +//! project itself is only its home entry, so a project deleted there can be +//! initialized again (under another name, until a compaction frees the name). //! //! A Sandbox is one daemon process (`runtime`: native, or namespaces with //! its init stage), known by its name. Its runtime runs diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index 7cb83fb3cb..8d0af8dbff 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -13,11 +13,11 @@ use crate::daemon::values; use crate::host::Host; use crate::internal::workspace; use crate::lifecycle::{operation, WORKSPACES}; -use crate::record::{DaemonRecord, WorkspaceRecord}; +use crate::record::{DaemonRecord, SandboxRecord, WorkspaceRecord}; use crate::registry::{sandbox, sandbox_live, Mount}; use crate::OperationCause; use crate::Unmounted; -use layerfs_bridge::control::{CommitControl, Reply, SdkRefusal, SdkValue}; +use layerfs_bridge::control::{CommitControl, MountPlace, Reply, SdkRefusal, SdkValue}; use layerfs_history::{BranchId, CommitId, WorkspaceId}; use layerfs_home::helpers::{lock, random, required}; use layerfs_home::names::WorkspaceName; @@ -102,7 +102,14 @@ pub fn mount(host: &Host, branch: Option<&str>, name: Option<&str>) -> Outcome { let id = WorkspaceId::from_authority(random()?).map_err(failed)?; let mounted = { let mut control = crate::daemon::trace::control(&state.controls)?; - workspace(&mut control).mount_named(id, (project, branch)) + // This Sandbox's record names the machine it runs on; the daemon + // that binds the mount registers both names beside its Workspace. + let machine = + SandboxRecord::read_in(&directory, sandbox)?.map(|record| record.machine().to_owned()); + let place = MountPlace::new(Some(state.id.as_str()), machine.as_deref()); + workspace(&mut control) + .placed(place.sandbox.as_deref(), place.machine.as_deref()) + .mount_named(id, (project, branch)) }; let mounted = mounted.map_err(|failure| match (&failure.bound, &failure.failure.cause) { // the daemon's lookup refusal, nothing bound: the call's answer diff --git a/control/crates/layerfs-store/Cargo.toml b/control/crates/layerfs-store/Cargo.toml index 4c8a3cf42e..779134cf82 100644 --- a/control/crates/layerfs-store/Cargo.toml +++ b/control/crates/layerfs-store/Cargo.toml @@ -25,3 +25,9 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs", "user", [features] # https:// remotes: Cloudflare R2, MinIO behind TLS. tls = ["layerfs-bucket/tls"] + +# The control-level PostgreSQL `store.status` test drives a real server and +# inserts the rows a compaction leaves; this is the pinned client the +# persistence PostgreSQL feature already locks, no new version. +[dev-dependencies] +postgres = "=0.19.14" diff --git a/control/crates/layerfs-store/src/calls/branch.rs b/control/crates/layerfs-store/src/calls/branch.rs index 023d276bb2..f4db191c3d 100644 --- a/control/crates/layerfs-store/src/calls/branch.rs +++ b/control/crates/layerfs-store/src/calls/branch.rs @@ -124,6 +124,15 @@ pub struct BranchPublish { pub force: Option, } +/// `branch.remove(project, branch)`. +#[derive(Clone, Debug, Default)] +pub struct BranchRemove { + /// The project. + pub project: String, + /// The Branch. + pub branch: String, +} + /// `branch.diff(project, from_branch, to_branch, { paths_only, page_size, /// page_pos, cursor })`. #[derive(Clone, Debug, Default)] @@ -187,6 +196,33 @@ pub fn publish(home: &Path, events: &mut Events<'_>, request: &BranchPublish) -> on_project(home, events, &request.project, StoreOp::Publish(request)) } +/// `branch.remove`: one Branch hidden by its Store marker. The result names +/// the project as the caller gave it, `/`: the Store sees +/// only the project's name within it and echoes that alone. +pub fn remove(home: &Path, events: &mut Events<'_>, request: &BranchRemove) -> Outcome { + let value = match on_project( + home, + events, + &request.project, + StoreOp::BranchRemove(request), + ) { + Ok(value) => value, + Err(refusal) => { + return Err(crate::unknown::named_unknown( + refusal, + "branch", + &request.branch, + None, + )) + } + }; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} + /// `branch.diff`: the changes between two Branches. pub fn diff(home: &Path, events: &mut Events<'_>, request: &BranchDiff) -> Outcome { on_project( diff --git a/control/crates/layerfs-store/src/calls/op.rs b/control/crates/layerfs-store/src/calls/op.rs index e8bdd8f653..84456ab72a 100644 --- a/control/crates/layerfs-store/src/calls/op.rs +++ b/control/crates/layerfs-store/src/calls/op.rs @@ -4,6 +4,7 @@ //! a `store.call` frame to the Store's worker). use super::branch::{ BranchCommits, BranchDiff, BranchFork, BranchGetCommit, BranchMerge, BranchPublish, + BranchRemove, }; use super::commit::CommitDiff; use super::file::{ @@ -62,6 +63,9 @@ pub enum StoreOp<'a> { Statistics, /// Entry counts at one project pin. ProjectStats(&'a super::project::ProjectStats), + /// `project.remove_file`: hide one path from every node at or below its + /// marker frontier. + RemoveFile(&'a super::project::ProjectRemoveFile), /// `project.list_branches`: the Branch `name`, or one page. ListBranches { /// The one Branch it lists. @@ -88,6 +92,8 @@ pub enum StoreOp<'a> { Merge(&'a BranchMerge), /// `branch.publish`. Publish(&'a BranchPublish), + /// `branch.remove`. + BranchRemove(&'a BranchRemove), /// `branch.diff_branches`. DiffBranches(&'a BranchDiff), /// `branch.diff_commits`. @@ -236,6 +242,7 @@ impl StoreOp<'_> { match self { Self::Statistics => "store.stats", Self::ProjectStats(_) => "project.stats", + Self::RemoveFile(_) => "project.remove_file", Self::ListBranches { .. } => "project.list_branches", Self::ListCommits { .. } => "branch.list_commits", Self::BranchCommits(_) => "branch.commits", @@ -244,6 +251,7 @@ impl StoreOp<'_> { Self::Fork(_) => "branch.fork", Self::Merge(_) => "branch.merge", Self::Publish(_) => "branch.publish", + Self::BranchRemove(_) => "branch.remove", Self::DiffBranches(_) => "branch.diff_branches", Self::DiffCommits(_) => "branch.diff_commits", Self::Read(_) => "fs.read", @@ -269,6 +277,7 @@ impl StoreOp<'_> { match self { Self::Statistics => &mut args, Self::ProjectStats(r) => args.at(&r.branch, r.commit.as_deref()), + Self::RemoveFile(r) => args.text("path", &r.path), Self::ListBranches { name, page } => args.opt("name", *name).page(*page), Self::ListCommits { branch, page } => args.text("branch", branch).page(*page), Self::BranchCommits(r) => args @@ -300,6 +309,7 @@ impl StoreOp<'_> { .opt("expectedBase", r.expected_base.as_deref()) .opt("expectedHead", r.expected_head.as_deref()) .boolean("force", r.force), + Self::BranchRemove(r) => args.text("branch", &r.branch), Self::DiffBranches(r) => args .text("from", &r.from_branch) .text("to", &r.to_branch) diff --git a/control/crates/layerfs-store/src/calls/project.rs b/control/crates/layerfs-store/src/calls/project.rs index d61b25fb0f..532fcb3e81 100644 --- a/control/crates/layerfs-store/src/calls/project.rs +++ b/control/crates/layerfs-store/src/calls/project.rs @@ -1,5 +1,6 @@ //! Saved-tree project statistics through the ordinary Store-call route. use super::op::{on_project, StoreOp}; +use layerfs_bridge::control::SdkValue; use layerfs_home::{events::Events, outcome::Outcome}; use std::path::Path; @@ -14,6 +15,15 @@ pub struct ProjectStats { pub commit: Option, } +/// `project.remove_file(project, path)`. +#[derive(Clone, Debug, Default)] +pub struct ProjectRemoveFile { + /// `/`. + pub project: String, + /// The absolute POSIX path. + pub path: String, +} + /// Count one immutable saved tree, excluding the root directory. pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Outcome { on_project( @@ -23,3 +33,29 @@ pub fn stats(home: &Path, events: &mut Events<'_>, request: &ProjectStats) -> Ou StoreOp::ProjectStats(request), ) } + +/// `project.remove_file`: one path hidden from every node at or below +/// its marker frontier. The result names the project as the caller gave it, +/// `/`. +pub fn remove_file(home: &Path, events: &mut Events<'_>, request: &ProjectRemoveFile) -> Outcome { + let value = match on_project(home, events, &request.project, StoreOp::RemoveFile(request)) { + Ok(value) => value, + Err(refusal) => { + // The detail names the project inside its Store, as the Store + // does; the result below keeps the caller's `/` + // spelling. + let project = request + .project + .rsplit_once('/') + .map_or(request.project.as_str(), |(_, project)| project); + return Err(crate::unknown::named_unknown( + refusal, "project", project, None, + )); + } + }; + Ok(layerfs_home::home::with_field( + value, + "project", + SdkValue::text(&request.project), + )) +} diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs new file mode 100644 index 0000000000..fb9a10def6 --- /dev/null +++ b/control/crates/layerfs-store/src/compact.rs @@ -0,0 +1,263 @@ +//! `store.compact`: one Store's §7 (SQLite) or §8 (PostgreSQL) rebuild, run +//! in this process. +//! +//! The call takes the Store's lease, so a move or remove cannot start, and +//! closes the Store this process holds open: a call still running on it is +//! `Busy` in the GC vocabulary (§10 keeps `InUse` for non-GC calls), before +//! any effect. It then opens the Store's own session, runs +//! the §7 (SQLite) or §8 (PostgreSQL) compaction and answers the §11 report. +//! A SQLite Store on another machine is `Unsupported` and is never sent to +//! that machine's worker. Spec S6 says the rebuild runs on the machine that +//! holds the file; 2.0 implements that only through `place::plan`'s `Local` +//! route. This local-only reading is the documented contract deviation D5 in +//! `core/docs/issues/project-41/gc-42-decisions.md` (with the API and control +//! docs); run the call on that machine. +use crate::identity; +use crate::identity::Kind; +use crate::lease; +use crate::open::close; +use crate::open_args::OpenArgs; +use crate::place::{self, Executor}; +use crate::postgresql; +use crate::record::{StoreRecord, StoreRef}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::GcScopeKind; +use layerfs_home::events::Events; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused, Outcome}; +use layerfs_persistence::compact_metrics::{self, CompactRefusal}; +use layerfs_persistence::PostgresHandles; +use layerfs_persistence::{ + CompactFailpoint, CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, + SqlitePersistenceProfile, +}; +use layerfs_store_service::history::{admission, lookup}; +use std::path::Path; + +/// The T-G65 crash harness's gate: the real `layerfs store compact` process it +/// starts carries this socket, and the SQLite engine stops at the failpoint +/// below instead of reporting an unknown outcome. Ordinary callers set +/// neither variable, so a run carries no failpoint. +const CRASH_GATE_ENV: &str = "LAYERFS_COMPACT_CRASH_GATE"; +/// The stop point the crash harness names; honored only beside the gate. +const CRASH_FAILPOINT_ENV: &str = "LAYERFS_COMPACT_CRASH_FAILPOINT"; + +/// The failpoint the crash harness named, or `None` in every ordinary run. +/// +/// A failpoint alone would stop the run in-process; the gate makes the stop +/// observable to the harness, which then `SIGKILL`s this process. Requiring +/// both keeps the hook inert for production callers and for builds without +/// the engine's Unix-socket gate. +#[cfg(unix)] +fn crash_failpoint() -> Option { + std::env::var_os(CRASH_GATE_ENV)?; + match std::env::var(CRASH_FAILPOINT_ENV).ok()?.as_str() { + "BeforeSeal" => Some(CompactFailpoint::BeforeSeal), + "AfterSeal" => Some(CompactFailpoint::AfterSeal), + "BeforeLoad" => Some(CompactFailpoint::BeforeLoad), + "AfterBuild" => Some(CompactFailpoint::AfterBuild), + "DuringVerify" => Some(CompactFailpoint::DuringVerify), + "BeforeInstall" => Some(CompactFailpoint::BeforeInstall), + "DuringRename" => Some(CompactFailpoint::DuringRename), + "AfterInstall" => Some(CompactFailpoint::AfterInstall), + "DuringSettle" => Some(CompactFailpoint::DuringSettle), + _ => None, + } +} + +/// No gate, no failpoint: a stop would hang the run in-process. +#[cfg(not(unix))] +fn crash_failpoint() -> Option { + None +} + +/// `store.compact(name)`: reclaim one Store's dead bytes, answering +/// the §11 report. +pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { + let home = home.to_path_buf(); + // A live operation is `Busy`, a dead owner's record `Unresolved`; the + // refusal is a bounded metric label and never changes what is returned. + let (_lease, record) = match lease::claim(&home, store) { + Ok(claimed) => claimed, + Err(refusal) => { + note_refusal(&refusal); + return Err(refusal); + } + }; + let store_ref = record.store_ref()?.clone(); + if matches!(store_ref, StoreRef::Postgres { .. }) { + return compact_postgresql(&record, &home, store); + } + match place::plan(&home, &store_ref)? { + Executor::Local => {} + Executor::Worker(_) => { + return Err(refused( + "Unsupported", + format!( + "store.compact of Store {store} is local-only: run it on that Store's machine (S6 deviation #42 D5; the worker has no compaction route)" + ), + )) + } + } + let path = paths::store_file(&home, store); + // A cached open in this process must end first. A call still running on + // it keeps the Store open and is `Busy` in the GC vocabulary (§10 keeps + // `InUse` for non-GC calls), naming the Store with no phase or op: no + // operation holds it, a call does. + if let Err(refusal) = close(&path) { + let refusal = gc_refusal(store, refusal); + note_refusal(&refusal); + return Err(refusal); + } + let args = OpenArgs { + home: Some(home), + store: Some(store.to_owned()), + project: None, + store_path: Some(path.display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let identity = identity::identity(&args)?; + let handles = SqliteHandles::open_writable( + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable), + &identity.binding, + identity.cursor_key, + ) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let op_id = admission::op_id(); + let (owner_runtime, owner_boot) = admission::owner_tokens(&format!( + "compact:{}", + layerfs_bridge::sdk_values::hex(&op_id) + )); + let receipt = handles + .compact(&CompactRequest { + op_id, + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + owner_runtime: Some(owner_runtime), + owner_boot: Some(owner_boot), + owner_live: Some(layerfs_home::process::recorded_owner_alive), + failpoint: crash_failpoint(), + ..CompactRequest::default() + }) + .map_err(|error| { + lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) + })?; + // One serializer for the shared §11 report: the SQLite receipt projects + // into `CompactReport`, and T9's PostgreSQL receipt will do the same. + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// The §8 PostgreSQL path: dial the Store's recorded URL, open its schema as +/// this home's authority, and run Phase M/A/B/C. The Store is usable at +/// G + 1 from the swap on; a lost reply is judged by the next call (§9). +/// +/// The PostgreSQL engine is always built beside SQLite (#726). +fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome { + let url = crate::record::url(record)?; + let id = identity::identity(&OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + store_record: Some(record.to_record().clone()), + ..OpenArgs::default() + })?; + let Kind::Postgresql { + schema, + synchronous_commit, + } = &id.kind + else { + return Err(refused( + "Failed", + format!("Store {store} is not a PostgreSQL Store"), + )); + }; + let selection = postgresql::open::selection(&url, schema, synchronous_commit); + let handles = PostgresHandles::open_writable(&selection, &id.binding, id.cursor_key) + .map_err(open_refusal)?; + let (owner_pid_start, owner_pid_boot) = match layerfs_home::process::me() { + Some((start, boot)) => (Some(start), Some(boot)), + None => (None, None), + }; + let op_id = admission::op_id(); + let (owner_runtime, owner_boot) = admission::owner_tokens(&format!( + "compact:{}", + layerfs_bridge::sdk_values::hex(&op_id) + )); + let receipt = handles + .compact(&CompactRequest { + op_id, + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + owner_runtime: Some(owner_runtime), + owner_boot: Some(owner_boot), + ..CompactRequest::default() + }) + .map_err(|error| { + lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) + })?; + Ok(crate::report::compact_report(&CompactReport::from(receipt))) +} + +/// An open refusal before the run: `Busy` is definite, everything else a +/// failure with its own text. +fn open_refusal(error: layerfs_storage::port::PersistenceError) -> SdkRefusal { + use layerfs_storage::port::PersistenceError; + let refusal = match error { + PersistenceError::Busy => refused("Busy", "the Store is held by another writer"), + PersistenceError::Refused { status } => refused( + "Busy", + format!("the Store refused the compaction: {status}"), + ), + other => failed(other), + }; + note_refusal(&refusal); + refusal +} + +/// Counts a Busy/Unresolved refusal of a call that never reached a run; the +/// measured SQLite run records the refusals it produces itself. +fn note_refusal(refusal: &SdkRefusal) { + match refusal.code.as_str() { + "Busy" => compact_metrics::record_refusal(CompactRefusal::Busy), + "Unresolved" => compact_metrics::record_refusal(CompactRefusal::Unresolved), + _ => {} + } +} + +/// Maps the open cache's refusal into the GC vocabulary (§10): a call still +/// running on the Store in this process keeps it open and is `Busy`, naming +/// the Store as the entity with a null `phase` and `op` because no operation +/// holds it; everything else is returned unchanged. The cache's own `InUse` +/// stays for the non-GC calls (`store.push`, `store.pull`, `store.remove`, +/// `store.disconnect`, `backup.restore`) that close it. +#[doc(hidden)] +pub fn gc_refusal(store: &str, refusal: SdkRefusal) -> SdkRefusal { + if refusal.code != "InUse" { + return refusal; + } + let mut busy = refused( + "Busy", + format!( + "Store {store} is busy: a call is running on it in this process; retry \ + `store.compact` after that call ends" + ), + ); + busy.detail = SdkValue::map([ + ( + "entity", + SdkValue::map([ + ("kind", SdkValue::text("store")), + ("name", SdkValue::text(store)), + ]), + ), + ("phase", SdkValue::Null), + ("op", SdkValue::Null), + ]); + busy +} diff --git a/control/crates/layerfs-store/src/identity.rs b/control/crates/layerfs-store/src/identity.rs index a6d233fa3a..90ead8fad6 100644 --- a/control/crates/layerfs-store/src/identity.rs +++ b/control/crates/layerfs-store/src/identity.rs @@ -18,6 +18,7 @@ use layerfs_history::{HistoryCatalogConfig, HistoryName}; use layerfs_home::outcome::{failed, refused, Outcome}; use layerfs_persistence::{PersistenceConfig, SealedStore, SqlitePersistenceProfile}; use layerfs_project::store::{CreateRequest, ProjectApi}; +use layerfs_store_service::history::lookup; use std::path::{Path, PathBuf}; type Result = std::result::Result; @@ -284,7 +285,7 @@ pub fn list_projects(open: &OpenArgs) -> Outcome { let stacks = service .store .published(layerfs_project::store::projects) - .map_err(failed)?; + .map_err(lookup::published)?; Ok(SdkValue::List( stacks .into_iter() diff --git a/control/crates/layerfs-store/src/lib.rs b/control/crates/layerfs-store/src/lib.rs index 97fdd79303..52558074dd 100644 --- a/control/crates/layerfs-store/src/lib.rs +++ b/control/crates/layerfs-store/src/lib.rs @@ -14,6 +14,7 @@ pub mod backup; pub mod call; #[doc(hidden)] pub mod calls; +pub mod compact; pub mod diagnostic; #[doc(hidden)] pub mod identity; @@ -31,6 +32,7 @@ pub mod project; #[doc(hidden)] pub mod record; mod related; +pub mod report; #[doc(hidden)] pub mod resolve; #[doc(hidden)] @@ -38,6 +40,8 @@ pub mod service; pub mod sqlite; pub mod stats; mod status; +#[doc(hidden)] +pub mod unknown; /// The Store connection reaper (not lifecycle, #535 D11), for its test. #[doc(hidden)] diff --git a/control/crates/layerfs-store/src/postgresql/connect.rs b/control/crates/layerfs-store/src/postgresql/connect.rs index 67228c5731..e0b3929a33 100644 --- a/control/crates/layerfs-store/src/postgresql/connect.rs +++ b/control/crates/layerfs-store/src/postgresql/connect.rs @@ -313,3 +313,19 @@ pub fn status(record: &StoreRecord) -> SdkValue { crate::postgresql::open::status(&url, &[SCHEMA.to_owned()]) .unwrap_or_else(|refusal| SdkValue::map([("unavailable", SdkValue::Text(refusal.message))])) } + +/// The Store's compaction operation as `store.status` (T10.5) renders it: +/// `{id, name, phase, started, uncertain, state, phases}`, or null when the +/// Store has none. One status query reads the Store-side `gc_op` and the +/// compaction `gc_intent`; no Store session is opened, no schema is changed +/// and no artifact or scratch schema is read. An unreachable server or a +/// query failure is null here: the `server` field already reports reach. +pub fn compact_status(record: &StoreRecord) -> SdkValue { + let Ok(url) = url(record) else { + return SdkValue::Null; + }; + match layerfs_persistence::postgresql_compact_status(&url, SCHEMA) { + Ok(Some(op)) => crate::report::compact_status(&op.status()), + _ => SdkValue::Null, + } +} diff --git a/control/crates/layerfs-store/src/project/add.rs b/control/crates/layerfs-store/src/project/add.rs index af7e24516f..31594a5c40 100644 --- a/control/crates/layerfs-store/src/project/add.rs +++ b/control/crates/layerfs-store/src/project/add.rs @@ -7,6 +7,8 @@ use crate::project::discovery::Project; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::{ProviderKind, StoreManifest, StoreProfile}; use layerfs_bridge::sdk_values::hex; +use layerfs_history::error::{HolderKind, HolderState, NameHolder, NameSource, NameTakenState}; +use layerfs_history::records::GcScopeKind; use layerfs_history::{BranchId, HistoryName, LayerStackId}; use layerfs_home::helpers::random; use layerfs_home::home::paths; @@ -15,6 +17,7 @@ use layerfs_home::outcome::{failed, refused, Outcome}; use layerfs_persistence::{PersistenceConfig, SealedStore, SqlitePersistenceProfile}; use layerfs_project::store::{AddRequest, InitError, ProjectApi}; use layerfs_project::{InitCounters, ProjectError}; +use layerfs_store_service::history::lookup; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -75,7 +78,7 @@ pub fn make_entry(home: &Path, store: &str, name: &str) -> Result { let entry = layerfs_home::home::paths::project_dir(home, store, name); std::fs::create_dir(&entry).map_err(|error| { if error.kind() == std::io::ErrorKind::AlreadyExists { - refused("Exists", format!("project {name} exists in Store {store}")) + home_name_taken(store, name) } else { failed(error) } @@ -85,6 +88,20 @@ pub fn make_entry(home: &Path, store: &str, name: &str) -> Result { Ok(entry) } +/// A project name the caller's home already records: `NameTaken` with its +/// `home` source, decided before the Store is asked anything. +pub(crate) fn home_name_taken(store: &str, name: &str) -> SdkRefusal { + let state = NameTakenState::new( + NameHolder::new(HolderKind::Project, name, HolderState::Live), + NameSource::Home, + ); + lookup::refuse( + "NameTaken", + format!("project {name} exists in Store {store}"), + lookup::name_taken_detail(&state), + ) +} + /// `project.init`'s add of folder `source` as the new project `name` of /// Store `store` in `home` (the caller's, or a worker's root), whose file is /// `/store//store.sqlite`. @@ -143,10 +160,7 @@ pub fn add_project(request: &ProjectAdd) -> Outcome { } let home = discovery::home_of(Some(&request.home))?; if name == store && legacy(&home, store)? { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(home_name_taken(store, name)); } let entry = make_entry(&home, store, name)?; let _guard = discovery::guard(&entry)?; @@ -213,9 +227,10 @@ pub fn add_project(request: &ProjectAdd) -> Outcome { /// diagnosis. pub fn add_failure(error: InitError, entry: Option<&Path>, name: &str, store: &str) -> SdkRefusal { let refusal = match &error { - InitError::Exists => refused( - "Exists", + InitError::NameTaken(state) => lookup::refuse( + "NameTaken", format!("Store {store} already has a project named {name}"), + lookup::name_taken_detail(state), ), InitError::NoAcquisition => refused( "Unsupported", @@ -232,6 +247,41 @@ pub fn add_failure(error: InitError, entry: Option<&Path>, name: &str, store: &s "InvalidInput", format!("adding {name} refused {}: {reason}", path.display()), ), + // A history refusal from Init is the call's own: `Busy`/ + // `MountsActive`/`Unresolved` keeps its code and detail, and an + // `Unknown` keeps spec §10 A6's entity, phase and operation instead + // of becoming `Failed` with a null detail (F43R4-2). The entry may + // lie behind a project the Store already holds, so it stays for the + // retry or for diagnosis. + InitError::Project(ProjectError::History(error)) | InitError::History(error) => { + // A bare `UnknownOutcome` from a history call that did not attach + // its context still knows the call: name the project and Init + // phase, with no durable operation (F43R5-1). + return lookup::history_in( + error.clone(), + GcScopeKind::Project, + name, + Some("init"), + None, + ); + } + // The acquisition retains an unknown outcome as + // `ProjectError::Uncertain { cause, .. }` (`backing.settle`). A + // history cause is still the call's own: `lookup::history_in` keeps + // spec §10 A6's entity, phase and operation (F43R6-1), while a + // definite history cause keeps its code. + InitError::Project(ProjectError::Uncertain { cause, .. }) => match cause.as_ref() { + ProjectError::History(error) => { + return lookup::history_in( + error.clone(), + GcScopeKind::Project, + name, + Some("init"), + None, + ); + } + _ => return failed(format!("adding project {name} to Store {store}: {error:?}")), + }, _ => return failed(format!("adding project {name} to Store {store}: {error:?}")), }; if let Some(entry) = entry { diff --git a/control/crates/layerfs-store/src/project/delete.rs b/control/crates/layerfs-store/src/project/delete.rs new file mode 100644 index 0000000000..096c995c4e --- /dev/null +++ b/control/crates/layerfs-store/src/project/delete.rs @@ -0,0 +1,996 @@ +//! `project.delete`: a project is retired in its Store and its home entry is +//! removed (spec §4, §4.2, §5.1 and §10). +//! +//! The marker is the commit point. The caller first saves the legacy +//! own-project identity, then writes the durable intent +//! `store//delete/.json` with +//! `{store, project, stack_id, deleted_ms, attempt_ms, state, op_id?}`, then marks the +//! Stack, then removes only its own home entry and unlinks the intent. A +//! retry reads the intent by the recorded Stack id, reuses its `deleted_ms` +//! and finishes the entry: `Marked` means this attempt marked a live Stack, +//! `Retired` means the marker was already there (including a hidden Store +//! project whose home entry is stale), and `Absent` means no live Stack is +//! there, so a retry finishing its sidecar removes the stale entry. A fresh +//! call with no sidecar that meets `Retired` or `Absent` is `NoSuchProject` +//! and keeps its entry. A lost worker reply is `Unknown`; the +//! intent stays and the retry classifies the Stack again. A home record or +//! listing whose Stack differs from the intent is a project made again under +//! the same name: the stale intent is discarded and the call is +//! `NoSuchProject`, so the re-created project and its entry are left alone +//! (spec §4). +//! +//! The worker op is `project.delete`; its frame is +//! `{store, stack_id, deleted_ms?, project?, op_id?, pending_retry?, +//! store_record?}`, and `project.delete.entry` removes only the worker's own +//! entry for a `marked` sidecar (frame `{store, stack_id, project?}`). The two +//! optional context fields are the ones the retired +//! `project.remove` call sent: the worker opens the Store under its own root +//! with the caller's `store_record`, and the project name lets its reply and +//! its own home entry name the project. The worker saves the legacy +//! own-project identity, writes the marker, removes its own entry and replies +//! `{kind, project, stack_id, deleted_ms, retired?}`; the caller removes its +//! own entry and echoes `home_entry_removed`. +//! +//! A caller retrying a sidecar it wrote *before* its admission (no marker +//! confirmed yet) that finds the row still live and no marker only settles: +//! it redoes the marker with the sidecar's `deleted_ms`, marks the sidecar +//! and keeps the caller and worker entries. The next call finds the marker +//! confirmed and finishes the entries. The normal delete and a retry whose +//! marker was already there remove the entries in the same call. +//! +//! A sidecar in `marked` is past the marker: only the entries remain. Its +//! retry takes no admission and writes no marker; it removes the worker's own +//! entry (through `project.delete.entry` when the Store runs there) and then +//! its home entry, and records `done` (spec §5.1). A `done` sidecar likewise +//! needs no Store call and only finishes its entry. +//! +//! The sidecar is written with its admission identity *before* the home path +//! admits the project scope (T4.6, `project.delete.admit` on the worker +//! route): a crash between the registered row and its reply then leaves the +//! identity on disk, so the retry re-registers the same row and settles it. +//! The admission is the same `gc_register` the Store-side removals rely on. A +//! live mount refuses `MountsActive` with its holders and a live operation +//! `Busy`; the pending sidecar stays for the retry and no marker is written. +//! The marker reuses the admission's operation identity, and the caller +//! releases it on every exit path (`project.delete.release`), so a refused or +//! failed call leaves no admission row behind. +use crate::open_args::OpenArgs; +use crate::place::{self, Executor, Worker}; +use crate::project::frame::{ + DeleteAdmitFrame, DeleteEntryFrame, DeleteEntryReply, DeleteFrame, DeleteReleaseFrame, + DeleteReply, +}; +use crate::project::record::ProjectRecord; +use crate::record::{StoreRecord, StoreRef}; +use crate::{identity, lease, service}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::{GcScopeKind, HistoryCatalog, HistoryError, LayerStackId}; +use layerfs_home::events::Events; +use layerfs_home::home::{now_ms, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_home::outcome::{failed, refused, Outcome}; +use layerfs_store_service::history::admission as store_admission; +use layerfs_store_service::history::deletion::{delete_stack_with, Deletion}; +use layerfs_store_service::history::lookup; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +mod remote; +pub use remote::listed_unknown; +use remote::{listed_stack, open_args, Remote}; + +type Result = std::result::Result; + +/// The state the durable intent records (`pending`, `marked`, `done`). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IntentState { + /// The identity is saved and the intent is durable; the marker may or may + /// not have been written (a lost reply leaves this state). + Pending, + /// The marker is committed. The home entry may still be there. + Marked, + /// The home entry was removed. The intent is unlinked next; a crash + /// between the two leaves this state for the retry to settle. + Done, +} + +impl IntentState { + /// The state's file and frame text. + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Marked => "marked", + Self::Done => "done", + } + } + + /// The state `text` names, if any. + pub fn parse(text: &str) -> Option { + match text { + "pending" => Some(Self::Pending), + "marked" => Some(Self::Marked), + "done" => Some(Self::Done), + _ => None, + } + } +} + +/// The durable delete intent at `store//delete/.json`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Intent { + /// The Store the project was in, as sent. + pub store: String, + /// The project's name in it, as sent. + pub project: String, + /// The LayerStack the home record held when the intent was written; a + /// retry resolves the Store by this id, never by name. + pub stack: LayerStackId, + /// The `deleted_ms` the marker holds, or is asked to hold. A retry reuses + /// this value; a `Retired` marker may replace it with its stored value. + pub deleted_ms: i64, + /// When the last attempt started, in milliseconds. + pub attempt_ms: i64, + /// Where the attempt got to. + pub state: IntentState, + /// The project-scope admission this attempt took, when it took one. A + /// retry re-registers the same identity, so a call whose reply was lost + /// settles its own row instead of meeting it as another operation. + pub op: Option<[u8; 16]>, +} + +static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0); + +impl Intent { + /// The intent's path in `home`. + pub fn path(home: &Path, store: &str, project: &str) -> PathBuf { + paths::store_dir(home, store) + .join("delete") + .join(format!("{project}.json")) + } + + /// The intent of project `project` of Store `store`, or `None`. + pub fn read(home: &Path, store: &str, project: &str) -> Result> { + let path = Self::path(home, store, project); + let Some(value) = layerfs_home::home::read_record(&path)? else { + return Ok(None); + }; + let malformed = || { + refused( + "InvalidInput", + format!("project.delete intent {} is malformed", path.display()), + ) + }; + let store = value.get_text("store").ok_or_else(malformed)?.to_owned(); + let project = value.get_text("project").ok_or_else(malformed)?.to_owned(); + let stack = value + .get_text("stack_id") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()) + .ok_or_else(malformed)?; + let deleted_ms = value.get_int("deleted_ms").ok_or_else(malformed)?; + let attempt_ms = value.get_int("attempt_ms").ok_or_else(malformed)?; + let state = value + .get_text("state") + .and_then(IntentState::parse) + .ok_or_else(malformed)?; + let op = match value.get_text("op_id") { + Some(text) => { + let bytes = unhex(text).ok_or_else(malformed)?; + let body: [u8; 16] = bytes.as_slice().try_into().map_err(|_| malformed())?; + Some(body) + } + None => None, + }; + Ok(Some(Self { + store, + project, + stack, + deleted_ms, + attempt_ms, + state, + op, + })) + } + + /// Writes the intent whole: a private temporary file, fsync, one rename, + /// then the directory's fsync. A reader sees the old intent or the new + /// one, never a part of either. + pub fn write(home: &Path, intent: &Self) -> Result<()> { + let path = Self::path(home, &intent.store, &intent.project); + let parent = path + .parent() + .ok_or_else(|| refused("Failed", "a delete intent needs a folder"))?; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(parent) + .map_err(failed)?; + let unique = NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed); + let temporary = parent.join(format!( + ".{}.{}.{}.tmp", + intent.project, + std::process::id(), + unique + )); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("store".to_owned(), SdkValue::text(&intent.store)), + ("project".to_owned(), SdkValue::text(&intent.project)), + ( + "stack_id".to_owned(), + SdkValue::text(hex(&intent.stack.to_bytes())), + ), + ("deleted_ms".to_owned(), SdkValue::Int(intent.deleted_ms)), + ("attempt_ms".to_owned(), SdkValue::Int(intent.attempt_ms)), + ("state".to_owned(), SdkValue::text(intent.state.as_str())), + ]; + if let Some(op) = intent.op { + fields.push(("op_id".to_owned(), SdkValue::text(hex(&op)))); + } + let value = SdkValue::Map(fields); + let mut text = layerfs_home::json::print(&value); + text.push('\n'); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(failed)?; + let written = file + .write_all(text.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(failed); + drop(file); + let renamed = written.and_then(|()| fs::rename(&temporary, &path).map_err(failed)); + if let Err(refusal) = renamed { + let _ = fs::remove_file(&temporary); + return Err(refusal); + } + // the directory entry is durable too; a filesystem that cannot + // fsync a directory has already made the rename durable + let _ = File::open(parent).and_then(|folder| folder.sync_all()); + Ok(()) + } + + /// Removes the intent, if it is there. + pub fn unlink(home: &Path, store: &str, project: &str) -> Result<()> { + let path = Self::path(home, store, project); + match fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + if let Some(parent) = path.parent() { + let _ = File::open(parent).and_then(|folder| folder.sync_all()); + } + Ok(()) + } + + /// Drops an intent that no longer describes the home record (a project + /// made again under the same name). + pub fn discard(home: &Path, store: &str, project: &str) -> Result<()> { + Self::unlink(home, store, project) + } +} + +/// The project-scope admission `project.delete` takes before its sidecar and +/// marker: the admission row's operation identity, reused by the marker so the +/// one row is excluded from its own scan and released once. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Admission { + /// The `gc_intent` operation identity. + pub op: [u8; 16], + /// The project scope it holds. + pub stack: LayerStackId, +} + +impl Admission { + /// One admission of `stack` under operation `op`. + pub const fn new(stack: LayerStackId, op: [u8; 16]) -> Self { + Self { op, stack } + } +} + +/// The Store side of one `project.delete` attempt and the caller's entry +/// removal, as one seam so the home path is testable without a Store. +pub trait Effects { + /// Saves the legacy own-project identity before the intent and the + /// marker; idempotent on a retry. + fn prepare(&mut self) -> Result<()>; + + /// Takes the project-scope admission under `op`, the identity the sidecar + /// recorded before this call: a live mount is `MountsActive` with its + /// holders, a live operation `Busy`, an uncertain owner `Unresolved`. A + /// refusal registers nothing under a new identity, and a retry + /// re-registers `op`'s own row instead of meeting it as another + /// operation. `project` names the row, so a later caller meeting it + /// reports `{kind: "project", name}` rather than the hex LayerStack id. + fn admit(&mut self, stack: LayerStackId, op: [u8; 16], project: &str) -> Result; + + /// Releases `admission` on every exit path: success, refusal and an + /// unknown Store outcome. A provider without admission releases nothing. + fn release(&mut self, admission: Admission) -> Result<()>; + + /// Marks (or reclassifies) `admission`'s Stack, reusing the admission's + /// operation identity. A retry passes the intent's `deleted_ms`. + /// `pending_retry` is true when the caller is retrying a sidecar written + /// before it admitted (no marker confirmed yet): the worker route then + /// keeps its own entry, as the caller keeps its home entry. + fn mark( + &mut self, + admission: Admission, + deleted_ms: i64, + pending_retry: bool, + ) -> Result; + + /// Removes the Store-side home entry a `marked` sidecar still names: the + /// worker's own entry when the Store runs there, nothing on the local + /// route, where that entry is the caller's home entry. The marker is + /// already durable, so this takes no admission and writes no marker + /// (spec §5.1). + fn settle(&mut self, _stack: LayerStackId) -> Result<()> { + Ok(()) + } + + /// Removes the caller's home entry; `Ok(())` when nothing remains. + fn remove_entry(&mut self, entry: &Path) -> Result<()>; +} + +/// The `project` part of a Stack record from a home record: the `manifest`'s +/// Stack for a local sqlite record, the record's `stack` for a PostgreSQL or +/// worker record. +fn stack_of_value(value: &SdkValue) -> Option { + if let Some(SdkValue::Bytes(bytes)) = value.get("manifest") { + if let Ok(manifest) = StoreManifest::decode(bytes) { + if let Ok(stack) = LayerStackId::from_slice(&manifest.stack) { + return Some(stack); + } + } + } + value + .get_text("stack") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()) +} + +/// The Stack a home record names, if it names one. +fn stack_of(record: &ProjectRecord) -> Option { + stack_of_value(record.to_record()) +} + +/// `{kind: "Removed", project, deleted_ms, home_entry_removed}` (spec §10; +/// `home_entry_removed: false` is still success, the marker being the commit +/// point). +fn removed(store: &str, project: &str, deleted_ms: i64, home_entry_removed: bool) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Removed")), + ("project", SdkValue::text(format!("{store}/{project}"))), + ("deleted_ms", SdkValue::Int(deleted_ms)), + ("home_entry_removed", SdkValue::Bool(home_entry_removed)), + ]) +} + +/// The side effects and the project-scope admission one finishing attempt +/// runs against: together, so `mark_and_finish` stays a small call. +struct Finish<'a> { + effects: &'a mut dyn Effects, + admission: Admission, + /// Whether the call had no durable intent: a fresh call may not finish a + /// retired project, while a retry resolves itself through its sidecar. + fresh: bool, +} + +/// The mutating part of `delete_with` after the sidecar is durable: the marker +/// and the entry removal (spec §4). The sidecar names the admission, so the +/// retry that redoes this marker reuses the same operation identity. +fn mark_and_finish( + home: &Path, + store: &str, + project: &str, + mut current: Intent, + mut deleted_ms: i64, + pending_retry: bool, + finish: Finish<'_>, +) -> Outcome { + let Finish { + effects, + admission, + fresh, + } = finish; + let deletion = effects.mark(admission, deleted_ms, pending_retry)?; + if fresh && !matches!(deletion, Deletion::Marked { .. }) { + // The call had no sidecar, so it had to find a live project, and the + // Store says this Stack is already retired (or was never there). A + // stale entry is not a project: `NoSuchProject`, with the sidecar + // this call wrote removed and the entry left in place (spec §4). + Intent::unlink(home, store, project)?; + return Err(refused( + "NoSuchProject", + format!("no project {store}/{project}"), + )); + } + deleted_ms = deletion.deleted_ms(deleted_ms); + current.deleted_ms = deleted_ms; + current.state = IntentState::Marked; + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + + // A retry of a sidecar written before the admission, whose marker was + // just written over a live row and no marker existed yet, settles only: + // the marker is now confirmed and the sidecar is `marked`, so the next + // call finishes the entry. The home and worker entries stay for that + // call; the normal delete path and a retry whose marker was already + // there (§ `Retired`) remove the entry now. + if pending_retry && matches!(deletion, Deletion::Marked { .. }) { + return Ok(removed(store, project, deleted_ms, false)); + } + + let home_entry_removed = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + if home_entry_removed { + current.state = IntentState::Done; + Intent::write(home, ¤t)?; + Intent::unlink(home, store, project)?; + } + Ok(removed(store, project, deleted_ms, home_entry_removed)) +} + +/// Runs `project.delete` for a home entry whose Store side is `effects`. +/// +/// `stack` is the Stack id the Store lists for the project, or the record's +/// id on a worker route, where the caller cannot list. A fresh call without +/// one is `NoSuchProject`: only a same-home retry, through its durable +/// intent, resolves a retired Stack (spec §4, §4.2). A listing or record that +/// names a different Stack than the sidecar is a re-created project: the +/// sidecar is discarded and the call is `NoSuchProject`, so that project's +/// entry is never touched. The same function serves the local and worker +/// routes, and a fake in tests. +/// +/// The identity save and then the sidecar, naming the admission, precede the +/// project scope's admission and the marker: a live mount is `MountsActive` +/// with its holders and a live operation `Busy`, reported with the pending +/// sidecar kept and no marker. The admission is released on every exit path. +pub fn delete_with( + home: &Path, + store: &str, + project: &str, + stack: Option, + effects: &mut impl Effects, +) -> Outcome { + let record = ProjectRecord::read(home, store, project)?; + let record_stack = record.as_ref().and_then(stack_of); + let intent = Intent::read(home, store, project)?; + + // The Stack the name resolves to now: the caller's listing first, then + // the record. A different Stack id for the same name is a project made + // again: the stale intent must not touch its entry, so it is discarded + // and the call is `NoSuchProject` (spec §4). The listing is checked too: + // a re-created project whose home record is missing or names no Stack + // must not be deleted by an older intent. + let current_stack = stack.or(record_stack); + if let (Some(stale), Some(current)) = (&intent, current_stack) { + if stale.stack != current { + Intent::discard(home, store, project)?; + return Err(refused( + "NoSuchProject", + format!("no project {store}/{project}"), + )); + } + } + + // Whether this call had a durable intent to fall back on. A fresh call + // must find a live project; a retry finishes a retired one (spec §4). + let fresh = intent.is_none(); + + // A sidecar read in `pending` was written before an earlier admission: a + // retry of it redoes the marker but does not finish the entry. + let pending_retry = intent + .as_ref() + .is_some_and(|intent| intent.state == IntentState::Pending); + + let (stack, deleted_ms) = match &intent { + Some(intent) => (intent.stack, intent.deleted_ms), + None => { + // A fresh call resolves the project only through the Stack its + // Store lists now, never through the home record: a record from + // another home may name a retired Stack, and the record's id is + // not evidence of a live project (spec §4, §4.2). The exception + // is a same-home retry, which has the durable intent read above. + let Some(stack) = stack else { + return Err(refused( + "NoSuchProject", + format!("no project {store}/{project}"), + )); + }; + (stack, now_ms()) + } + }; + + if let Some(settled) = &intent { + if settled.state == IntentState::Marked { + // The marker is durable and only the entries remain: no + // admission and no second marker (spec §5.1). The worker's own + // entry goes first, then the caller's home entry; a failed home + // removal leaves the sidecar `marked` for the next retry. + effects.prepare()?; + effects.settle(stack)?; + let removed_entry = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + if removed_entry { + let mut done = settled.clone(); + done.state = IntentState::Done; + done.attempt_ms = now_ms(); + Intent::write(home, &done)?; + Intent::unlink(home, store, project)?; + } + return Ok(removed(store, project, deleted_ms, removed_entry)); + } + if settled.state == IntentState::Done { + // The done sidecar needs no Store call: the marker is already + // there and only the home entry is removed. + effects.prepare()?; + let removed_entry = effects + .remove_entry(&paths::project_dir(home, store, project)) + .is_ok(); + let mut done = settled.clone(); + done.attempt_ms = now_ms(); + let _ = Intent::write(home, &done); + let _ = Intent::unlink(home, store, project); + return Ok(removed(store, project, deleted_ms, removed_entry)); + } + } + + // The identity save, then the sidecar with its operation identity, then + // the admission, then the marker. Writing the sidecar first means a crash + // between the registered row and its reply leaves the identity on disk + // for the retry; a refusal keeps the pending sidecar and writes no + // marker. + effects.prepare()?; + let mut current = match intent { + Some(intent) => intent, + None => Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms, + attempt_ms: now_ms(), + state: IntentState::Pending, + op: None, + }, + }; + let op = current.op.unwrap_or_else(store_admission::op_id); + current.op = Some(op); + if current.state == IntentState::Pending { + current.attempt_ms = now_ms(); + Intent::write(home, ¤t)?; + } + let admission = match effects.admit(stack, op, project) { + Ok(admission) => admission, + Err(refusal) => { + let _ = effects.release(Admission::new(stack, op)); + return Err(refusal); + } + }; + let attempt = mark_and_finish( + home, + store, + project, + current, + deleted_ms, + pending_retry, + Finish { + effects: &mut *effects, + admission, + fresh, + }, + ); + let released = effects.release(admission); + match attempt { + Ok(outcome) => { + released?; + Ok(outcome) + } + // The original failure is the reported one; a failed release is named + // so a leaked admission row is not silent. + Err(error) => Err(match released { + Ok(()) => error, + Err(cause) => SdkRefusal { + message: format!( + "{}; releasing the project admission: {}", + error.message, cause.message + ), + ..error + }, + }), + } +} + +/// Removes a directory tree, treating an absent entry as removed. +fn remove_tree(entry: &Path) -> Result<()> { + match fs::remove_dir_all(entry) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(failed(format!("removing {}: {error}", entry.display()))), + } +} + +/// The Store-side opens of `project.delete` in this process. +struct Local { + home: PathBuf, + store: String, + project: String, + open: OpenArgs, +} + +impl Effects for Local { + fn prepare(&mut self) -> Result<()> { + if self.project != self.store || !identity::legacy(&self.home, &self.store)? { + return Ok(()); + } + let id = identity::identity(&self.open)?; + layerfs_home::home::private_dir(&identity::store_dir(&self.home, &self.store)?)?; + identity::save_record(&self.home, &self.store, &id) + } + + fn admit(&mut self, stack: LayerStackId, op: [u8; 16], project: &str) -> Result { + let intent = store_admission::project_removal_at(stack, op, Some(project)); + service::with_service(&self.open, |service| { + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(Admission::new(stack, op)), + // A provider without admission cannot exclude a removal; the + // marker's own admission still runs, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(Admission::new(stack, op)), + Err(error) => Err(lookup::history_in( + error, + GcScopeKind::Project, + project, + None, + Some(&op), + )), + } + }) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + let project = self.project.clone(); + service::with_service( + &self.open, + |service| match store_admission::release_or_mark(service.store.history(), &admission.op) + { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(()), + Err(error) => Err(lookup::history_in( + error, + GcScopeKind::Project, + &project, + None, + Some(&admission.op), + )), + }, + ) + } + + fn mark( + &mut self, + admission: Admission, + deleted_ms: i64, + _pending_retry: bool, + ) -> Result { + let intent = store_admission::project_removal_at( + admission.stack, + admission.op, + Some(self.project.as_str()), + ); + service::with_service(&self.open, |service| { + delete_stack_with(&service.store, admission.stack, deleted_ms, &intent) + }) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<()> { + remove_tree(entry) + } +} + +/// The Store-side opens of `project.delete` in a sqlite Store's worker. +/// +/// `project.delete(project)`: retires project `project` in its Store and +/// removes the caller's home entry. The marker is the commit point; a home +/// removal failure is success with `home_entry_removed: false`, and the +/// durable intent lets a same-home retry finish it. +pub fn delete( + home: &Path, + events: &mut Events<'_>, + project: &ProjectRef, + related: crate::related::Related, +) -> Outcome { + let _ = events; + let home = home.to_path_buf(); + let store = project.store.as_str(); + let name = project.project.as_str(); + let record = StoreRecord::read(&home, store)?; + let project_record = ProjectRecord::read(&home, store, name)?; + // §5: a recorded Sandbox is another user of the project, even with + // nothing mounted; refuse before the sidecar and the marker. + if record.is_some() || project_record.is_some() { + (related.can_remove_project)(&home, project.to_string().as_str())?; + } + let stack = project_record.as_ref().and_then(stack_of); + match record { + Some(record) => { + record.unblocked()?; + let store_ref = record.store_ref()?.clone(); + // the caller holds the Store's lease while it sends the call + // there, as every call that runs in a worker does + let _lease = lease::hold(&home, store)?; + match place::plan(&home, &store_ref)? { + Executor::Local => { + let open = open_args(&home, store, name, Some(&record))?; + // Resolve the project by the Stack its Store lists now, + // never by the home record alone: a fresh delete of a + // retired project is `NoSuchProject`, and a stale record + // from another home is not evidence (spec §4.2). A retry + // resolves itself through its sidecar inside + // `delete_with`. + let stack = if project_record.is_some() { + listed_stack(&open, name)? + } else { + None + }; + let mut effects = Local { + home: home.clone(), + store: store.to_owned(), + project: name.to_owned(), + open, + }; + delete_with(&home, store, name, stack, &mut effects) + } + Executor::Worker(worker) => { + if stack.is_none() && project_record.is_some() { + return Err(refused( + "Unsupported", + format!( + "project {store}/{name} has no recorded LayerStack id and its Store is on another machine; \ + list it on that machine first" + ), + )); + } + let mut effects = Remote { + worker, + store: store.to_owned(), + project: name.to_owned(), + store_record: record.to_record().clone(), + }; + delete_with(&home, store, name, stack, &mut effects) + } + } + } + None => { + // A Store made by its own project's Init has no Store record: its + // identity lives in the project entry, and saving it before that + // entry goes is exactly the legacy own-project identity save. + if name != store || project_record.is_none() { + return Err(refused("NoSuchStore", format!("no Store named {store}"))); + } + let open = open_args(&home, store, name, None)?; + let stack = match stack { + Some(stack) => Some(stack), + None => listed_stack(&open, name)?, + }; + let mut effects = Local { + home: home.clone(), + store: store.to_owned(), + project: name.to_owned(), + open, + }; + delete_with(&home, store, name, stack, &mut effects) + } + } +} + +/// The `OpenArgs` a worker's `project.delete` ops open the Store with: the +/// Store file under the worker's own root and the caller's Store record. +fn worker_open( + root: &Path, + store: &str, + project: Option, + store_record: SdkValue, +) -> OpenArgs { + OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + project, + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + } +} + +/// The project name a worker `project.delete*` op records: the caller's, or +/// the Stack record's when the caller sent none, so a later `Busy` or +/// `Unresolved` reports `{kind: "project", name}` rather than the hex +/// LayerStack id. A Stack record that is already gone leaves the id, which is +/// all the caller left to go on. +fn worker_project( + history: &dyn HistoryCatalog, + stack: LayerStackId, + project: Option, +) -> Result> { + if project.is_some() { + return Ok(project); + } + Ok(history + .layer_stack(stack) + .map_err(lookup::history)? + .map(|record| record.name.as_str().to_owned())) +} + +/// The `{kind: "Admitted", stack_id}` answer. +fn admitted(stack: LayerStackId) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Admitted")), + ("stack_id", SdkValue::text(hex(&stack.to_bytes()))), + ]) +} + +/// The `{kind: "Released", op_id}` answer. +fn released(op: [u8; 16]) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Released")), + ("op_id", SdkValue::text(hex(&op))), + ]) +} + +/// The worker's `project.delete.admit`: registers the caller's project-scope +/// admission before the caller writes its sidecar and before the marker, so a +/// live mount refuses with `MountsActive` (its holders reported) and a live +/// operation with `Busy`, with no sidecar written. The registered row names +/// the project from the frame, or from the Stack record when the caller sent +/// no name, so a later `Busy`/`Unresolved` reports `{kind: "project", name}` +/// rather than the hex LayerStack id. +pub fn admit_project(root: &Path, frame: &DeleteAdmitFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.admit needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + service::with_service(&open, |service| { + let project = worker_project(service.store.history(), frame.stack, frame.project.clone())?; + let intent = store_admission::project_removal_at(frame.stack, frame.op, project.as_deref()); + match service.store.history().gc_register(&intent) { + Ok(()) => Ok(admitted(frame.stack)), + // A provider without admission cannot exclude a removal; the + // marker still admits, as it did before T6. + Err(HistoryError::Unsupported(_)) => Ok(admitted(frame.stack)), + Err(error) => { + let entity = project.unwrap_or_else(|| hex(frame.stack.as_slice())); + Err(lookup::history_in( + error, + GcScopeKind::Project, + &entity, + None, + Some(&frame.op), + )) + } + } + }) +} + +/// The worker's `project.delete.release`: deletes the caller's admission row +/// after the marker, on every exit path. +pub fn release_project(root: &Path, frame: &DeleteReleaseFrame) -> Outcome { + let store_record = frame.store_record.clone().ok_or_else(|| { + refused( + "InvalidInput", + "project.delete.release needs the Store's record", + ) + })?; + let open = worker_open(root, frame.store.as_str(), None, store_record); + service::with_service(&open, |service| { + match store_admission::release_or_mark(service.store.history(), &frame.op) { + Ok(()) | Err(HistoryError::Unsupported(_)) => Ok(released(frame.op)), + Err(error) => Err(lookup::history(error)), + } + }) +} + +/// The worker's `project.delete` on a Store under `root`: save the legacy +/// own-project identity, write the marker, remove the worker's own entry and +/// answer `{kind, project, stack_id, deleted_ms}`. The marker reuses the +/// caller's admission operation identity when its frame carries one, so the +/// caller's admitted row is excluded from its own scan; without one the +/// marker takes a fresh identity and its own admission, as before. +pub fn delete_project(root: &Path, frame: &DeleteFrame) -> Outcome { + let store = frame.store.as_str(); + let store_record = frame + .store_record + .clone() + .ok_or_else(|| refused("InvalidInput", "project.delete needs the Store's record"))?; + let open = worker_open(root, store, frame.project.clone(), store_record); + // The legacy own-project entry carries the Store's identity until the + // marker; save it first, idempotently. + let own = frame.project.as_deref() == Some(store) && identity::legacy(root, store)?; + if own { + let id = identity::identity(&open)?; + layerfs_home::home::private_dir(&identity::store_dir(root, store)?)?; + identity::save_record(root, store, &id)?; + } + let op = frame.op_id.unwrap_or_else(store_admission::op_id); + let requested = frame.deleted_ms.unwrap_or_else(now_ms); + let deletion = service::with_service(&open, |service| { + let project = worker_project(service.store.history(), frame.stack, frame.project.clone())?; + let intent = store_admission::project_removal_at(frame.stack, op, project.as_deref()); + delete_stack_with(&service.store, frame.stack, requested, &intent) + })?; + let deleted_ms = deletion.deleted_ms(requested); + let retired = matches!(deletion, Deletion::Retired { .. }); + // A caller retrying a sidecar written before its admission, over a live + // row with no marker, settles only: it keeps its home entry, so the + // worker keeps its own copy for the call that finishes the delete. + let deferred = + frame.pending_retry.unwrap_or(false) && matches!(deletion, Deletion::Marked { .. }); + let project = if deferred { + frame.project.clone().unwrap_or_else(|| store.to_owned()) + } else { + remove_worker_entry(root, store, frame.stack, frame.project.as_deref())? + .or_else(|| frame.project.clone()) + .unwrap_or_else(|| store.to_owned()) + }; + Ok(DeleteReply { + project, + stack: frame.stack, + deleted_ms, + retired: Some(retired), + } + .encode()) +} + +/// The worker's `project.delete.entry`: removes the worker's own home entry +/// for a Stack whose marker is already durable, answering +/// `{kind: "Removed", project, stack_id}`. It opens no Store and writes no +/// marker (spec §5.1). +pub fn remove_project_entry(root: &Path, frame: &DeleteEntryFrame) -> Outcome { + let store = frame.store.as_str(); + let project = remove_worker_entry(root, store, frame.stack, frame.project.as_deref())? + .or_else(|| frame.project.clone()) + .unwrap_or_else(|| store.to_owned()); + Ok(DeleteEntryReply { + project, + stack: frame.stack, + } + .encode()) +} + +/// Removes the worker's own home entry for `stack`; returns the project name +/// it removed, if any. +#[doc(hidden)] +pub fn remove_worker_entry( + root: &Path, + store: &str, + stack: LayerStackId, + hint: Option<&str>, +) -> Result> { + let names = match hint { + Some(name) => vec![name.to_owned()], + None => paths::names(&paths::store_dir(root, store).join("project")).map_err(failed)?, + }; + let mut removed = None; + for name in names { + let record = paths::project_record(root, store, &name); + let Some(value) = layerfs_home::home::read_record(&record)? else { + continue; + }; + let matches = stack_of_value(&value) == Some(stack) || name == store; + if matches { + remove_tree(&paths::project_dir(root, store, &name))?; + removed.get_or_insert(name); + } + } + Ok(removed) +} diff --git a/control/crates/layerfs-store/src/project/delete/remote.rs b/control/crates/layerfs-store/src/project/delete/remote.rs new file mode 100644 index 0000000000..59c3d5694f --- /dev/null +++ b/control/crates/layerfs-store/src/project/delete/remote.rs @@ -0,0 +1,217 @@ +//! The worker-side opens of `project.delete`: the remote effects and the +//! `OpenArgs`/Stack lookups a local Store call needs. + +use super::*; + +pub(super) struct Remote { + pub(super) worker: Worker, + pub(super) store: String, + pub(super) project: String, + pub(super) store_record: SdkValue, +} + +impl Effects for Remote { + fn prepare(&mut self) -> Result<()> { + // the worker saves its own legacy identity before its marker + Ok(()) + } + + fn admit(&mut self, stack: LayerStackId, op: [u8; 16], project: &str) -> Result { + let frame = DeleteAdmitFrame { + store: StoreName::parse(&self.store)?, + stack, + op, + // the row is stored: naming it makes a later `Busy`/`Unresolved` + // report the project rather than the hex LayerStack id + project: Some(project.to_owned()), + store_record: Some(self.store_record.clone()), + }; + let answer = match self + .worker + .call("project.delete.admit", frame.encode(), &mut |_| {}) + { + Ok(answer) => answer, + Err(refusal) => { + // The request went out, so the worker may have registered the + // row before its answer was lost: settle the identity before + // reporting the unknown outcome. + if refusal.code == "Unknown" { + let _ = self.release(Admission::new(stack, op)); + } + return Err(crate::unknown::named_unknown( + refusal, + "project", + &self.project, + Some(&op), + )); + } + }; + if let Err(refusal) = admit_reply(&answer, stack) { + let _ = self.release(Admission::new(stack, op)); + return Err(refusal); + } + Ok(Admission::new(stack, op)) + } + + fn release(&mut self, admission: Admission) -> Result<()> { + let frame = DeleteReleaseFrame { + store: StoreName::parse(&self.store)?, + op: admission.op, + store_record: Some(self.store_record.clone()), + }; + self.worker + .call("project.delete.release", frame.encode(), &mut |_| {}) + .map_err(|refusal| { + crate::unknown::named_unknown( + refusal, + "project", + &self.project, + Some(&admission.op), + ) + })?; + Ok(()) + } + + fn mark( + &mut self, + admission: Admission, + deleted_ms: i64, + pending_retry: bool, + ) -> Result { + let frame = DeleteFrame { + store: StoreName::parse(&self.store)?, + project: Some(self.project.clone()), + stack: admission.stack, + deleted_ms: Some(deleted_ms), + op_id: Some(admission.op), + pending_retry: Some(pending_retry), + store_record: Some(self.store_record.clone()), + }; + let answer = self + .worker + .call("project.delete", frame.encode(), &mut |_| {}) + .map_err(|refusal| { + crate::unknown::named_unknown( + refusal, + "project", + &self.project, + Some(&admission.op), + ) + })?; + let reply = DeleteReply::decode(&answer)?; + if reply.stack != admission.stack { + return Err(refused( + "Failed", + "the worker answered for another LayerStack", + )); + } + Ok(if reply.retired == Some(true) { + Deletion::Retired { + deleted_ms: reply.deleted_ms, + } + } else { + Deletion::Marked { + deleted_ms: reply.deleted_ms, + } + }) + } + + fn settle(&mut self, stack: LayerStackId) -> Result<()> { + let frame = DeleteEntryFrame { + store: StoreName::parse(&self.store)?, + stack, + project: Some(self.project.clone()), + }; + let answer = self + .worker + .call("project.delete.entry", frame.encode(), &mut |_| {}) + .map_err(|refusal| { + // no admission identity is held on this path + crate::unknown::named_unknown(refusal, "project", &self.project, None) + })?; + let reply = DeleteEntryReply::decode(&answer)?; + if reply.stack != stack { + return Err(refused( + "Failed", + "the worker answered for another LayerStack", + )); + } + Ok(()) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<()> { + remove_tree(entry) + } +} + +/// Checks the worker's `project.delete.admit` answer. +pub(super) fn admit_reply(answer: &SdkValue, stack: LayerStackId) -> Result<()> { + if answer.get_text("kind") != Some("Admitted") { + return Err(refused( + "Failed", + "the worker project.delete admission is not `Admitted`", + )); + } + let answered = answer + .get_text("stack_id") + .and_then(unhex) + .and_then(|bytes| LayerStackId::from_slice(&bytes).ok()); + if answered != Some(stack) { + return Err(refused("Failed", "the worker admitted another LayerStack")); + } + Ok(()) +} + +/// The `OpenArgs` a local Store call opens with. `record` is absent for a +/// Store made by its own project Init, whose identity is the project entry. +pub(super) fn open_args( + home: &Path, + store: &str, + project: &str, + record: Option<&StoreRecord>, +) -> Result { + let (store_path, postgresql) = match record { + Some(record) => match record.store_ref()? { + StoreRef::Postgres { .. } => ( + record.postgresql_key(home, store), + Some(crate::postgresql::connect::selection(record)?), + ), + StoreRef::Sqlite { .. } => (paths::store_file(home, store), None), + }, + None => (paths::store_file(home, store), None), + }; + Ok(OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + project: Some(project.to_owned()), + store_path: Some(store_path.display().to_string()), + store_record: record.map(|record| record.to_record().clone()), + postgresql, + }) +} + +/// Names the project in a `project.delete` Store-listing refusal whose detail +/// is empty: `Unknown` from a bare `UnknownOutcome` or a poisoned Store +/// object, rather than `Unknown {}` (F43R5-2). `listed_stack` wraps both +/// callers' failures with this. +pub fn listed_unknown(refusal: SdkRefusal, name: &str) -> SdkRefusal { + crate::unknown::named_unknown(refusal, "project", name, None) +} + +/// The Stack id a Store lists for project `name`, for a record that does +/// not name one (spec §4.2 fills it from the filtered listing by name). +pub(super) fn listed_stack(open: &OpenArgs, name: &str) -> Result> { + service::with_service(open, |service| { + let stacks = service + .store + .published(layerfs_project::store::projects) + .map_err(lookup::published)?; + Ok(stacks + .into_iter() + .find(|stack| stack.name.as_str() == name) + .map(|stack| stack.id)) + }) + // A listing that cannot be determined still knows the project it asked + // about (F43R5-2). + .map_err(|refusal| listed_unknown(refusal, name)) +} diff --git a/control/crates/layerfs-store/src/project/export/walk.rs b/control/crates/layerfs-store/src/project/export/walk.rs index d97f2d4dee..e4d5997b69 100644 --- a/control/crates/layerfs-store/src/project/export/walk.rs +++ b/control/crates/layerfs-store/src/project/export/walk.rs @@ -19,6 +19,7 @@ use layerfs_history::NodeId; use layerfs_home::outcome::refused; use layerfs_store_service::history::{lookup, MAIN}; use layerfs_store_service::read::cached::Cached; +use layerfs_store_service::read::visibility::Visibility; use layerfs_store_service::store::StorePorts; use layerfs_store_service::StoreService; use layerfs_telemetry::timer::Timing; @@ -45,6 +46,9 @@ pub(super) struct PinnedSnapshot { pub(super) node: String, pub(super) root: String, tree: TreeTable, + /// §4.3: the pinned node's marker visibility. Hidden entries are omitted + /// from the exported folder and from its counts. + visibility: Visibility, } /// Resolves `(project, branch)` once into an immutable `(NodeId, ObjectId, @@ -65,12 +69,16 @@ pub(super) fn pin_snapshot( let root = lookup::root(&service.store, node)?; let reader = service.store.ports(stack.scope); let tree = TreeTable::open(reader.as_ref(), FilesystemRootId(root)).map_err(lookup::content)?; + // The pinned node's stack and serial, fail-closed: a marker read error + // must refuse the export rather than export a removed path. + let visibility = Visibility::for_node(&service.store, node)?; Ok(PinnedSnapshot { reader, label, node: node_text(node), root: hex(root.as_bytes()), tree, + visibility, }) } @@ -163,6 +171,11 @@ pub(super) fn stream_snapshot( rel.push(b'/'); } rel.extend_from_slice(name.as_bytes()); + // A hidden entry is omitted; a hidden directory's subtree with it, as + // the read surface omits it and the marker hides everything below it. + if pinned.visibility.hidden_relative(&rel).is_some() { + continue; + } match value.kind { InodeKind::Directory => { counts.directories += 1; diff --git a/control/crates/layerfs-store/src/project/family.rs b/control/crates/layerfs-store/src/project/family.rs index 81f43a1047..87880f1a14 100644 --- a/control/crates/layerfs-store/src/project/family.rs +++ b/control/crates/layerfs-store/src/project/family.rs @@ -5,25 +5,25 @@ //! `native` or a PostgreSQL Store, in the machine's worker for a sqlite //! Store on any other machine (#536 H11). The folder is always on the //! caller's host: for a worker this process copies it to the machine -//! first (H11a). Records in the home list projects; `remove` drops one -//! project's history from its Store. +//! first (H11a). Records in the home list projects. use crate::calls::Page; use crate::lease; -use crate::place::{self, Executor}; +use crate::open_args::OpenArgs; +use crate::place::{self, Executor, Worker}; use crate::project::add::{add_project, ProjectAdd}; -use crate::project::discovery::home_of; -use crate::project::frame::{InitFrame, RemoveFrame}; +use crate::project::frame::{InitFrame, ListingFrame, ListingReply}; use crate::project::postgres_init::PgProjectAdd; use crate::project::record::ProjectRecord; use crate::record::{StoreRecord, StoreRef}; use crate::resolve; +use crate::service; use crate::sqlite::family::progress; -use crate::Related; -use layerfs_bridge::control::SdkValue; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_home::home::paths; use layerfs_home::names::{ProjectRef, StoreName}; use layerfs_home::outcome::{failed, refused, Outcome, View}; +use layerfs_store_service::history::lookup; use std::path::Path; /// Default Init time limit: one hour. @@ -49,6 +49,94 @@ pub struct ProjectInit { pub timeout: Option, } +/// One project a Store's filtered listing names: its name and LayerStack. +struct LiveProject { + name: String, + stack: Vec, +} + +/// The live projects Store `store` lists, after the retired/hidden ones are +/// filtered out (spec §4.2). Every route reads the listing where the +/// Store's calls run (T4.5c): in this process for a sqlite Store on this +/// machine and for a PostgreSQL Store, in the machine's worker for a sqlite +/// Store on another machine. A Store that cannot be listed is `Err`. +fn live_projects( + home: &Path, + store: &str, + record: &StoreRecord, +) -> std::result::Result, SdkRefusal> { + let store_ref = record.store_ref()?; + record.unblocked()?; + if let Executor::Worker(mut worker) = place::plan(home, store_ref)? { + return live_projects_on_worker(&mut worker, store, record); + } + let (store_path, postgresql) = match store_ref { + StoreRef::Postgres { .. } => ( + record.postgresql_key(home, store), + Some(crate::postgresql::connect::selection(record)?), + ), + StoreRef::Sqlite { .. } => (paths::store_file(home, store), None), + }; + let open = OpenArgs { + home: Some(home.to_path_buf()), + store: Some(store.to_owned()), + store_path: Some(store_path.display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql, + ..OpenArgs::default() + }; + let stacks = service::with_service(&open, |service| { + service + .store + .published(layerfs_project::store::projects) + .map_err(lookup::published) + })?; + Ok(stacks + .into_iter() + .map(|stack| LiveProject { + name: stack.name.as_str().to_owned(), + stack: stack.id.to_bytes().to_vec(), + }) + .collect()) +} + +/// The listing a worker-backed Store sends (T4.5c): the worker runs the +/// same `layerfs_project::store::projects` under its own root, so a project +/// retired in the Store is hidden here as it is for a local one. +fn live_projects_on_worker( + worker: &mut Worker, + store: &str, + record: &StoreRecord, +) -> std::result::Result, SdkRefusal> { + let frame = ListingFrame { + store: StoreName::parse(store)?, + store_record: Some(record.to_record().clone()), + }; + // A listing whose answer was lost is `Unknown`; name the Store it listed + // so the detail carries `{entity: {kind, name}, phase, op}` (spec §10 A6). + let answer = worker + .call("project.list", frame.encode(), &mut |_| {}) + .map_err(|refusal| crate::unknown::named_unknown(refusal, "store", store, None))?; + let reply = ListingReply::decode(&answer)?; + Ok(reply + .projects + .into_iter() + .map(|project| LiveProject { + name: project.name, + stack: project.stack.to_bytes().to_vec(), + }) + .collect()) +} + +/// Whether `project` is in `live`: by the stack id its record holds, or, +/// for a record without one, by its name (spec §4.2). +fn listed(project: &ProjectRecord, name: &str, live: &[LiveProject]) -> bool { + match project.stack() { + Some(stack) => live.iter().any(|item| item.stack == stack), + None => live.iter().any(|item| item.name == name), + } +} + /// One project as callers see it; a record that does not read shows no /// `created`. fn summary(home: &Path, store: &str, name: &str, record: &StoreRecord, view: View) -> SdkValue { @@ -58,9 +146,18 @@ fn summary(home: &Path, store: &str, name: &str, record: &StoreRecord, view: Vie project.to_sdk(view, record) } -/// `project.get(project)`: one project's record. +/// `project.get(project)`: one project's record. A project its Store no +/// longer lists (retired, or absent) is `NoSuchProject` even when its home +/// entry remains (spec §4.2). pub fn get(home: &Path, project: &ProjectRef) -> Outcome { let target = resolve::target(home, &project.to_string())?; + let live = live_projects(home, &target.store, &target.record) + .map_err(|refusal| crate::unknown::named_unknown(refusal, "store", &target.store, None))?; + let record = ProjectRecord::read(home, &target.store, &target.project)? + .ok_or_else(|| refused("NoSuchProject", format!("no project {project}")))?; + if !listed(&record, &target.project, &live) { + return Err(refused("NoSuchProject", format!("no project {project}"))); + } Ok(summary( home, &target.store, @@ -72,21 +169,51 @@ pub fn get(home: &Path, project: &ProjectRef) -> Outcome { /// `project.list(store?, { page_size, page_pos })`, by Store then name. pub fn list(home: &Path, request: &ProjectList) -> Outcome { - let stores = match &request.store { + let stores: Vec<(String, StoreRecord)> = match &request.store { Some(store) => { - if !paths::store_record(home, store).exists() { - return Err(refused("NotFound", format!("no Store named {store}"))); + // A named listing agrees with `project.get` and every other Store + // call: a Store the home does not record is `NoSuchStore` (spec + // §4.2), never the home-level `NotFound`. + let record = StoreRecord::read(home, store)? + .ok_or_else(|| refused("NoSuchStore", format!("no Store named {store}")))?; + vec![(store.to_string(), record)] + } + None => { + let mut records = Vec::new(); + for store in paths::names(&home.join("store")).map_err(failed)? { + if let Some(record) = StoreRecord::read(home, &store)? { + records.push((store, record)); + } } - vec![store.to_string()] + records } - None => paths::names(&home.join("store")).map_err(failed)?, }; let mut items = Vec::new(); - for store in stores { - let Some(record) = StoreRecord::read(home, &store)? else { - continue; + for (store, record) in stores { + // The listing is the Store's own filtered one (spec §4.2), read + // where its calls run (T4.5c): a home entry whose project is hidden + // or absent is not listed. An all-Stores listing omits a Store it + // cannot list; a named one reports the refusal. + let live = match live_projects(home, &store, &record) { + Ok(live) => live, + Err(refusal) => { + if request.store.is_some() { + // A named Store it cannot list is `Busy`/`Unknown`, + // naming the Store in the `Unknown` detail (spec §10 A6). + return Err(crate::unknown::named_unknown( + refusal, "store", &store, None, + )); + } + continue; + } }; for name in paths::names(&paths::store_dir(home, &store).join("project")).map_err(failed)? { + let Some(project) = ProjectRecord::read(home, &store, &name)? else { + continue; + }; + if !listed(&project, &name, &live) { + continue; + } items.push(summary(home, &store, &name, &record, View::List)); } } @@ -218,10 +345,7 @@ fn init_on_worker( timeout, } = call; if paths::project_record(&home, store, name).exists() { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(crate::project::add::home_name_taken(store.as_str(), name)); } let total = send::measure(Path::new(folder), name)?; let copy = send::Copy { @@ -249,7 +373,11 @@ fn init_on_worker( let sent_ms = layerfs_home::home::now_ms(); let added = worker.call("project.init", request, &mut |event| events.event(event)); drop(lease); - let added = added.map_err(|mut refusal| { + let added = added.map_err(|refusal| { + // A lost answer is `Unknown`; name the project it may have + // initialized (spec §10 A6), whether the transport lost the reply or + // the worker reported a bare `UnknownOutcome` (F43R4-2). + let mut refusal = crate::unknown::named_unknown(refusal, "project", name, None); if refusal.code == "Unknown" { let left = send::left(machine, ©.id); refusal.message = format!( @@ -302,10 +430,7 @@ fn init_postgresql( let home = home.to_path_buf(); let path = paths::project_record(&home, store, name); if path.exists() { - return Err(refused( - "Exists", - format!("Store {store} already has a project named {name}"), - )); + return Err(crate::project::add::home_name_taken(store.as_str(), name)); } let add = PgProjectAdd { add: ProjectAdd { @@ -329,49 +454,3 @@ fn init_postgresql( } Ok(out) } - -/// `project.remove(project)`: drops the project's Layers, Branches and -/// Commits from its Store in one write, then its record. Its Sandboxes must -/// be closed first; the other projects stay. -pub fn remove( - home: &Path, - events: &mut Events<'_>, - project: &ProjectRef, - related: Related, -) -> Outcome { - let text = project.to_string(); - let home = home.to_path_buf(); - let target = resolve::target(&home, &text)?; - target.record.unblocked()?; - (related.can_remove_project)(&home, &text)?; - let request = RemoveFrame { - store: project.store.clone(), - project: target.project.clone(), - store_record: Some(target.record.to_record().clone()), - }; - if let StoreRef::Postgres { .. } = target.store_ref { - return Err(refused( - "Unsupported", - "removing a project from a PostgreSQL Store is not built", - )); - } - let removed = match place::plan(&home, &target.store_ref)? { - Executor::Local => { - let _lease = lease::hold(&home, &target.store)?; - crate::project::remove::remove_project(&home_of(Some(&home))?, &request)? - } - Executor::Worker(mut worker) => { - let _lease = lease::hold(&home, &target.store)?; - worker.call("project.remove", request.encode(), &mut |e| events.event(e))? - } - }; - let entry = paths::project_dir(&home, &target.store, &target.project); - if entry.exists() { - std::fs::remove_dir_all(&entry).map_err(failed)?; - } - Ok(layerfs_home::home::with_field( - removed, - "project", - SdkValue::text(text), - )) -} diff --git a/control/crates/layerfs-store/src/project/frame.rs b/control/crates/layerfs-store/src/project/frame.rs index 15850df28a..19e1547fd1 100644 --- a/control/crates/layerfs-store/src/project/frame.rs +++ b/control/crates/layerfs-store/src/project/frame.rs @@ -1,8 +1,12 @@ //! The `project` worker frames (layerfs-project#109): `project.init`, -//! `project.remove`, `project.stage` and `project.export`, each encoded by the caller and +//! `project.list` (T4.5c), `project.stage`, +//! `project.export`, `project.delete` (T4.5b) and its admission ops, each +//! encoded by the caller and //! decoded once in the worker, in the order the worker always refused them. //! The worker sets the home to its own root; a frame names none. use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::{hex, unhex}; +use layerfs_history::LayerStackId; use layerfs_home::helpers::required; use layerfs_home::names::StoreName; use layerfs_home::outcome::refused; @@ -26,18 +30,6 @@ pub struct InitFrame { pub store_record: Option, } -/// `project.remove`: drop project `project` from Store `store` (here, or in -/// its worker). -#[derive(Clone, Debug)] -pub struct RemoveFrame { - /// The Store. - pub store: StoreName, - /// The project's name in it. - pub project: String, - /// The Store's record. - pub store_record: Option, -} - /// `project.export` in a worker: stream Branch `branch` of project /// `project` of Store `store` back to the caller. #[derive(Clone, Debug)] @@ -93,6 +85,29 @@ fn staged_id(frame: &SdkValue) -> Result { Ok(id.to_owned()) } +/// The LayerStack identity a `stack_id` field holds: its 17 bytes in hex. +fn stack_id(text: &str) -> Result { + let bytes = + unhex(text).ok_or_else(|| refused("InvalidInput", "`stack_id` must be hex digits"))?; + LayerStackId::from_slice(&bytes).map_err(|_| { + refused( + "InvalidInput", + "`stack_id` must be a LayerStack identity (17 bytes)", + ) + }) +} + +/// The operation identity an `op_id` field holds: its 16 bytes in hex. +fn op_id(text: &str) -> Result<[u8; 16]> { + let bytes = unhex(text).ok_or_else(|| refused("InvalidInput", "`op_id` must be hex digits"))?; + bytes.as_slice().try_into().map_err(|_| { + refused( + "InvalidInput", + "`op_id` must be an operation identity (16 bytes)", + ) + }) +} + impl InitFrame { /// The `project.init` frame: `{ store, name, staged, timeoutMs, /// store_record }`, each sent when set. @@ -119,30 +134,6 @@ impl InitFrame { } } -impl RemoveFrame { - /// The `project.remove` frame: `{ store, project, store_record }`. - pub fn encode(&self) -> SdkValue { - let mut fields = vec![ - ("store", SdkValue::text(self.store.as_str())), - ("project", SdkValue::text(&self.project)), - ]; - fields.extend(self.store_record.clone().map(|r| ("store_record", r))); - map(fields) - } - - /// A `project.remove` frame: no Store, no project, then a bad Store - /// name are refused. - pub fn decode(frame: &SdkValue) -> Result { - let store = required(frame.get_text("store"), "store")?; - let project = required(frame.get_text("project"), "project")?.to_owned(); - Ok(Self { - store: StoreName::parse(store)?, - project, - store_record: frame.get("store_record").cloned(), - }) - } -} - impl StageFrame { /// The `project.stage` frame: `{ store, name, staged, step, part? }`. pub fn encode(self) -> SdkValue { @@ -226,3 +217,400 @@ impl ExportFrame { }) } } + +/// `project.delete`: retire Stack `stack` of Store `store` (here, or in its +/// worker, spec §4). The caller writes `deleted_ms` to its durable intent +/// before the call and resends it on a retry; `project` and `store_record` +/// are the caller context for the worker own entry and its Store open. +/// `op_id` is the admission the caller already took, so the marker reuses it +/// instead of scanning past a second row. `pending_retry` says the caller +/// retries an intent written before that admission, so the worker keeps its +/// own entry for the call that finishes the delete. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteFrame { + /// The Store. + pub store: StoreName, + /// The project, as sent; the worker names its own entry with it. + pub project: Option, + /// The LayerStack id the caller recorded (never a name). + pub stack: LayerStackId, + /// The `deleted_ms` the marker should hold; `None` lets the worker take + /// the current time. + pub deleted_ms: Option, + /// The admission operation identity the marker reuses, when the caller + /// took one. + pub op_id: Option<[u8; 16]>, + /// True when the caller retries a sidecar written before its admission + /// (no marker confirmed yet): the worker keeps its own entry. + pub pending_retry: Option, + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.admit`: register the caller's project-scope admission +/// before it writes its sidecar and sends the marker (T4.6). The frame is +/// `{store, stack_id, op_id, project?, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteAdmitFrame { + /// The Store. + pub store: StoreName, + /// The LayerStack the caller recorded. + pub stack: LayerStackId, + /// The admission operation identity. + pub op: [u8; 16], + /// The project's name, when the caller knows it: the registered intent + /// carries it, so a later `Busy`/`Unresolved` names the project rather + /// than the hex LayerStack id. An older caller omits it and the worker + /// fills it from the Stack record. + pub project: Option, + /// The Store record. + pub store_record: Option, +} + +/// `project.delete.release`: delete the caller's admission row after the +/// marker, on every exit path (T4.6). The frame is +/// `{store, op_id, store_record?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteReleaseFrame { + /// The Store. + pub store: StoreName, + /// The admission operation identity. + pub op: [u8; 16], + /// The Store record. + pub store_record: Option, +} + +/// `project.delete` worker reply: `{kind: "Removed", project, stack_id, +/// deleted_ms, retired?}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteReply { + /// The project the worker own entry named (or the Store name). + pub project: String, + /// The LayerStack the marker names. + pub stack: LayerStackId, + /// The stored `deleted_ms` (a retry reuses it). + pub deleted_ms: i64, + /// Whether the marker already existed (`Retired`); a caller retrying a + /// pending sidecar then finishes its entry. `None` from an older worker + /// is treated as a freshly written marker. + pub retired: Option, +} + +impl DeleteFrame { + /// The `project.delete` frame: `{ store, stack_id, deleted_ms?, project?, + /// op_id?, store_record? }`, each optional field sent when set. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ]; + fields.extend(self.deleted_ms.map(|ms| ("deleted_ms", SdkValue::Int(ms)))); + fields.extend( + self.project + .as_deref() + .map(|p| ("project", SdkValue::text(p))), + ); + fields.extend(self.op_id.map(|op| ("op_id", SdkValue::text(hex(&op))))); + fields.extend( + self.pending_retry + .map(|retry| ("pending_retry", SdkValue::Bool(retry))), + ); + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete` frame: no Store, a bad Store name, no Stack id, a + /// Stack id that is not a LayerStack identity, or an `op_id` that is not + /// an operation identity are refused. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + Ok(Self { + store, + project: frame.get_text("project").map(str::to_owned), + stack, + deleted_ms: frame.get_int("deleted_ms"), + op_id: frame.get_text("op_id").map(op_id).transpose()?, + pending_retry: frame.get("pending_retry").and_then(|value| match value { + SdkValue::Bool(retry) => Some(*retry), + _ => None, + }), + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteAdmitFrame { + /// The `project.delete.admit` frame: `{store, stack_id, op_id, project?, + /// store_record?}`, `project` sent when the caller knows it. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend( + self.project + .as_deref() + .map(|project| ("project", SdkValue::text(project))), + ); + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.admit` frame: its Store, Stack and operation + /// identity are required and checked; an absent `project` is an unnamed + /// caller. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + stack, + op, + project: frame.get_text("project").map(str::to_owned), + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteReleaseFrame { + /// The `project.delete.release` frame: `{store, op_id, store_record?}`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("op_id", SdkValue::text(hex(&self.op))), + ]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.delete.release` frame: its Store and operation identity + /// are required and checked. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let op = op_id(required(frame.get_text("op_id"), "op_id")?)?; + Ok(Self { + store, + op, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl DeleteReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("kind", SdkValue::text("Removed")), + ("project", SdkValue::text(&self.project)), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(self.deleted_ms)), + ]; + fields.extend( + self.retired + .map(|retired| ("retired", SdkValue::Bool(retired))), + ); + map(fields) + } + + /// The reply fields; `kind` must be `Removed`. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Removed") { + return Err(refused( + "Failed", + "the worker project.delete reply is not `Removed`", + )); + } + let project = frame + .get_text("project") + .ok_or_else(|| refused("InvalidInput", "reply has no `project`"))? + .to_owned(); + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + let deleted_ms = frame + .get_int("deleted_ms") + .ok_or_else(|| refused("InvalidInput", "reply has no `deleted_ms`"))?; + Ok(Self { + project, + stack, + deleted_ms, + retired: frame.get("retired").and_then(|value| match value { + SdkValue::Bool(retired) => Some(*retired), + _ => None, + }), + }) + } +} + +/// `project.delete.entry`: remove the worker's own home entry for a Stack +/// whose marker is already durable (a `marked` sidecar retry, T4.5b). The +/// frame is `{store, stack_id, project?}`; it opens no Store, so it carries +/// no Store record. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteEntryFrame { + /// The Store. + pub store: StoreName, + /// The LayerStack whose entry goes. + pub stack: LayerStackId, + /// The project's name, when the caller knows it. + pub project: Option, +} + +/// `project.delete.entry` worker reply: `{kind: "Removed", project, +/// stack_id}`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeleteEntryReply { + /// The project its entry named (or the Store name). + pub project: String, + /// The LayerStack the removal named. + pub stack: LayerStackId, +} + +impl DeleteEntryFrame { + /// The frame: `{store, stack_id, project?}`, with `project` sent when the + /// caller knows it. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![ + ("store", SdkValue::text(self.store.as_str())), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ]; + fields.extend( + self.project + .as_deref() + .map(|project| ("project", SdkValue::text(project))), + ); + map(fields) + } + + /// A `project.delete.entry` frame: its Store and Stack are required and + /// checked; an absent `project` is an unnamed caller. + pub fn decode(frame: &SdkValue) -> Result { + let store = StoreName::parse(required(frame.get_text("store"), "store")?)?; + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + Ok(Self { + store, + stack, + project: frame.get_text("project").map(str::to_owned), + }) + } +} + +impl DeleteEntryReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + SdkValue::map([ + ("kind", SdkValue::text("Removed")), + ("project", SdkValue::text(&self.project)), + ("stack_id", SdkValue::text(hex(&self.stack.to_bytes()))), + ]) + } + + /// The reply fields; `kind` must be `Removed`. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Removed") { + return Err(refused( + "Failed", + "the worker project.delete entry reply is not `Removed`", + )); + } + let project = frame + .get_text("project") + .ok_or_else(|| refused("InvalidInput", "reply has no `project`"))? + .to_owned(); + let stack = stack_id(required(frame.get_text("stack_id"), "stack_id")?)?; + Ok(Self { project, stack }) + } +} + +/// `project.list` in a worker (T4.5c): Store `store`'s filtered project +/// listing, the same `layerfs_project::store::projects` a local or +/// PostgreSQL Store is read with, so a project retired in the Store is +/// hidden here too. A worker's root does not hold the caller's home +/// entries; the caller intersects this listing with them (`project.list`, +/// `project.get`). +#[derive(Clone, Debug)] +pub struct ListingFrame { + /// The Store to list. + pub store: StoreName, + /// The Store's record: it opens the Store under the worker's root. + pub store_record: Option, +} + +/// One project a worker's Store listing names. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListedProject { + /// Its name in the Store. + pub name: String, + /// Its LayerStack. + pub stack: LayerStackId, +} + +/// `project.list` worker reply: `{ kind: "Listed", projects: [{ name, +/// stack_id }] }`, in the Store's name order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListingReply { + /// The Store's live projects. + pub projects: Vec, +} + +impl ListingFrame { + /// The `project.list` frame: `{ store, store_record? }`. + pub fn encode(&self) -> SdkValue { + let mut fields = vec![("store", SdkValue::text(self.store.as_str()))]; + fields.extend(self.store_record.clone().map(|r| ("store_record", r))); + map(fields) + } + + /// A `project.list` frame: no Store, then a bad Store name are refused. + pub fn decode(frame: &SdkValue) -> Result { + Ok(Self { + store: StoreName::parse(required(frame.get_text("store"), "store")?)?, + store_record: frame.get("store_record").cloned(), + }) + } +} + +impl ListingReply { + /// The reply a worker sends. + pub fn encode(&self) -> SdkValue { + let projects = self + .projects + .iter() + .map(|project| { + map(vec![ + ("name", SdkValue::text(&project.name)), + ("stack_id", SdkValue::text(hex(&project.stack.to_bytes()))), + ]) + }) + .collect(); + map(vec![ + ("kind", SdkValue::text("Listed")), + ("projects", SdkValue::List(projects)), + ]) + } + + /// The reply fields; `kind` must be `Listed`, and every project names + /// its Store name and its LayerStack identity. + pub fn decode(frame: &SdkValue) -> Result { + if frame.get_text("kind") != Some("Listed") { + return Err(refused( + "Failed", + "the worker project.list reply is not `Listed`", + )); + } + let Some(SdkValue::List(items)) = frame.get("projects") else { + return Err(refused("InvalidInput", "reply has no `projects` list")); + }; + let mut projects = Vec::with_capacity(items.len()); + for item in items { + let name = item + .get_text("name") + .ok_or_else(|| refused("InvalidInput", "reply project has no `name`"))? + .to_owned(); + let stack = stack_id(required(item.get_text("stack_id"), "stack_id")?)?; + projects.push(ListedProject { name, stack }); + } + Ok(Self { projects }) + } +} diff --git a/control/crates/layerfs-store/src/project/listing.rs b/control/crates/layerfs-store/src/project/listing.rs new file mode 100644 index 0000000000..42b21ae70b --- /dev/null +++ b/control/crates/layerfs-store/src/project/listing.rs @@ -0,0 +1,46 @@ +//! `project.list` in a worker (T4.5c): the Store's filtered project +//! listing, read under the worker's own root with the same +//! `layerfs_project::store::projects` a local or PostgreSQL Store uses, so +//! a project retired in the Store is hidden on that route too. +use crate::open_args::OpenArgs; +use crate::project::frame::{ListedProject, ListingFrame, ListingReply}; +use crate::service; +use layerfs_home::home::paths; +use layerfs_home::outcome::{refused, Outcome}; +use layerfs_store_service::history::lookup; +use std::path::Path; + +/// The `project.list` worker op: Store `store`'s live projects, in name +/// order, under `root` (the worker's own root). The caller intersects the +/// answer with its own home entries (`project.list`, `project.get`). +pub fn list_projects(root: &Path, frame: &ListingFrame) -> Outcome { + let store = frame.store.as_str(); + let store_record = frame + .store_record + .clone() + .ok_or_else(|| refused("InvalidInput", "project.list needs the Store's record"))?; + let open = OpenArgs { + home: Some(root.to_path_buf()), + store: Some(store.to_owned()), + store_path: Some(paths::store_file(root, store).display().to_string()), + store_record: Some(store_record), + postgresql: None, + ..OpenArgs::default() + }; + let stacks = service::with_service(&open, |service| { + service + .store + .published(layerfs_project::store::projects) + .map_err(lookup::published) + })?; + Ok(ListingReply { + projects: stacks + .into_iter() + .map(|stack| ListedProject { + name: stack.name.as_str().to_owned(), + stack: stack.id, + }) + .collect(), + } + .encode()) +} diff --git a/control/crates/layerfs-store/src/project/mod.rs b/control/crates/layerfs-store/src/project/mod.rs index 127b636a79..1b35d9d8ca 100644 --- a/control/crates/layerfs-store/src/project/mod.rs +++ b/control/crates/layerfs-store/src/project/mod.rs @@ -1,17 +1,18 @@ //! Public Project organization over host Init, and a project's home entry //! and discovery records (`discovery`). pub mod add; +pub mod delete; pub mod discovery; pub mod export; #[doc(hidden)] pub mod family; pub mod frame; pub mod init; +pub mod listing; pub mod parts; pub mod postgres_init; #[doc(hidden)] pub mod record; -pub mod remove; pub mod send; pub mod staged; pub mod tree; diff --git a/control/crates/layerfs-store/src/project/record.rs b/control/crates/layerfs-store/src/project/record.rs index 02578ba974..7bddbd071b 100644 --- a/control/crates/layerfs-store/src/project/record.rs +++ b/control/crates/layerfs-store/src/project/record.rs @@ -4,6 +4,8 @@ //! `sandbox/files.rs` reads and writes with the rest kept. use crate::record::StoreRecord; use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_bridge::sdk_values::unhex; use layerfs_home::home::{now_ms, paths, read_record, with_field, write_entity}; use layerfs_home::outcome::View; use std::path::Path; @@ -76,6 +78,19 @@ impl ProjectRecord { self.fields.get("created") } + /// The project's LayerStack identity as its record holds it (S §4.2): + /// the `manifest`'s stack for a local SQLite record, the record's + /// `stack` for a PostgreSQL or worker record. A record with neither has + /// none; its Store's filtered listing is then matched by name. + pub fn stack(&self) -> Option> { + if let Some(SdkValue::Bytes(bytes)) = self.fields.get("manifest") { + if let Ok(manifest) = StoreManifest::decode(bytes) { + return Some(manifest.stack.to_vec()); + } + } + self.fields.get_text("stack").and_then(unhex) + } + /// The record as written. pub fn to_record(&self) -> &SdkValue { &self.fields diff --git a/control/crates/layerfs-store/src/project/remove.rs b/control/crates/layerfs-store/src/project/remove.rs deleted file mode 100644 index 4e5200e347..0000000000 --- a/control/crates/layerfs-store/src/project/remove.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! A project removed from its Store (`project.remove`, here or in the -//! Store's worker). -use crate::identity::{identity, legacy, save_record, store_dir}; -use crate::open_args::OpenArgs; -use crate::project::frame::RemoveFrame; -use layerfs_bridge::control::SdkValue; -use layerfs_bridge::sdk_values::hex; -use layerfs_home::home::paths; -use layerfs_home::outcome::{failed, refused, Outcome}; -use std::path::Path; - -/// `project.remove`'s work on the Store's machine (here, or in its worker -/// as the worker op `project.remove`), in `home` (the caller's, or the -/// worker's root; checked by the caller with [`home_of`](super::discovery::home_of)): -/// drops the project's stack (its Layers, Branches and Commits) from the -/// Store in one write, then its entry from the home. The other projects -/// stay. The entry of the project a Store was made by (its own Init) holds -/// the Store's identity, so a Store record is saved before that entry goes. -/// Its packs and objects stay in the Store; reclaiming them is A8 (#411). -/// The caller stops the project's Sandboxes and clients first. -pub fn remove_project(home: &Path, request: &RemoveFrame) -> Outcome { - let (store, name) = (request.store.as_str(), request.project.as_str()); - let open = OpenArgs { - home: Some(home.to_path_buf()), - store: Some(store.to_owned()), - project: Some(name.to_owned()), - store_path: Some(paths::store_file(home, store).display().to_string()), - store_record: request.store_record.clone(), - postgresql: None, - }; - let identity = identity(&open)?; - let own = name == store && legacy(home, store)?; - let entry = layerfs_home::home::paths::project_dir(home, store, name); - let stack = crate::service::with_service(&open, |service| { - let stacks = service - .store - .published(layerfs_project::store::projects) - .map_err(failed)?; - let Some(stack) = stacks.into_iter().find(|stack| stack.name.as_str() == name) else { - return Err(refused( - "NoSuchProject", - format!("Store {store} holds no project named {name}"), - )); - }; - service - .store - .history() - .remove_layerstack(stack.id) - .map_err(|error| { - failed(format!( - "removing project {name} from Store {store}: {error:?}" - )) - })?; - Ok(stack.id) - })?; - // the own project's entry carried the Store's identity until now - if own { - layerfs_home::home::private_dir(&store_dir(home, store)?)?; - save_record(home, store, &identity)?; - } - if entry.exists() { - std::fs::remove_dir_all(&entry).map_err(failed)?; - } - Ok(SdkValue::map([ - ("stack", SdkValue::text(hex(&stack.to_bytes()))), - ("packsKept", SdkValue::Bool(true)), - ])) -} diff --git a/control/crates/layerfs-store/src/related.rs b/control/crates/layerfs-store/src/related.rs index 56b471049a..64e8221abb 100644 --- a/control/crates/layerfs-store/src/related.rs +++ b/control/crates/layerfs-store/src/related.rs @@ -5,16 +5,17 @@ use layerfs_bridge::control::SdkRefusal; use std::path::Path; -/// An `InUse` check on the Store or project `name` in home `home`. +/// A related-use check on the Store or project `name` in home `home`. type Check = fn(home: &Path, name: &str) -> Result<(), SdkRefusal>; -/// The `InUse` checks over a Store's and a project's users. +/// The checks over a Store's users. #[derive(Clone, Copy)] pub struct Related { /// `store.push`, `store.pull`, `store.remove`, `store.disconnect` and /// `backup.restore`: refused while another live process holds the Store /// or a Sandbox of one of its projects is recorded. pub store_idle: Check, - /// `project.remove`: refused while a Sandbox of the project is recorded. + /// `project.delete`: refused `Busy` while a Sandbox of the project is + /// recorded, even with nothing mounted. pub can_remove_project: Check, } diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs new file mode 100644 index 0000000000..4983cb82a7 --- /dev/null +++ b/control/crates/layerfs-store/src/report.rs @@ -0,0 +1,132 @@ +//! The wire rendering of the §11 compaction report and the `store.status` +//! compaction projection (T10). +//! +//! One serializer renders the shared [`CompactReport`], so the SQLite path and +//! T9's PostgreSQL path cannot disagree on a key: every key is snake_case (§10; +//! §11 names are logical), a zero field is named by its wire key, and +//! `residue.reason` appears only when something remains. +use layerfs_bridge::control::SdkValue; +use layerfs_persistence::{CompactReport, CompactStatus}; + +/// The §11 report as `store.compact` returns it: +/// `{kind: "Compacted", path, generation {before, after}, ...}`. +#[must_use] +pub fn compact_report(report: &CompactReport) -> SdkValue { + let int = |value: u64| SdkValue::Int(i64::try_from(value).unwrap_or(i64::MAX)); + let references: Vec = report + .kept_waste + .delta_base_references + .iter() + .map(|id| SdkValue::text(layerfs_bridge::sdk_values::hex(id))) + .collect(); + // §11/S7: a capped field carries its cap. A list cut at the cap renders as + // `{value, cap}`; every other report keeps the plain identity list. + let references = if report.kept_waste.delta_base_references_capped { + SdkValue::map([ + ("value", SdkValue::List(references)), + ( + "cap", + int(layerfs_persistence::DELTA_BASE_REFERENCES_CAP as u64), + ), + ]) + } else { + SdkValue::List(references) + }; + let mut residue = vec![ + ("bytes".to_owned(), int(report.residue.bytes)), + ( + "artifacts".to_owned(), + SdkValue::List( + report + .residue + .artifacts + .iter() + .map(SdkValue::text) + .collect(), + ), + ), + ]; + if let Some(reason) = &report.residue.reason { + residue.push(("reason".to_owned(), SdkValue::text(reason))); + } + let zero_reasons = report + .zero_reasons + .iter() + .map(|zero| { + SdkValue::map([ + ("field", SdkValue::text(zero.wire_field())), + ("reason", SdkValue::text(zero.reason)), + ]) + }) + .collect(); + SdkValue::Map( + [ + ("kind", SdkValue::text("Compacted")), + ("path", SdkValue::text(report.path.as_str())), + ( + "generation", + SdkValue::map([ + ("before", int(report.generation.before)), + ("after", int(report.generation.after)), + ]), + ), + ("dead_bytes_bound", int(report.dead_bytes_bound)), + ("dead_bytes_refined", int(report.dead_bytes_refined)), + ("logical_discarded", int(report.logical_discarded)), + ("backend_reusable", int(report.backend_reusable)), + ("physical_shrink", int(report.physical_shrink)), + ( + "kept_waste", + SdkValue::map([ + ( + "dead_in_retained_packs_bytes", + int(report.kept_waste.dead_in_retained_packs_bytes), + ), + ( + "delta_bases", + SdkValue::map([ + ("bytes", int(report.kept_waste.delta_bases_bytes)), + ("references", references), + ]), + ), + ( + "inode_blocks_bytes", + int(report.kept_waste.inode_blocks_bytes), + ), + ]), + ), + ("peak_temp_bytes", int(report.peak_temp_bytes)), + ("residue", SdkValue::Map(residue.into_iter().collect())), + ("estimate", SdkValue::Bool(report.estimate)), + ("zero_reasons", SdkValue::List(zero_reasons)), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ) +} + +/// The `store.status` projection of a compaction: `{id, name, phase, started, +/// uncertain, state, phases}`. The caller splits the op fields into its `op` +/// and leaves `state`/`phases` beside them (§11). +#[must_use] +pub fn compact_status(status: &CompactStatus) -> SdkValue { + SdkValue::map([ + ("id", SdkValue::text(&status.id)), + ("name", SdkValue::text(status.name)), + ("phase", SdkValue::text(&status.phase)), + ("started", SdkValue::Int(status.started)), + ("uncertain", SdkValue::Bool(status.uncertain)), + ("state", SdkValue::text(status.state)), + ( + "phases", + SdkValue::List( + status + .phases + .iter() + .map(|phase| SdkValue::text(*phase)) + .collect(), + ), + ), + ]) +} diff --git a/control/crates/layerfs-store/src/sqlite/transfer/step.rs b/control/crates/layerfs-store/src/sqlite/transfer/step.rs index 50dca83651..98dc7021d0 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/step.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/step.rs @@ -12,6 +12,11 @@ type Result = std::result::Result; pub enum StoreStep { /// Whether the file is there, its size and the free bytes. Stat, + /// The compaction operation recorded beside the Store, for `store.status`. + /// + /// It reads only `.compact-op`; never the Store database, an + /// artifact or a directory listing. + CompactOp, /// The idle Store's bytes, sha256 and heads. Snapshot, /// A consistent snapshot of the live Store, staged beside it. @@ -68,6 +73,7 @@ impl StoreStep { pub fn name(&self) -> &'static str { match self { Self::Stat => "stat", + Self::CompactOp => "compact-op", Self::Snapshot => "snapshot", Self::Stage => "stage", Self::Unstage => "unstage", @@ -161,6 +167,7 @@ impl StepFrame { layerfs_home::names::StoreName::parse(store)?; let step = match op { "stat" => StoreStep::Stat, + "compact-op" => StoreStep::CompactOp, "snapshot" => StoreStep::Snapshot, "stage" => StoreStep::Stage, "unstage" => StoreStep::Unstage, diff --git a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs index ef250836ef..fdca5f159a 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs @@ -25,6 +25,16 @@ fn with_suffix(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(name) } +/// Whether the process a compaction record names still runs on this machine. +/// The kernel start time and boot id, when recorded, keep a reused id from +/// passing as the owner. +fn owner_alive(op: &layerfs_persistence::CompactOpStatus) -> bool { + let Some(pid) = op.owner_pid else { + return false; + }; + layerfs_home::process::alive_as(pid, op.owner_pid_start, op.owner_pid_boot) +} + fn length(path: &Path) -> Result { match std::fs::metadata(path) { Ok(meta) => Ok(meta.len()), @@ -124,6 +134,12 @@ pub fn run_observed(root: &Path, frame: &StepFrame, progress: &mut dyn FnMut(u64 ), ("incoming_bytes", SdkValue::Int(length(&incoming)? as i64)), ])), + // the compaction record beside the Store only: `store.status` shows + // its op and phase without opening the Store database or an artifact + StoreStep::CompactOp => match layerfs_persistence::compact_op_status(&file) { + None => Ok(SdkValue::Null), + Some(op) => Ok(crate::report::compact_status(&op.status(owner_alive(&op)))), + }, // snapshot: the idle Store's bytes, sha256 and history heads. A Store // nobody holds has no write-ahead log left unless a process crashed; // if a crashed opener left one, checkpoint and close it first. diff --git a/control/crates/layerfs-store/src/status.rs b/control/crates/layerfs-store/src/status.rs index e62157e506..664f7d4a87 100644 --- a/control/crates/layerfs-store/src/status.rs +++ b/control/crates/layerfs-store/src/status.rs @@ -6,33 +6,79 @@ use crate::sqlite::family::{fields_map, record_of, step_on}; use crate::sqlite::transfer::step::StoreStep; use layerfs_bridge::control::SdkValue; use layerfs_home::outcome::Outcome; +use layerfs_persistence::compact_metrics::{self, CompactStatusState}; use std::path::Path; -/// `store.status`: who holds it, the operation its record holds (`op`, as -/// H6 shows it), its size. It starts no machine: a Store whose machine is -/// not running is `down`, with the reason (O34). It changes nothing (#536 -/// §12 item 3): no lock, and a stale lease is ignored, not removed. +/// `store.status`: who holds it, the operation on it (`op`, as H6 shows it), +/// its size. It starts no machine: a Store whose machine is not running is +/// `down`, with the reason (O34). It changes nothing (#536 §12 item 3): no +/// lock, and a stale lease is ignored, not removed. +/// +/// While a compaction runs, or after a dead owner left one, `op` is that run's +/// projection and `state` is `busy`/`unresolved` (§11); `phases` names the +/// backend's ordered phases. A sqlite compaction record is read from +/// `.compact-op` alone, and a PostgreSQL one from the Store-side +/// `gc_op` with its `gc_intent` on one status query (T10.5): neither opens +/// the Store database, changes a schema, nor reads an artifact. pub fn status(home: &Path, name: &str) -> Outcome { let record = record_of(home, name)?; let (holders, _) = lease::read_holders(home, name)?; let mut down = None; let mut more = Vec::new(); let store_ref = record.store_ref()?; + let compaction: SdkValue; if let StoreRef::Postgres { .. } = store_ref { more.push(("server", crate::postgresql::connect::status(&record))); + // T10.5: a PostgreSQL compaction's op is the Store's `gc_op` row read + // with its `gc_intent`; this path opens no Store and reads no file. + compaction = crate::postgresql::connect::compact_status(&record); } else { more.push(("machine", SdkValue::text(store_ref.machine()?))); - // the file's size where it lives, starting nothing (P4); a machine - // that is down says why - let file = place::plan(home, store_ref) - .and_then(|mut executor| step_on(home, &mut executor, name, StoreStep::Stat)); + // the file's size where it lives, starting nothing (P4), and the + // compaction record beside it; a machine that is down says why + let mut placed = place::plan(home, store_ref); + let (file, op) = match placed.as_mut() { + Ok(executor) => ( + step_on(home, executor, name, StoreStep::Stat), + step_on(home, executor, name, StoreStep::CompactOp).unwrap_or_default(), + ), + Err(why) => (Err(why.clone()), SdkValue::Null), + }; + compaction = op; let file = file.unwrap_or_else(|why| { down = Some(why.message.clone()); SdkValue::map([("unavailable", SdkValue::Text(why.message))]) }); more.push(("file", file)); } - let state = record.state(down.is_some(), &holders).as_str(); + // A compaction's record is the operation the Store is running, even when + // some other process holds its lease; otherwise the record's own `op` + // (a push, pull or remove) is what `store.status` shows. + let op = if compaction.get("id").is_some() { + let field = |field: &str| compaction.get(field).cloned().unwrap_or_default(); + SdkValue::map([ + ("id", field("id")), + ("name", field("name")), + ("phase", field("phase")), + ("started", field("started")), + ("uncertain", field("uncertain")), + ]) + } else { + record.op().map(|op| op.to_sdk()).unwrap_or_default() + }; + let phases = compaction.get("phases").cloned().unwrap_or_default(); + let state = compaction + .get_text("state") + .unwrap_or_else(|| record.state(down.is_some(), &holders).as_str()); + // The status path observes an unsettled run by its bounded state label, + // never by a Store name or phase path. + if compaction.get("id").is_some() { + match state { + "busy" => compact_metrics::record_status(CompactStatusState::Busy), + "unresolved" => compact_metrics::record_status(CompactStatusState::Unresolved), + _ => {} + } + } let mut fields = vec![ ("name", SdkValue::text(name)), ("kind", SdkValue::text(store_ref.kind().as_str())), @@ -46,7 +92,8 @@ pub fn status(home: &Path, name: &str) -> Outcome { .collect(), ), ), - ("op", record.op().map(|op| op.to_sdk()).unwrap_or_default()), + ("op", op), + ("phases", phases), ("down_reason", down.map(SdkValue::Text).unwrap_or_default()), ]; fields.extend(more); diff --git a/control/crates/layerfs-store/src/unknown.rs b/control/crates/layerfs-store/src/unknown.rs new file mode 100644 index 0000000000..02566e905c --- /dev/null +++ b/control/crates/layerfs-store/src/unknown.rs @@ -0,0 +1,48 @@ +//! Spec §10 A6: the `Unknown` detail a GC call can name when the worker's +//! answer was lost. A lost reply is `refused("Unknown", ...)` with a null +//! detail; the caller still knows the entity it addressed, and the operation +//! identity when it held one. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; + +/// Names the entity on an `Unknown` refusal whose detail carries nothing. +/// +/// A reply lost on the transport has a null detail; a bare `UnknownOutcome` +/// a call cannot name yet renders the empty map from the store service's +/// `lookup::history` (spec gap G1). Both mean "no detail", so the caller +/// fills them. A refusal that already names something, or any other code, is +/// returned unchanged. `phase` stays null: the lost call did not report one, +/// and `op` is the durable operation the caller held, if any. +#[doc(hidden)] +pub fn named_unknown( + mut refusal: SdkRefusal, + kind: &str, + name: &str, + op: Option<&[u8; 16]>, +) -> SdkRefusal { + if refusal.code != "Unknown" || !no_detail(&refusal.detail) { + return refusal; + } + refusal.detail = SdkValue::map([ + ( + "entity", + SdkValue::map([ + ("kind", SdkValue::text(kind)), + ("name", SdkValue::text(name)), + ]), + ), + ("phase", SdkValue::Null), + ( + "op", + op.map_or(SdkValue::Null, |op| { + SdkValue::text(layerfs_bridge::sdk_values::hex(op)) + }), + ), + ]); + refusal +} + +/// Whether a detail carries no fields: a lost answer's null, or the empty map +/// a bare `UnknownOutcome` renders. +fn no_detail(detail: &SdkValue) -> bool { + *detail == SdkValue::Null || matches!(detail, SdkValue::Map(fields) if fields.is_empty()) +} diff --git a/control/crates/layerfs-store/tests/compact_in_process_call.rs b/control/crates/layerfs-store/tests/compact_in_process_call.rs new file mode 100644 index 0000000000..cb10b33e1c --- /dev/null +++ b/control/crates/layerfs-store/tests/compact_in_process_call.rs @@ -0,0 +1,61 @@ +//! The in-process call refusal `store.compact` reports (#51 Round 3, F51R3-1). +//! +//! A call still running on the Store in this process keeps the open cache's +//! `InUse`; the compaction maps it into the GC vocabulary as `Busy` naming +//! the Store with a null phase and op. Product source holds no tests (the +//! product-boundary guard), so `compact::gc_refusal` is `#[doc(hidden)] pub` +//! for this target alone. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_home::outcome::refused; +use layerfs_store::compact::gc_refusal; + +fn in_use() -> SdkRefusal { + refused("InUse", "a call is running on the Store in this process") +} + +fn field<'a>(detail: &'a SdkValue, name: &str) -> &'a SdkValue { + let SdkValue::Map(fields) = detail else { + panic!("detail is not a map: {detail:?}"); + }; + fields + .iter() + .find(|(field, _)| field == name) + .map(|(_, value)| value) + .unwrap_or_else(|| panic!("detail has no `{name}`: {detail:?}")) +} + +/// F51R3-1: a call running on the Store in this process is `Busy`, never +/// `InUse`, and names the Store with no phase or op. +#[test] +fn a_running_call_is_busy_for_the_store_not_in_use() { + let busy = gc_refusal("shop", in_use()); + assert_eq!(busy.code, "Busy", "{busy:?}"); + assert!( + busy.message.contains("Store shop is busy"), + "{}", + busy.message + ); + assert!(!busy.message.contains("InUse"), "{}", busy.message); + assert_eq!( + field(&busy.detail, "entity"), + &SdkValue::map([ + ("kind", SdkValue::text("store")), + ("name", SdkValue::text("shop")), + ]) + ); + assert_eq!(field(&busy.detail, "phase"), &SdkValue::Null); + assert_eq!(field(&busy.detail, "op"), &SdkValue::Null); +} + +/// The mapping touches only the open cache's `InUse`; every other refusal is +/// what it was, so `Busy`, `Failed` and the rest keep their detail. +#[test] +fn exactly_the_open_cache_refusal_is_mapped() { + for (code, message) in [ + ("Failed", "the open cache failed"), + ("Busy", "the Store is held by another writer"), + ] { + let original = refused(code, message); + assert_eq!(gc_refusal("shop", original.clone()), original); + } +} diff --git a/control/crates/layerfs-store/tests/compact_report_status.rs b/control/crates/layerfs-store/tests/compact_report_status.rs new file mode 100644 index 0000000000..73d97b4092 --- /dev/null +++ b/control/crates/layerfs-store/tests/compact_report_status.rs @@ -0,0 +1,327 @@ +//! T10: the §11 report wire shape and the `store.status` compaction op +//! projection. The report renders snake_case from the shared `CompactReport`; +//! status reads only the `.compact-op` sidecar and never opens the +//! Store database. +use layerfs_bridge::control::SdkValue; +use layerfs_content::ObjectId; +use layerfs_home::home::{paths, private_dir, write_entity}; +use layerfs_home::names::StoreName; +use layerfs_persistence::{ + CompactKeptWaste, CompactPath, CompactReceipt, CompactReport, CompactResidue, + CompactZeroReason, KeptWaste, +}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t10-report-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + let SdkValue::List(items) = value else { + return Vec::new(); + }; + items + .iter() + .filter_map(|item| match item { + SdkValue::Text(text) => Some(text.clone()), + _ => None, + }) + .collect() +} + +/// A home holding one empty sqlite Store `s` on native. +fn store_home() -> PathBuf { + let home = tempdir("home"); + let record = SdkValue::map([ + ("name", SdkValue::text("s")), + ("kind", SdkValue::text("sqlite")), + ("machine", SdkValue::text("native")), + ]); + write_entity(&paths::store_record(&home, "s"), &record).unwrap(); + private_dir(&paths::store_dir(&home, "s")).unwrap(); + std::fs::write(paths::store_file(&home, "s"), b"").unwrap(); + home +} + +/// A complete `.compact-op` record, as the persistence layer writes it. +/// `returned` marks a record whose run has already replied to its caller. +fn sidecar_with(home: &Path, phase: &str, owner: u32, returned: bool) -> PathBuf { + let mut path = paths::store_file(home, "s").into_os_string(); + path.push(".compact-op"); + let path = PathBuf::from(path); + let zeros = "00".repeat(32); + let marker = if returned { "returned=1\n" } else { "" }; + std::fs::write( + &path, + format!( + "version=1\nop_id=00112233445566778899aabbccddeeff\ngeneration=3\n\ +phase={phase}\nstarted_ms=1234\nowner_pid={owner}\nowner_pid_start=\nowner_pid_boot=\n\ +catalog_id=0102030405060708090a0b0c0d0e0f10\nnext_history_serial=5\nnext_pack_id=6\n\ +next_ordinal=7\nmark_serial=4\nmark_pack_id=6\nmark_digest={zeros}\nmark_totals=1,2,3,4\n\ +mark_stamp=9\nhistory_rows=10\nlive_heads_digest={zeros}\n{marker}" + ), + ) + .unwrap(); + path +} + +fn sidecar(home: &Path, phase: &str, owner: u32) -> PathBuf { + sidecar_with(home, phase, owner, false) +} + +fn receipt() -> CompactReceipt { + CompactReceipt { + path: CompactPath::Rebuild, + generation_before: 3, + generation_after: 4, + dead_bytes_bound: 100, + dead_bytes_refined: 80, + logical_discarded: 40, + backend_reusable: 4096, + physical_shrink: 8192, + kept_waste: CompactKeptWaste { + dead_in_retained_packs_bytes: 5, + delta_bases_bytes: 6, + delta_base_references: vec![[7; 32]], + delta_base_references_capped: false, + inode_blocks_bytes: 8, + }, + peak_temp_bytes: 900, + residue: CompactResidue { + bytes: 0, + artifacts: Vec::new(), + reason: None, + }, + estimate: true, + zero_reasons: vec![ + CompactZeroReason { + field: "deadBytesBound", + reason: "no-waste", + }, + CompactZeroReason { + field: "logicalDiscarded", + reason: "no-history", + }, + ], + } +} + +#[test] +fn a_capped_reference_list_carries_its_cap() { + let mut capped = receipt(); + capped.kept_waste.delta_base_references = + vec![[9; 32]; layerfs_persistence::DELTA_BASE_REFERENCES_CAP]; + capped.kept_waste.delta_base_references_capped = true; + let wire = layerfs_store::report::compact_report(&CompactReport::from(&capped)); + let bases = wire.get("kept_waste").unwrap().get("delta_bases").unwrap(); + let references = bases.get("references").unwrap(); + assert_eq!( + texts(references.get("value").unwrap()).len(), + layerfs_persistence::DELTA_BASE_REFERENCES_CAP + ); + assert_eq!( + references.get_int("cap"), + Some(layerfs_persistence::DELTA_BASE_REFERENCES_CAP as i64), + "a capped field carries its cap (S7)" + ); + // The bytes are still the whole kept set's bytes. + assert_eq!(bases.get_int("bytes"), Some(6)); +} + +/// A mark over the delta-base cap records that it cut its list, and the §11 +/// projection renders `{value, cap}` from that mark. The bug was that the mark +/// truncated its list before the projection could see the cut, so the capped +/// shape never reached the wire. +#[test] +fn a_mark_over_the_delta_base_cap_renders_value_and_cap() { + let references: Vec = (0..=layerfs_persistence::DELTA_BASE_REFERENCES_CAP) + .map(|index| { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64).to_le_bytes()); + ObjectId::from_bytes(&bytes).unwrap() + }) + .collect(); + let waste = KeptWaste::from_parts(0, 12, references, 0); + let mut receipt = receipt(); + receipt.kept_waste = CompactKeptWaste::from_waste(&waste); + let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt)); + let bases = wire.get("kept_waste").unwrap().get("delta_bases").unwrap(); + let references = bases.get("references").unwrap(); + assert_eq!( + texts(references.get("value").unwrap()).len(), + layerfs_persistence::DELTA_BASE_REFERENCES_CAP, + "the wire keeps exactly the cap's identities" + ); + assert_eq!( + references.get_int("cap"), + Some(layerfs_persistence::DELTA_BASE_REFERENCES_CAP as i64), + "a mark that cut its list renders the cap (S7)" + ); + // The byte total is still the whole kept set's bytes. + assert_eq!(bases.get_int("bytes"), Some(12)); +} + +#[test] +fn the_report_wire_is_the_snake_case_contract_shape() { + let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt())); + assert_eq!(text(wire.get("kind").unwrap()), "Compacted"); + assert_eq!(text(wire.get("path").unwrap()), "rebuild"); + let generation = wire.get("generation").unwrap(); + assert_eq!(generation.get_int("before"), Some(3)); + assert_eq!(generation.get_int("after"), Some(4)); + assert_eq!(wire.get_int("dead_bytes_bound"), Some(100)); + assert_eq!(wire.get_int("dead_bytes_refined"), Some(80)); + assert_eq!(wire.get_int("logical_discarded"), Some(40)); + assert_eq!(wire.get_int("backend_reusable"), Some(4096)); + assert_eq!(wire.get_int("physical_shrink"), Some(8192)); + assert_eq!(wire.get_int("peak_temp_bytes"), Some(900)); + assert_eq!(wire.get("estimate"), Some(&SdkValue::Bool(true))); + let waste = wire.get("kept_waste").unwrap(); + assert_eq!(waste.get_int("dead_in_retained_packs_bytes"), Some(5)); + assert_eq!(waste.get_int("inode_blocks_bytes"), Some(8)); + let bases = waste.get("delta_bases").unwrap(); + assert_eq!(bases.get_int("bytes"), Some(6)); + assert_eq!( + texts(bases.get("references").unwrap()), + vec!["0707070707070707070707070707070707070707070707070707070707070707"] + ); + let residue = wire.get("residue").unwrap(); + assert_eq!(residue.get_int("bytes"), Some(0)); + assert!(texts(residue.get("artifacts").unwrap()).is_empty()); + assert!(residue.get("reason").is_none()); + // One reason per zero field, named by its snake_case wire key (§10). + let SdkValue::List(zero) = wire.get("zero_reasons").unwrap() else { + panic!("zero_reasons is a list"); + }; + let fields: Vec<&str> = zero + .iter() + .map(|zero| text(zero.get("field").unwrap())) + .collect(); + assert_eq!(fields, ["dead_bytes_bound", "logical_discarded"]); + assert_eq!( + text(zero[0].get("reason").unwrap()), + "no-waste", + "the closed vocabulary passes through" + ); + assert!(wire.get("zeroReason").is_none(), "no camelCase key"); +} + +#[test] +fn status_emits_bounded_busy_and_unresolved_metrics() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + let capture = layerfs_persistence::compact_metrics::CompactMetricsCapture::start(); + // Idle: no compaction record, so no `compact.status` sample. + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + // A live owner is `busy`; a gone one is `unresolved`. + sidecar(&home, "build", std::process::id()); + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + sidecar(&home, "build", 99_999_999); + let _ = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + let metrics = capture.finish(); + let value = |state: &str| { + metrics + .samples() + .iter() + .find(|sample| sample.name == "compact.status" && sample.label("state") == Some(state)) + .map_or(0, |sample| sample.value) + }; + assert_eq!(value("busy"), 1); + assert_eq!(value("unresolved"), 1); + assert_eq!(value("idle"), 0, "an idle status is not a compaction op"); + std::fs::remove_dir_all(home).unwrap(); +} + +/// #49 round 3: a returned run's record names an owner pid that is still +/// alive, but the run is over. `store.status` must show that leftover as +/// `unresolved`, never `busy`, and keep its outcome uncertain. +#[test] +fn a_returned_leftover_is_unresolved_even_with_a_live_owner_pid() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + let op_path = sidecar_with(&home, "install", std::process::id(), true); + let status = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(status.get_text("state"), Some("unresolved")); + let op = status.get("op").unwrap(); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + assert_eq!(text(op.get("phase").unwrap()), "install"); + assert_eq!( + op.get("uncertain"), + Some(&SdkValue::Bool(true)), + "a returned leftover's outcome is still unknown" + ); + std::fs::remove_file(op_path).unwrap(); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn status_projects_the_sidecar_phase_and_state() { + let home = store_home(); + let name = StoreName::parse("s").unwrap(); + + // No op: the existing idle shape, no phases. + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + + // A live owner is busy and certain; `build` is the sidecar's phase. + let op_path = sidecar(&home, "build", std::process::id()); + let busy = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + assert_eq!(text(op.get("name").unwrap()), "store.compact"); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get_int("started"), Some(1234)); + // The record remains unsettled while it runs; `state` is what tells a + // live owner from a leftover. + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + texts(busy.get("phases").unwrap()), + ["mark", "build", "verify", "install", "cleanup"] + ); + + // A gone owner leaves an uncertain op: unresolved, same phase, same id. + sidecar(&home, "build", 99_999_999); + let leftover = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("phase").unwrap()), "build"); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!( + text(op.get("id").unwrap()), + "op-00112233445566778899aabbccddeeff" + ); + + // A settled run removes the record: the idle shape is back. + std::fs::remove_file(op_path).unwrap(); + let idle = layerfs_store::sqlite::family::status(&home, &name).unwrap(); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + assert_eq!(idle.get_text("state"), Some("idle")); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-store/tests/postgresql_status.rs b/control/crates/layerfs-store/tests/postgresql_status.rs new file mode 100644 index 0000000000..7369069fd6 --- /dev/null +++ b/control/crates/layerfs-store/tests/postgresql_status.rs @@ -0,0 +1,403 @@ +//! T10.5: the PostgreSQL branch of `store.status`, through the control call. +//! +//! `store.status` of a PostgreSQL Store reads the Store-side `gc_op` row with +//! its compaction `gc_intent` in one status query, projects the operation +//! `{id, name, phase, started, uncertain}`, and separates a live owner +//! (`busy`) from a leftover (`unresolved`); `phases` is PostgreSQL's §8 order +//! `mark`, `build`, `swap`, `cleanup`. A Store with no row — never compacted, +//! or a run that settled and removed its row — is idle, with `op: null` and +//! `phases: null`. +//! +//! This is the control-level end-to-end case T10.5 was missing: the +//! persistence suite proves the reader, and this test drives the call the SDK +//! and CLI actually reach (`layerfs_store::sqlite::family::status`, the +//! kind-neutral `store.status` route) against a real PostgreSQL Store. The +//! rows a compaction leaves are written directly, so the test proves the +//! projection, not a failpoint: a live owner is a real non-idle backend +//! recorded in `pg_stat_activity`. +//! +//! Needs a writable PostgreSQL server: set `LAYERFS_TEST_POSTGRES_URL` to a +//! libpq connection string or URL (docs/architecture/engine/store/testing.md). +//! The test creates and drops its own database, because `store.status` reads +//! the one schema a 2.0 PostgreSQL Store uses (`layerfs`); a shared schema +//! would collide with another run. Without the variable it reports NOT_RUN. +use layerfs_bridge::control::SdkValue; +use layerfs_history::HistoryCatalogConfig; +use layerfs_home::home; +use layerfs_home::names::StoreName; +use layerfs_persistence::{PostgresHandles, PostgresSelection}; +use layerfs_storage::StoragePolicy; +use layerfs_store::record::StoreRecord; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The §8 phases `store.status` names for a PostgreSQL compaction, in order. +const PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; + +const BINDING: &[u8] = b"t105-control-postgresql-status"; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "layerfs-t105-control-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The URL of `database` on the same server as `source`: a URL's path is +/// replaced; a libpq key/value string gains a final `dbname`, which wins. +fn database_url(source: &str, database: &str) -> String { + if source.starts_with("postgres://") || source.starts_with("postgresql://") { + let (base, query) = match source.split_once('?') { + Some((base, query)) => (base, format!("?{query}")), + None => (source, String::new()), + }; + let scheme = base.find("://").expect("a PostgreSQL URL") + 3; + match base[scheme..].find('/').map(|at| scheme + at) { + Some(at) => format!("{}/{database}{query}", &base[..at]), + None => format!("{base}/{database}{query}"), + } + } else { + format!("{source} dbname={database}") + } +} + +/// One disposable database: `store.status` reads the fixed `layerfs` schema, +/// so a test run needs a database nothing else uses. +struct Database { + source: String, + name: String, +} + +impl Database { + fn create(source: &str) -> Self { + let name = format!( + "lfs_t105_ctl_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let mut admin = postgres::Client::connect(source, postgres::NoTls).unwrap(); + admin + .batch_execute(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .unwrap(); + admin + .batch_execute(&format!("CREATE DATABASE {name}")) + .unwrap(); + Self { + source: source.to_owned(), + name, + } + } + + fn url(&self) -> String { + database_url(&self.source, &self.name) + } +} + +impl Drop for Database { + fn drop(&mut self) { + if let Ok(mut admin) = postgres::Client::connect(&self.source, postgres::NoTls) { + let _ = admin.batch_execute(&format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + self.name + )); + } + } +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(out, "{byte:02x}"); + } + out +} + +fn op_id(byte: u8) -> String { + format!("op-{}", hex(&[byte; 16])) +} + +fn text(value: &SdkValue) -> &str { + match value { + SdkValue::Text(text) => text, + other => panic!("not text: {other:?}"), + } +} + +fn texts(value: &SdkValue) -> Vec { + match value { + SdkValue::List(items) => items + .iter() + .map(|item| { + if let SdkValue::Text(text) = item { + text.clone() + } else { + panic!("not text: {item:?}"); + } + }) + .collect(), + other => panic!("not a list: {other:?}"), + } +} + +/// What a compaction leaves in `gc_op`/`gc_intent`; `pid` and +/// `backend_start` name the owner the §9 rule tests for liveness. +struct OpRow<'a> { + op_id: [u8; 16], + phase: &'a str, + generation: i64, + started_ms: i64, + pid: i64, + backend_start: &'a str, +} + +fn insert(admin: &mut postgres::Client, row: &OpRow<'_>) { + let artifact = format!("lfs_gc_{}", hex(&row.op_id)); + admin + .execute( + "INSERT INTO layerfs.gc_op \ + (id,generation,phase,scratch_schema,mark_serial,mark_pack_id,mark_digest,\ + started_ms,artifact,pid,backend_start) \ + VALUES (1,$1,$2,$3,$4,$5,$6,$7,$3,$8,$9)", + &[ + &row.generation, + &row.phase, + &artifact, + &11i64, + &12i64, + &vec![9u8; 32], + &row.started_ms, + &row.pid, + &row.backend_start, + ], + ) + .unwrap(); + admin + .execute( + "INSERT INTO layerfs.gc_intent \ + (op_id,kind,scope_kind,owner_runtime,owner_boot,state,uncertain,phase,\ + generation,artifact,updated_ms) \ + VALUES ($1,'compaction','store',$2,$3,'live',FALSE,$4,$5,$6,$7)", + &[ + &row.op_id.to_vec(), + &"runtime-t105", + &"boot-t105", + &row.phase, + &row.generation, + &artifact, + &1_700_000_000_000i64, + ], + ) + .unwrap(); +} + +fn clear(admin: &mut postgres::Client) { + admin + .batch_execute( + "DELETE FROM layerfs.gc_op; \ + DELETE FROM layerfs.gc_intent WHERE kind='compaction'", + ) + .unwrap(); +} + +/// Waits until the recorded owner backend is gone from `pg_stat_activity`, +/// as the persistence fixture does. +fn wait_gone(admin: &mut postgres::Client, pid: i32, start: &str) { + for _ in 0..400 { + let live: bool = admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid=$1 AND backend_start::text=$2)", + &[&pid, &start], + ) + .unwrap() + .get(0); + if !live { + return; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + panic!("the recorded owner backend never ended"); +} + +/// The real control entry point: `store.status` for the Store in `home`. +fn status(home: &Path) -> SdkValue { + let name = StoreName::parse("pgs").unwrap(); + layerfs_store::sqlite::family::status(home, &name).unwrap() +} + +#[test] +fn postgresql_store_status_projects_busy_leftover_and_idle() { + let Some(source) = url() else { + return; + }; + let database = Database::create(&source); + let url = database.url(); + let handles = PostgresHandles::create( + &PostgresSelection::new(&url, "layerfs"), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: [71; 32], + incarnation: 7, + }, + ) + .unwrap(); + + let home = tempdir("home"); + home::ensure(&home).unwrap(); + // `store.connect` records exactly this shape; `store.status` reads it. + StoreRecord::connected("pgs", &url, "layerfs", "off", None) + .write(&home) + .unwrap(); + let mut admin = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + + // Idle: no compaction row, so no op and no phases. + let idle = status(&home); + assert_eq!(idle.get_text("name"), Some("pgs")); + assert_eq!(idle.get_text("kind"), Some("postgresql")); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!( + idle.get("op"), + Some(&SdkValue::Null), + "an idle Store has no op" + ); + assert_eq!( + idle.get("phases"), + Some(&SdkValue::Null), + "an idle Store has no phases" + ); + assert!( + matches!(idle.get("server"), Some(SdkValue::Map(_))), + "the server status is reported beside the op" + ); + + // A live owner: an open, non-idle backend is what the §9 dead-owner rule + // treats as still running the op. + let mut owner = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + owner.batch_execute("BEGIN; SELECT 1").unwrap(); + let (pid, start): (i32, String) = { + let row = owner + .query_one( + "SELECT pg_backend_pid(),backend_start::text FROM pg_stat_activity \ + WHERE pid=pg_backend_pid()", + &[], + ) + .unwrap(); + (row.get(0), row.get(1)) + }; + let started_ms = 1_700_000_000_000i64; + let live = OpRow { + op_id: [0xab; 16], + phase: "build", + generation: 7, + started_ms, + pid: i64::from(pid), + backend_start: &start, + }; + insert(&mut admin, &live); + + let capture = layerfs_persistence::compact_metrics::CompactMetricsCapture::start(); + let busy = status(&home); + assert_eq!(busy.get_text("state"), Some("busy")); + let op = busy.get("op").unwrap(); + assert_eq!( + op, + &SdkValue::map([ + ("id", SdkValue::text(op_id(0xab))), + ("name", SdkValue::text("store.compact")), + ("phase", SdkValue::text("build")), + ("started", SdkValue::Int(started_ms)), + ("uncertain", SdkValue::Bool(true)), + ]), + "the op is exactly the five §11 fields" + ); + assert_eq!(texts(busy.get("phases").unwrap()), PHASES); + assert!( + op.get("state").is_none() && op.get("phases").is_none(), + "state and phases sit beside the op" + ); + + // A gone owner leaves the same row: uncertain, and now unresolved. + owner.close().unwrap(); + wait_gone(&mut admin, pid, &start); + let leftover = status(&home); + assert_eq!(leftover.get_text("state"), Some("unresolved")); + let op = leftover.get("op").unwrap(); + assert_eq!(text(op.get("id").unwrap()), op_id(0xab)); + assert_eq!(op.get_text("name"), Some("store.compact")); + assert_eq!(op.get_text("phase"), Some("build")); + assert_eq!(op.get_int("started"), Some(started_ms)); + assert_eq!(op.get("uncertain"), Some(&SdkValue::Bool(true))); + assert_eq!(texts(leftover.get("phases").unwrap()), PHASES); + + // The two observed states emit the bounded `compact.status{state}` sample + // once each; idle emits none. + let metrics = capture.finish(); + let sample = |state: &str| { + metrics + .samples() + .iter() + .find(|sample| sample.name == "compact.status" && sample.label("state") == Some(state)) + .map_or(0, |sample| sample.value) + }; + assert_eq!(sample("busy"), 1); + assert_eq!(sample("unresolved"), 1); + assert_eq!(sample("idle"), 0, "an idle status is not a compaction op"); + + // Every recorded phase is projected under its §8 name; a dead owner is + // an unresolved leftover for each. + for (index, phase) in PHASES.iter().enumerate() { + clear(&mut admin); + let byte = u8::try_from(index).unwrap(); + insert( + &mut admin, + &OpRow { + op_id: [byte; 16], + phase, + generation: 7, + started_ms, + pid: 99_999_999, + backend_start: "2000-01-01 00:00:00+00", + }, + ); + let projected = status(&home); + assert_eq!(projected.get_text("state"), Some("unresolved")); + assert_eq!( + projected.get("op").unwrap().get_text("phase"), + Some(*phase), + "{phase}: the row records the phase" + ); + let expected_id = op_id(byte); + assert_eq!( + projected.get("op").unwrap().get_text("id"), + Some(expected_id.as_str()) + ); + assert_eq!(texts(projected.get("phases").unwrap()), PHASES); + } + + // A settled run removes the row: idle again, with no op and no phases. + clear(&mut admin); + let idle = status(&home); + assert_eq!(idle.get_text("state"), Some("idle")); + assert_eq!(idle.get("op"), Some(&SdkValue::Null)); + assert_eq!(idle.get("phases"), Some(&SdkValue::Null)); + + drop(admin); + drop(handles); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-store/tests/project_add_unknown.rs b/control/crates/layerfs-store/tests/project_add_unknown.rs new file mode 100644 index 0000000000..9179907dfc --- /dev/null +++ b/control/crates/layerfs-store/tests/project_add_unknown.rs @@ -0,0 +1,216 @@ +//! The round-4 #43 refusals: `project.init`'s lost-admission `Unknown` and +//! the shared `Unknown` detail fill (#43 Round 4, F43R4-1/F43R4-2). +//! +//! Product source holds no tests (the product-boundary guard), so +//! `project::add::add_failure` (already public) and +//! `unknown::named_unknown` (`#[doc(hidden)] pub`) are exercised here. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_history::records::{GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryError, OpConflict, UnknownOutcomeState}; +use layerfs_project::store::InitError; +use layerfs_project::{ProjectError, RetainedAcquisition}; +use layerfs_storage::port::acquisition::Owner; +use layerfs_store::project::add::add_failure; +use layerfs_store::unknown::named_unknown; + +fn unknown_state() -> UnknownOutcomeState { + UnknownOutcomeState::new( + GcScopeKind::Project, + "shop", + Some("init".to_owned()), + Some("00ff".repeat(8)), + ) +} + +fn assert_unknown_names_shop(refusal: &SdkRefusal) { + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(entity.get_text("name"), Some("shop"), "{refusal:?}"); + assert_eq!( + refusal.detail.get_text("phase"), + Some("init"), + "{refusal:?}" + ); + let expected = "00ff".repeat(8); + assert_eq!(refusal.detail.get_text("op"), Some(expected.as_str())); +} + +/// F43R4-2: Init's registration/release `UnknownOutcome` reaches the client +/// through the standard history refusal: `Unknown` naming the project, the +/// Init phase and the durable operation, not `Failed` with a null detail. +#[test] +fn a_lost_init_admission_is_unknown_naming_the_project_phase_and_op() { + let lost = HistoryError::UnknownOutcome.with_unknown(unknown_state()); + for error in [ + InitError::Project(ProjectError::History(lost.clone())), + InitError::History(lost.clone()), + ] { + let refusal = add_failure(error, None, "shop", "store"); + assert_unknown_names_shop(&refusal); + } +} + +/// The same arms keep a definite Init-intent refusal: a live Store-wide +/// operation (`BusyOp`) is `Busy` with the operation detail, never `Failed`. +#[test] +fn a_busy_init_intent_keeps_its_code_and_operation_detail() { + let conflict = OpConflict::new( + GcIntentKind::Compaction, + GcScopeKind::Store, + "store", + Some("compact".to_owned()), + "aa".repeat(8), + false, + ); + let error = InitError::History(HistoryError::BusyOp(Box::new(conflict))); + let refusal = add_failure(error, None, "shop", "store"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("store")); + assert_eq!(entity.get_text("name"), Some("store")); +} + +fn refused(code: &str, detail: SdkValue) -> SdkRefusal { + SdkRefusal { + code: code.to_owned(), + message: "the worker did not answer".to_owned(), + detail, + } +} + +fn named_store(refusal: &SdkRefusal) { + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("store"), "{refusal:?}"); + assert_eq!(entity.get_text("name"), Some("shop"), "{refusal:?}"); + assert!( + refusal.detail.get("phase").expect("phase") == &SdkValue::Null, + "{refusal:?}" + ); +} + +#[test] +fn a_null_detail_is_filled_with_the_entity() { + let refusal = named_unknown(refused("Unknown", SdkValue::Null), "store", "shop", None); + assert_eq!(refusal.code, "Unknown"); + named_store(&refusal); + assert!(refusal.detail.get("op").expect("op") == &SdkValue::Null); +} + +/// F43R4-1: a bare `UnknownOutcome` renders `SdkValue::map::<0>([])`, not +/// `Null`; the caller must fill that shape too. +#[test] +fn an_empty_map_detail_is_filled_with_the_entity_and_operation() { + let op = [0xab; 16]; + let refusal = named_unknown( + refused("Unknown", SdkValue::map::<0>([])), + "store", + "shop", + Some(&op), + ); + assert_eq!(refusal.code, "Unknown"); + named_store(&refusal); + let expected = "ab".repeat(16); + assert_eq!(refusal.detail.get_text("op"), Some(expected.as_str())); +} + +#[test] +fn a_detail_that_already_names_something_is_kept() { + let detail = SdkValue::map([( + "entity", + SdkValue::map([ + ("kind", SdkValue::text("store")), + ("name", SdkValue::text("other")), + ]), + )]); + let refusal = named_unknown(refused("Unknown", detail.clone()), "store", "shop", None); + assert_eq!(refusal.detail, detail); +} + +#[test] +fn a_definite_refusal_is_kept() { + let refusal = named_unknown( + refused("Busy", SdkValue::map::<0>([])), + "store", + "shop", + None, + ); + assert_eq!(refusal.code, "Busy"); + assert!(matches!(refusal.detail, SdkValue::Map(fields) if fields.is_empty())); +} + +/// F43R5-1: a bare `UnknownOutcome` from any Init history call (the name +/// check, the binding hints, the stack publication or the first-Branch fork) +/// reaches the client as `Unknown` naming the project and the Init phase, +/// with no durable operation, never as `Unknown {}`. +#[test] +fn a_bare_init_unknown_outcome_names_the_project_and_init_phase() { + for error in [ + InitError::Project(ProjectError::History(HistoryError::UnknownOutcome)), + InitError::History(HistoryError::UnknownOutcome), + ] { + let refusal = add_failure(error, None, "shop", "store"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(entity.get_text("name"), Some("shop"), "{refusal:?}"); + assert_eq!( + refusal.detail.get_text("phase"), + Some("init"), + "{refusal:?}" + ); + assert!( + refusal.detail.get("op").expect("op") == &SdkValue::Null, + "{refusal:?}" + ); + } +} + +fn uncertain(cause: ProjectError) -> InitError { + InitError::Project(ProjectError::Uncertain { + cause: Box::new(cause), + retained: RetainedAcquisition { + owner: Owner { + operation: 1, + epoch: 2, + }, + work: None, + }, + }) +} + +/// F43R6-1: `init.rs`'s `reserve_inodes` failure carries an `UnknownOutcome` +/// whose context `backing.settle` retains as `ProjectError::Uncertain { cause }`; +/// `add_failure` still reaches the client as `Unknown` naming the project, +/// the Init phase and the durable operation, never `Failed` with a null +/// detail. +#[test] +fn a_lost_reserve_inodes_is_unknown_through_uncertain() { + let error = uncertain(ProjectError::History( + HistoryError::UnknownOutcome.with_unknown(unknown_state()), + )); + let refusal = add_failure(error, None, "shop", "store"); + assert_unknown_names_shop(&refusal); +} + +/// The new `Uncertain` arm keeps a definite history cause's own code: it does +/// not become `Unknown` merely because the acquisition retained it. +#[test] +fn a_definite_history_cause_in_uncertain_keeps_its_code() { + let conflict = OpConflict::new( + GcIntentKind::Compaction, + GcScopeKind::Store, + "store", + Some("compact".to_owned()), + "aa".repeat(8), + false, + ); + let error = uncertain(ProjectError::History(HistoryError::BusyOp(Box::new( + conflict, + )))); + let refusal = add_failure(error, None, "shop", "store"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("store")); + assert_eq!(entity.get_text("name"), Some("store")); +} diff --git a/control/crates/layerfs-store/tests/project_delete.rs b/control/crates/layerfs-store/tests/project_delete.rs new file mode 100644 index 0000000000..e5540806c3 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete.rs @@ -0,0 +1,762 @@ +//! `project.delete`'s home path (T4.5b): the durable intent before the marker, +//! the retry classifications (`Marked`, `Retired`, `Absent`, a hidden Store +//! project), the `home_entry_removed` echo, and the worker's own entry. A fake +//! `Effects` stands in for the Store so the home state machine is tested +//! without opening one. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::home::{paths, write_record}; +use layerfs_home::names::StoreName; +use layerfs_home::outcome::{failed, refused}; +use layerfs_store::project::delete::{ + delete_with, remove_project_entry, remove_worker_entry, Admission, Effects, Intent, IntentState, +}; +use layerfs_store::project::frame::DeleteEntryFrame; +use layerfs_store_service::history::deletion::Deletion; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +fn home(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-project-delete-{tag}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&directory); + layerfs_home::home::layout::create(&directory).unwrap(); + directory +} + +fn project_record(home: &Path, store: &str, project: &str, stack: LayerStackId) { + write_record( + &paths::project_record(home, store, project), + &SdkValue::map([ + ("v", SdkValue::Int(1)), + ("name", t(project)), + ("stack", t(&hex(&stack.to_bytes()))), + ("created", SdkValue::Int(1)), + ]), + ) + .unwrap(); +} + +fn intent(_home: &Path, store: &str, project: &str, stack: LayerStackId, id: i64) -> Intent { + Intent { + store: store.to_owned(), + project: project.to_owned(), + stack, + deleted_ms: id, + attempt_ms: 1, + state: IntentState::Pending, + op: None, + } +} + +/// A fake home/worker: the Store side and entry remover, with the project +/// scope admission a live mount or operation would refuse. `mark` reads the +/// intent as the marker would see it, so a test can see the state that +/// preceded it. +struct Fake { + home: PathBuf, + store: String, + project: String, + outcomes: VecDeque, + seen: Vec<(LayerStackId, i64)>, + states: Vec, + prepared: usize, + fail_removal: bool, + /// What `admit` refuses with, before registering anything. + refuse: Option, + /// What `mark` refuses with, before writing anything. + mark_refusal: Option, + /// Admissions taken, in order. + admitted: Vec, + /// The project name each admission carried, in order. + projects: Vec, + /// Admissions released, in order. + released: Vec, + /// The `pending_retry` flag each marker attempt carried, in order. + pending_retries: Vec, + /// The Stacks whose worker-side entry a `marked` sidecar asked to remove. + settled: Vec, +} + +impl Fake { + fn new(home: &Path, store: &str, project: &str, outcomes: Vec) -> Self { + Self { + home: home.to_path_buf(), + store: store.to_owned(), + project: project.to_owned(), + outcomes: outcomes.into(), + seen: Vec::new(), + states: Vec::new(), + prepared: 0, + fail_removal: false, + refuse: None, + mark_refusal: None, + admitted: Vec::new(), + projects: Vec::new(), + released: Vec::new(), + pending_retries: Vec::new(), + settled: Vec::new(), + } + } + + fn failing(mut self) -> Self { + self.fail_removal = true; + self + } + + /// A Store with a live mount (or another live operation) at the project + /// scope: `admit` refuses with the same code the Store's admission does. + fn refusing(mut self, code: &str, detail: SdkValue) -> Self { + self.refuse = Some(SdkRefusal { + code: code.to_owned(), + message: format!("test admission refusal: {code}"), + detail, + }); + self + } + + /// A fake whose admission refuses. + fn refused(&self) -> bool { + self.refuse.is_some() + } +} + +impl Effects for Fake { + fn prepare(&mut self) -> Result<(), SdkRefusal> { + self.prepared += 1; + Ok(()) + } + + fn admit( + &mut self, + stack: LayerStackId, + op: [u8; 16], + project: &str, + ) -> Result { + if self.refused() { + return Err(self.refuse.clone().expect("refused above")); + } + // The admission row is stored: it must name the project, so a later + // caller meeting it reports `{kind: "project", name: }`. + self.projects.push(project.to_owned()); + // The sidecar names this admission and is durable before the row is + // taken, so a crash between the row and its reply leaves it for the + // retry. + let pending = Intent::read(&self.home, &self.store, &self.project) + .unwrap() + .expect("the sidecar precedes the admission"); + assert_eq!(pending.op, Some(op), "the sidecar names this admission"); + let admission = Admission::new(stack, op); + self.admitted.push(admission); + Ok(admission) + } + + fn release(&mut self, admission: Admission) -> Result<(), SdkRefusal> { + self.released.push(admission); + Ok(()) + } + + fn mark( + &mut self, + admission: Admission, + deleted_ms: i64, + pending_retry: bool, + ) -> Result { + if let Some(refusal) = &self.mark_refusal { + return Err(refusal.clone()); + } + self.pending_retries.push(pending_retry); + self.seen.push((admission.stack, deleted_ms)); + let state = Intent::read(&self.home, &self.store, &self.project) + .unwrap() + .expect("the intent is durable before the marker") + .state; + self.states.push(state); + Ok(self + .outcomes + .pop_front() + .expect("one fake outcome per marker attempt")) + } + + fn settle(&mut self, stack: LayerStackId) -> Result<(), SdkRefusal> { + self.settled.push(stack); + Ok(()) + } + + fn remove_entry(&mut self, entry: &Path) -> Result<(), SdkRefusal> { + if self.fail_removal { + return Err(refused("Failed", "test removal failure")); + } + match std::fs::remove_dir_all(entry) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(failed(error)), + } + } +} + +#[test] +fn a_fresh_delete_marks_then_removes_the_entry_and_unlinks_the_intent() { + let home = home("fresh"); + let (store, project, stack) = ("shop", "app", stack(1)); + project_record(&home, store, project, stack); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 777 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_text("project"), Some("shop/app")); + assert_eq!(outcome.get_int("deleted_ms"), Some(777)); + assert!(outcome.get_bool("home_entry_removed")); + assert_eq!(fake.seen.len(), 1); + assert_eq!(fake.seen[0].0, stack); + assert_eq!(fake.pending_retries, [false], "a fresh delete is no retry"); + // the identity save precedes the intent, and the intent precedes the mark + assert_eq!(fake.prepared, 1); + assert_eq!(fake.states, [IntentState::Pending]); + // the project scope was admitted after the sidecar and released after + assert_eq!(fake.admitted.len(), 1); + assert_eq!( + fake.projects, + ["app".to_owned()], + "the admission names the project, not the Stack id" + ); + assert_eq!(fake.released, fake.admitted); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_refused_admission_keeps_a_pending_sidecar_and_writes_no_marker() { + let refused_home = home("refused"); + let (store, project, stack) = ("shop", "app", stack(20)); + project_record(&refused_home, store, project, stack); + let holders = SdkValue::map([( + "holders", + SdkValue::List(vec![SdkValue::map([ + ("kind", t("mount")), + ("entity", t("main")), + ])]), + )]); + let mut fake = + Fake::new(&refused_home, store, project, Vec::new()).refusing("MountsActive", holders); + let refusal = delete_with(&refused_home, store, project, Some(stack), &mut fake).unwrap_err(); + // the live mount's refusal is the call's, with its holders + assert_eq!(refusal.code, "MountsActive"); + assert!(refusal.detail.get("holders").is_some()); + // the identity save and the sidecar precede the admission; no marker + assert_eq!(fake.prepared, 1); + assert!(fake.seen.is_empty(), "{:?}", fake.seen); + let pending = Intent::read(&refused_home, store, project) + .unwrap() + .expect("the pending sidecar precedes the admission"); + assert_eq!(pending.state, IntentState::Pending); + let op = pending.op.expect("the sidecar names its admission"); + // the refused admission registered nothing; the caller settles the + // identity it wrote without adopting a row + assert!(fake.admitted.is_empty()); + assert_eq!(fake.released, [Admission::new(stack, op)]); + // the home entry and the project's record stay + assert!(paths::project_dir(&refused_home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&refused_home); + + // a live operation at the project scope is `Busy`, with the same rule + let busy_home = home("busy"); + project_record(&busy_home, store, project, stack); + let conflict = SdkValue::map([ + ("kind", t("removal")), + ("entity", t("app")), + ("phase", SdkValue::Null), + ("op", t("11")), + ("uncertain", SdkValue::Bool(false)), + ]); + let mut fake = Fake::new(&busy_home, store, project, Vec::new()).refusing("Busy", conflict); + let refusal = delete_with(&busy_home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + assert_eq!(fake.prepared, 1); + assert!(fake.seen.is_empty()); + assert_eq!( + Intent::read(&busy_home, store, project) + .unwrap() + .expect("the pending sidecar stays") + .state, + IntentState::Pending + ); + assert!(paths::project_dir(&busy_home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&busy_home); +} + +#[test] +fn a_retry_reuses_the_sidecar_admission_identity() { + let home = home("reuse"); + let (store, project, stack) = ("shop", "app", stack(22)); + project_record(&home, store, project, stack); + let mut pending = intent(&home, store, project, stack, 111); + pending.op = Some([3; 16]); + Intent::write(&home, &pending).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 111 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + // the retry re-registers the recorded identity, then releases it + assert_eq!(fake.admitted, [Admission::new(stack, [3; 16])]); + assert_eq!(fake.released, fake.admitted); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_failed_marker_releases_the_admission_it_took() { + let home = home("marker-refused"); + let (store, project, stack) = ("shop", "app", stack(21)); + project_record(&home, store, project, stack); + let mut fake = Fake::new(&home, store, project, Vec::new()); + fake.mark_refusal = Some(refused("Busy", "the project is held")); + let refusal = delete_with(&home, store, project, Some(stack), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "Busy"); + // the admission taken after the sidecar is released on the failure + assert_eq!(fake.admitted.len(), 1); + assert_eq!(fake.released, fake.admitted); + // the sidecar stays `pending`, naming the admission, and the entry stays + let left = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!(left.state, IntentState::Pending); + assert!(left.op.is_some(), "the sidecar names the admission"); + assert!(paths::project_dir(&home, store, project).is_dir()); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry() { + let home = home("retired"); + let (store, project, stack) = ("shop", "app", stack(2)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 222 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(fake.pending_retries, [true], "a pending sidecar is a retry"); + assert_eq!(outcome.get_int("deleted_ms"), Some(222)); + assert!( + outcome.get_bool("home_entry_removed"), + "the marker was already there" + ); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_live_row_with_a_pending_intent_settles_the_marker_and_keeps_the_entry() { + let home = home("marked"); + let (store, project, stack) = ("shop", "app", stack(3)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 111 }], + ); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + // The sidecar was written before an earlier admission, the row is still + // live and there is no marker: the retry redoes the marker with the same + // deleted_ms, settles the sidecar and keeps the entry. + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(fake.states, [IntentState::Pending]); + assert_eq!(fake.pending_retries, [true]); + assert!(fake.admitted[0].op.iter().any(|byte| *byte != 0)); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert!(!outcome.get_bool("home_entry_removed")); + assert!( + paths::project_dir(&home, store, project).is_dir(), + "the entry stays" + ); + let settled = Intent::read(&home, store, project) + .unwrap() + .expect("the sidecar is marked for the finishing call"); + assert_eq!(settled.state, IntentState::Marked); + assert_eq!(settled.deleted_ms, 111); + + // The marker is confirmed now: the next call finds a `marked` sidecar, + // takes no admission and writes no marker, and only finishes the entry. + let mut finish = Fake::new(&home, store, project, Vec::new()); + let outcome = delete_with(&home, store, project, Some(stack), &mut finish).unwrap(); + assert!(finish.seen.is_empty(), "a marked sidecar marks nothing"); + assert!( + finish.admitted.is_empty(), + "a marked sidecar admits nothing" + ); + assert_eq!(finish.settled, [stack]); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn an_absent_stack_finishes_a_stale_entry() { + let home = home("absent"); + let (store, project, stack) = ("shop", "app", stack(4)); + project_record(&home, store, project, stack); + Intent::write(&home, &intent(&home, store, project, stack, 111)).unwrap(); + let mut fake = Fake::new(&home, store, project, vec![Deletion::Absent]); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(fake.seen, [(stack, 111)]); + assert_eq!(outcome.get_int("deleted_ms"), Some(111)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_fresh_delete_of_a_retired_project_is_no_such_project() { + let home = home("retired-fresh"); + let (store, project, stack) = ("shop", "app", stack(5)); + project_record(&home, store, project, stack); + // Another home holds the stale record but not the sidecar of the delete + // that retired the Stack: the record's Stack id is not a live project. + // The local route resolves the Store's listing, so it passes no Stack. + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 333 }], + ); + let refusal = delete_with(&home, store, project, None, &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject"); + assert!( + fake.seen.is_empty(), + "nothing is marked for a missing project" + ); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + assert!(paths::project_dir(&home, store, project).is_dir()); + + // A caller that hands over the record's Stack anyway (a worker route + // cannot list) is trusted only by the marker: `Retired` without a + // durable intent is the same `NoSuchProject`, with no entry removed. + let mut marked = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 333 }], + ); + let refusal = delete_with(&home, store, project, Some(stack), &mut marked).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject"); + assert_eq!(marked.seen.len(), 1); + assert_eq!(marked.seen[0].0, stack); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + assert!(paths::project_dir(&home, store, project).is_dir()); + + // The same-home retry has the durable sidecar: it still removes the + // stale entry of a Store project that no longer lists the name. + Intent::write(&home, &intent(&home, store, project, stack, 333)).unwrap(); + let mut retry = Fake::new( + &home, + store, + project, + vec![Deletion::Retired { deleted_ms: 333 }], + ); + let outcome = delete_with(&home, store, project, None, &mut retry).unwrap(); + assert_eq!(outcome.get_int("deleted_ms"), Some(333)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_home_removal_failure_is_success_false_and_leaves_a_marked_intent() { + let home = home("residue"); + let (store, project, stack) = ("shop", "app", stack(6)); + project_record(&home, store, project, stack); + let mut fake = Fake::new( + &home, + store, + project, + vec![Deletion::Marked { deleted_ms: 44 }], + ) + .failing(); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(!outcome.get_bool("home_entry_removed")); + // the marker is the commit point; the entry and intent stay for the retry + assert!(paths::project_dir(&home, store, project).is_dir()); + let left = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!(left.state, IntentState::Marked); + assert_eq!(left.deleted_ms, 44); + + // a same-home retry of the marked sidecar takes no admission and writes + // no marker: it removes the entry and returns `home_entry_removed: true` + let mut retry = Fake::new(&home, store, project, Vec::new()); + let outcome = delete_with(&home, store, project, Some(stack), &mut retry).unwrap(); + assert!(retry.seen.is_empty(), "a marked sidecar marks nothing"); + assert!(retry.admitted.is_empty(), "a marked sidecar admits nothing"); + assert_eq!(retry.settled, [stack], "the worker entry is settled once"); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, store, project).exists()); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_done_intent_settles_the_entry_without_a_second_marker() { + let home = home("done"); + let (store, project, stack) = ("shop", "app", stack(7)); + project_record(&home, store, project, stack); + let mut done = intent(&home, store, project, stack, 44); + done.state = IntentState::Done; + Intent::write(&home, &done).unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + let outcome = delete_with(&home, store, project, Some(stack), &mut fake).unwrap(); + assert!(fake.seen.is_empty(), "a done intent marks nothing"); + assert!(fake.settled.is_empty(), "a done intent makes no Store call"); + assert_eq!(outcome.get_int("deleted_ms"), Some(44)); + assert!(outcome.get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_stale_intent_does_not_delete_a_project_the_listing_names_again() { + let home = home("recreated-listed"); + let (store, project, old, new) = ("shop", "app", stack(8), stack(9)); + // The re-created project has no readable home record yet (a partial + // creation or a stale entry), but its Store lists the new Stack: the + // listing is enough to see that the sidecar is stale. The stale intent + // goes, and the call is `NoSuchProject`: the re-created project is not + // deleted by it. + Intent::write(&home, &intent(&home, store, project, old, 111)).unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + let refusal = delete_with(&home, store, project, Some(new), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject", "{refusal:?}"); + assert!(fake.seen.is_empty(), "a stale intent marks nothing"); + assert!(fake.admitted.is_empty(), "a stale intent admits nothing"); + assert_eq!( + fake.prepared, 0, + "nothing of the re-created project is touched" + ); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_recreated_project_discards_the_stale_intent_and_keeps_its_entry() { + let home = home("recreated"); + let (store, project, old, new) = ("shop", "app", stack(8), stack(9)); + project_record(&home, store, project, new); + Intent::write(&home, &intent(&home, store, project, old, 111)).unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + let refusal = delete_with(&home, store, project, Some(new), &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject", "{refusal:?}"); + assert!( + fake.seen.is_empty(), + "the record that is there now is not marked" + ); + assert!(fake.admitted.is_empty()); + assert!( + paths::project_dir(&home, store, project).is_dir(), + "the re-created project's entry stays" + ); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn an_untracked_record_with_no_stack_is_no_such_project() { + let home = home("untracked"); + let (store, project) = ("shop", "app"); + write_record( + &paths::project_record(&home, store, project), + &SdkValue::map([("v", SdkValue::Int(1)), ("name", t(project))]), + ) + .unwrap(); + let mut fake = Fake::new(&home, store, project, Vec::new()); + // Nothing lists the record and there is no sidecar: no project, and no + // legacy identity is saved for one that is gone. + let refusal = delete_with(&home, store, project, None, &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject"); + assert!(fake.seen.is_empty()); + assert_eq!(fake.prepared, 0); + assert!(paths::project_dir(&home, store, project).is_dir()); + + // no record and no intent is NoSuchProject too + let refusal = delete_with(&home, "shop", "gone", None, &mut fake).unwrap_err(); + assert_eq!(refusal.code, "NoSuchProject"); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn the_intent_moves_pending_marked_done_and_unlinks() { + let home = home("intent"); + let (store, project, stack) = ("shop", "app", stack(10)); + let mut value = intent(&home, store, project, stack, 5); + Intent::write(&home, &value).unwrap(); + let path = Intent::path(&home, store, project); + assert!(path.is_file()); + assert_eq!( + Intent::read(&home, store, project).unwrap().unwrap().state, + IntentState::Pending + ); + // no temporary file is left beside it + let mut names: Vec = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + assert_eq!(names, [format!("{project}.json")]); + + value.state = IntentState::Marked; + value.deleted_ms = 6; + value.op = Some([4; 16]); + Intent::write(&home, &value).unwrap(); + let read = Intent::read(&home, store, project).unwrap().unwrap(); + assert_eq!((read.state, read.deleted_ms), (IntentState::Marked, 6)); + assert_eq!(read.op, Some([4; 16]), "the admission identity persists"); + + value.state = IntentState::Done; + Intent::write(&home, &value).unwrap(); + assert_eq!( + Intent::read(&home, store, project).unwrap().unwrap().state, + IntentState::Done + ); + Intent::unlink(&home, store, project).unwrap(); + assert_eq!(Intent::read(&home, store, project).unwrap(), None); + // unlinking an absent intent is not an error + Intent::unlink(&home, store, project).unwrap(); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn the_entry_frame_removes_only_the_recorded_entry() { + let root = home("worker-entry-frame"); + let (store, first) = ("shop", stack(13)); + project_record(&root, store, "app", first); + project_record(&root, store, "other", stack(14)); + let frame = DeleteEntryFrame { + store: StoreName::parse(store).unwrap(), + stack: first, + project: Some("app".to_owned()), + }; + let answer = remove_project_entry(&root, &frame).unwrap(); + assert_eq!(answer.get_text("kind"), Some("Removed")); + assert_eq!(answer.get_text("project"), Some("app")); + assert!(!paths::project_dir(&root, store, "app").exists()); + assert!(paths::project_dir(&root, store, "other").is_dir()); + // a retry finds the entry gone and answers the same, idempotently + let answer = remove_project_entry(&root, &frame).unwrap(); + assert_eq!(answer.get_text("project"), Some("app")); + assert!( + paths::project_dir(&root, store, "other").is_dir(), + "the other entry is untouched" + ); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn the_worker_removes_only_its_entry_for_the_recorded_stack() { + let root = home("worker-entry"); + let (store, first) = ("shop", stack(11)); + project_record(&root, store, "app", first); + project_record(&root, store, "other", stack(12)); + // the hint names the caller's project; the other entry stays + let removed = remove_worker_entry(&root, store, first, Some("app")).unwrap(); + assert_eq!(removed.as_deref(), Some("app")); + assert!(!paths::project_dir(&root, store, "app").exists()); + assert!(paths::project_dir(&root, store, "other").is_dir()); + // without a hint the recorded stack id finds the entry + let removed = remove_worker_entry(&root, store, stack(12), None).unwrap(); + assert_eq!(removed.as_deref(), Some("other")); + assert!(!paths::project_dir(&root, store, "other").exists()); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn a_worker_reply_names_the_stack_and_deleted_ms() { + use layerfs_store::project::frame::DeleteReply; + let first = stack(13); + let reply = DeleteReply { + project: "app".to_owned(), + stack: first, + deleted_ms: 42, + retired: Some(true), + }; + let encoded = reply.encode(); + assert_eq!(encoded.get_text("kind"), Some("Removed")); + assert!(encoded.get_bool("retired")); + assert_eq!( + encoded.get_text("stack_id"), + Some(hex(&first.to_bytes()).as_str()) + ); + assert_eq!( + DeleteReply::decode(&encoded).unwrap(), + reply, + "the worker reply round-trips" + ); + // a reply for a different Stack is detectable + let other = DeleteReply { + stack: stack(14), + ..reply.clone() + }; + assert_ne!(DeleteReply::decode(&other.encode()).unwrap().stack, first); +} + +/// F43R5-2: `project.delete` resolves a missing Stack id from the Store's +/// listing. A listing that cannot be determined is a bare `UnknownOutcome` +/// (empty-map detail) or a poisoned Store object (null detail); both must +/// name the project, never stay a bare `Unknown {}`. +#[test] +fn a_listing_unknown_names_the_project() { + use layerfs_history::HistoryError; + use layerfs_store::project::delete::listed_unknown; + use layerfs_store_service::history::lookup::published; + use layerfs_store_service::store::PortError; + use std::sync::Arc; + + for refusal in [ + published(Arc::new(PortError::History(HistoryError::UnknownOutcome))), + published(Arc::new(PortError::Poisoned)), + ] { + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + let refusal = listed_unknown(refusal, "shop"); + assert_eq!(refusal.code, "Unknown", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project"), "{refusal:?}"); + assert_eq!(entity.get_text("name"), Some("shop"), "{refusal:?}"); + assert!( + refusal.detail.get("phase").expect("phase") == &SdkValue::Null, + "{refusal:?}" + ); + assert!( + refusal.detail.get("op").expect("op") == &SdkValue::Null, + "{refusal:?}" + ); + } +} diff --git a/control/crates/layerfs-store/tests/project_delete_frames.rs b/control/crates/layerfs-store/tests/project_delete_frames.rs new file mode 100644 index 0000000000..2b0d218d6b --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete_frames.rs @@ -0,0 +1,266 @@ +//! The `project.delete` worker frame and its reply: the map the caller +//! builds, the fields the worker decodes, and the reply it sends back. +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::names::StoreName; +use layerfs_store::project::frame::{ + DeleteAdmitFrame, DeleteEntryFrame, DeleteEntryReply, DeleteFrame, DeleteReleaseFrame, + DeleteReply, +}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn map(fields: Vec<(&str, SdkValue)>) -> SdkValue { + SdkValue::Map(fields.into_iter().map(|(n, v)| (n.to_owned(), v)).collect()) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +#[test] +fn the_frame_names_the_store_stack_and_optional_context() { + let stack = stack(1); + let op = [7u8; 16]; + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let frame = DeleteFrame { + store: StoreName::parse("shop").unwrap(), + project: Some("app".to_owned()), + stack, + deleted_ms: Some(5), + op_id: Some(op), + pending_retry: Some(true), + store_record: Some(record.clone()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(5)), + ("project", t("app")), + ("op_id", t(&hex(&op))), + ("pending_retry", SdkValue::Bool(true)), + ("store_record", record), + ]) + ); + let decoded = DeleteFrame::decode(&encoded).unwrap(); + assert_eq!(decoded.store, frame.store); + assert_eq!(decoded.project, frame.project); + assert_eq!(decoded.stack, stack); + assert_eq!(decoded.deleted_ms, Some(5)); + assert_eq!(decoded.op_id, Some(op)); + assert_eq!(decoded.pending_retry, Some(true)); + assert_eq!(decoded.store_record, frame.store_record); +} + +#[test] +fn a_frame_with_only_the_store_and_stack_decodes() { + let stack = stack(2); + let minimal = map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]); + let decoded = DeleteFrame::decode(&minimal).unwrap(); + assert_eq!((decoded.project, decoded.deleted_ms), (None, None)); + assert_eq!(decoded.op_id, None); + assert_eq!(decoded.pending_retry, None); + assert_eq!(decoded.store_record, None); + assert_eq!(decoded.stack, stack); +} + +#[test] +fn the_admission_and_release_frames_round_trip() { + let stack = stack(4); + let op = [9u8; 16]; + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let admit = DeleteAdmitFrame { + store: StoreName::parse("shop").unwrap(), + stack, + op, + project: Some("app".to_owned()), + store_record: Some(record.clone()), + }; + let encoded = admit.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t(&hex(&op))), + ("project", t("app")), + ("store_record", record.clone()), + ]) + ); + assert_eq!(DeleteAdmitFrame::decode(&encoded).unwrap(), admit); + + let release = DeleteReleaseFrame { + store: StoreName::parse("shop").unwrap(), + op, + store_record: None, + }; + let encoded = release.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("op_id", t(&hex(&op)))]) + ); + assert_eq!(DeleteReleaseFrame::decode(&encoded).unwrap(), release); +} + +#[test] +fn an_unnamed_admit_frame_is_the_older_caller() { + // The caller that does not know the project's name omits `project`; the + // worker fills it from the Stack record, so the frame still decodes. + let stack = stack(6); + let op = [5u8; 16]; + let frame = map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t(&hex(&op))), + ]); + let decoded = DeleteAdmitFrame::decode(&frame).unwrap(); + assert_eq!(decoded.project, None); + assert_eq!((decoded.stack, decoded.op), (stack, op)); + assert_eq!(decoded.store_record, None); + // and the encode of an unnamed frame leaves the field out + assert_eq!(decoded.encode(), frame); +} + +#[test] +fn the_admission_frames_refuse_a_bad_operation_identity() { + let decode = |fields| DeleteAdmitFrame::decode(&map(fields)).unwrap_err().message; + let stack = stack(5); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("zz")), + ]), + "`op_id` must be hex digits" + ); + assert_eq!( + decode(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("op_id", t("ab")), + ]), + "`op_id` must be an operation identity (16 bytes)" + ); + assert_eq!( + delete_release_decode(&map(vec![("store", t("shop"))])), + "missing text argument `op_id`" + ); +} + +fn delete_release_decode(frame: &SdkValue) -> String { + DeleteReleaseFrame::decode(frame).unwrap_err().message +} + +#[test] +fn the_worker_refuses_a_bad_frame_in_order() { + let decode = |fields| DeleteFrame::decode(&map(fields)).unwrap_err().message; + assert_eq!(decode(vec![]), "missing text argument `store`"); + assert!(decode(vec![("store", t("../x"))]).starts_with("store name")); + assert_eq!( + decode(vec![("store", t("shop"))]), + "missing text argument `stack_id`" + ); + assert_eq!( + decode(vec![("store", t("shop")), ("stack_id", t("zz"))]), + "`stack_id` must be hex digits" + ); + assert_eq!( + decode(vec![("store", t("shop")), ("stack_id", t("ab"))]), + "`stack_id` must be a LayerStack identity (17 bytes)" + ); +} + +#[test] +fn the_entry_frame_and_reply_round_trip() { + let stack = stack(7); + let frame = DeleteEntryFrame { + store: StoreName::parse("shop").unwrap(), + stack, + project: Some("app".to_owned()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("project", t("app")), + ]) + ); + assert_eq!(DeleteEntryFrame::decode(&encoded).unwrap(), frame); + // an unnamed caller omits `project`, and the encode leaves it out + let unnamed = map(vec![ + ("store", t("shop")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]); + let decoded = DeleteEntryFrame::decode(&unnamed).unwrap(); + assert_eq!(decoded.project, None); + assert_eq!(decoded.stack, stack); + assert_eq!(decoded.encode(), unnamed); + + let reply = DeleteEntryReply { + project: "app".to_owned(), + stack, + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Removed")), + ("project", t("app")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]) + ); + assert_eq!(DeleteEntryReply::decode(&encoded).unwrap(), reply); + let other = map(vec![("kind", t("UpToDate"))]); + assert_eq!( + DeleteEntryReply::decode(&other).unwrap_err().message, + "the worker project.delete entry reply is not `Removed`" + ); +} + +#[test] +fn the_reply_round_trips_and_requires_removed() { + let stack = stack(3); + let reply = DeleteReply { + project: "app".to_owned(), + stack, + deleted_ms: 42, + retired: Some(true), + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Removed")), + ("project", t("app")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(42)), + ("retired", SdkValue::Bool(true)), + ]) + ); + assert_eq!(DeleteReply::decode(&encoded).unwrap(), reply); + // An older worker's reply without `retired` decodes as a fresh marker. + let older = DeleteReply::decode(&map(vec![ + ("kind", t("Removed")), + ("project", t("app")), + ("stack_id", t(&hex(&stack.to_bytes()))), + ("deleted_ms", SdkValue::Int(42)), + ])) + .unwrap(); + assert_eq!(older.retired, None); + let other = map(vec![("kind", t("UpToDate"))]); + assert_eq!( + DeleteReply::decode(&other).unwrap_err().message, + "the worker project.delete reply is not `Removed`" + ); +} diff --git a/control/crates/layerfs-store/tests/project_delete_store.rs b/control/crates/layerfs-store/tests/project_delete_store.rs new file mode 100644 index 0000000000..b0f3b16bb6 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_delete_store.rs @@ -0,0 +1,333 @@ +//! `project.delete`'s home path against a real local Store (T4.6): the +//! control layer writes its sidecar naming the admission, then admits the +//! project scope through `gc_register`/`gc_release`, so a live mount refuses +//! with `MountsActive` (its holders) and a live operation with `Busy`; a +//! refusal leaves a pending sidecar, no marker and no leaked admission +//! intent. +use layerfs_bridge::control::SdkValue; +use layerfs_history::records::{GcIntent, GcIntentKind, GcScopeKind}; +use layerfs_history::{HistoryName, LayerStackId, NamedSnapshot}; +use layerfs_home::events::Events; +use layerfs_home::home::{layout, paths}; +use layerfs_home::names::{ProjectRef, StoreName}; +use layerfs_store::open_args::OpenArgs; +use layerfs_store::project::add::{add_project, ProjectAdd}; +use layerfs_store::project::delete::{admit_project, delete, Intent}; +use layerfs_store::project::frame::DeleteAdmitFrame; +use layerfs_store::record::StoreRecord; +use layerfs_store::service::with_service; +use layerfs_store::sqlite::create::create; +use layerfs_store::Related; +use layerfs_store_service::history::lookup; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + layout::create(&directory).unwrap(); + directory +} + +/// A folder to initialize a project from: no home layout inside it. +fn plaindir(tag: &str) -> PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-delete-store-folder-{tag}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// True when the sandbox filesystem cannot hold a directly opened Store, so +/// these real-Store cases cannot run here. The fake-home cases in +/// `project_delete` cover the same home path without a Store. +fn no_direct_store(refusal: &layerfs_bridge::control::SdkRefusal) -> bool { + refusal.code == "Unsupported" && refusal.message.contains("does not carry SQLite locks") +} + +/// A home whose Store `shop` holds the live project `app`, and the +/// `OpenArgs` (with the Store record) that reach it. `None` when this +/// filesystem cannot hold a directly opened Store. +fn shop(tag: &str) -> Option<(PathBuf, OpenArgs, LayerStackId)> { + let home = tempdir(tag); + match create(&home, &StoreName::parse("shop").unwrap()) { + Ok(_) => {} + Err(refusal) if no_direct_store(&refusal) => { + eprintln!("NOT RUN: this filesystem cannot hold a directly opened Store: {refusal:?}"); + let _ = std::fs::remove_dir_all(&home); + return None; + } + Err(refusal) => panic!("store.create: {refusal:?}"), + } + let source = plaindir(tag); + std::fs::write(source.join("a.txt"), b"a\n").unwrap(); + add_project(&ProjectAdd { + home: home.clone(), + store: StoreName::parse("shop").unwrap(), + name: "app".to_owned(), + source, + timeout_ms: Some(60_000), + store_record: None, + }) + .unwrap(); + let record = StoreRecord::read(&home, "shop").unwrap().unwrap(); + let open = OpenArgs { + home: Some(home.clone()), + store: Some("shop".to_owned()), + project: Some("app".to_owned()), + store_path: Some(paths::store_file(&home, "shop").display().to_string()), + store_record: Some(record.to_record().clone()), + postgresql: None, + }; + let stack = with_service(&open, |service| { + Ok(lookup::project(&service.store, "app").unwrap().id) + }) + .unwrap(); + Some((home, open, stack)) +} + +fn unused(_: &std::path::Path, _: &str) -> Result<(), layerfs_bridge::control::SdkRefusal> { + Ok(()) +} + +/// `project.delete shop/app` through the control-layer home path; the +/// Sandbox check is a no-op here (that check has its own inventory test). +fn remove(home: &std::path::Path) -> Result { + let mut sink = |_| {}; + delete( + home, + &mut Events::new(&mut sink), + &ProjectRef::parse("shop/app").unwrap(), + Related { + store_idle: unused, + can_remove_project: unused, + }, + ) +} + +/// The live intents at the project scope. +fn intents(open: &OpenArgs, stack: LayerStackId) -> Vec { + with_service(open, |service| { + Ok(service + .store + .history() + .gc_intents(GcScopeKind::Project, stack.as_slice()) + .unwrap()) + }) + .unwrap() +} + +/// Registers a live mount of the project's `main` Branch, as a Workspace +/// bind does before it demands the root. +fn mount_main(open: &OpenArgs, stack: LayerStackId, op: [u8; 16]) { + // `lookup::branch` refuses `main` by design; the snapshot lookup names it + let branch = with_service(open, |service| { + let project = HistoryName::new("app").unwrap(); + let main = HistoryName::new("main").unwrap(); + match service + .store + .published(|history| history.branch_snapshot_named(&project, &main)) + .unwrap() + { + NamedSnapshot::Found(snapshot) => Ok(snapshot.branch.id), + other => panic!("no main Branch: {other:?}"), + } + }) + .unwrap(); + let intent = GcIntent::shared( + op, + GcIntentKind::Mount, + GcScopeKind::Branch, + branch.as_slice(), + ) + .in_project(stack.as_slice()) + .named(Some("app"), Some("main")) + .placed(Some([9; 32]), Some("sandbox-9"), Some("machine-9")); + with_service(open, |service| { + service.store.history().gc_register(&intent).unwrap(); + Ok(()) + }) + .unwrap(); +} + +fn release(open: &OpenArgs, op: [u8; 16]) { + with_service(open, |service| { + service.store.history().gc_release(&op).unwrap(); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn a_live_mount_refuses_the_delete_after_the_sidecar_and_leaves_no_intent() { + let Some((home, open, stack)) = shop("mount") else { + return; + }; + mount_main(&open, stack, [9; 16]); + assert!(intents(&open, stack).is_empty(), "no project-scope row yet"); + + let refusal = remove(&home).expect_err("a live mount holds the project"); + assert_eq!(refusal.code, "MountsActive", "{refusal:?}"); + let SdkValue::List(holders) = refusal.detail.get("holders").expect("holders") else { + panic!("{refusal:?}"); + }; + assert_eq!(holders[0].get_text("kind"), Some("mount")); + assert_eq!(holders[0].get_text("sandbox"), Some("sandbox-9")); + + // A pending sidecar names the identity, but no marker and no leaked + // admission intent. + let pending = Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar precedes the admission"); + assert_eq!( + pending.state, + layerfs_store::project::delete::IntentState::Pending + ); + assert!(pending.op.is_some()); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert!( + intents(&open, stack).is_empty(), + "the refused admission left no project-scope row" + ); + with_service(&open, |service| { + assert_eq!( + lookup::project(&service.store, "app") + .expect("still live") + .id, + stack + ); + Ok(()) + }) + .unwrap(); + + // The mount goes and the same home path retries the pending sidecar: it + // writes the marker and settles, keeping the entry for the next call. + release(&open, [9; 16]); + let outcome = remove(&home).unwrap(); + assert_eq!(outcome.get_text("kind"), Some("Removed")); + assert_eq!(outcome.get_text("project"), Some("shop/app")); + assert!( + !outcome.get_bool("home_entry_removed"), + "a pending retry settles without finishing the entry" + ); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + let settled = Intent::read(&home, "shop", "app") + .unwrap() + .expect("the sidecar is marked"); + assert_eq!( + settled.state, + layerfs_store::project::delete::IntentState::Marked + ); + assert!( + intents(&open, stack).is_empty(), + "the admission was released" + ); + + // The marker is confirmed: the next call removes the entry and unlinks + // the sidecar. + let outcome = remove(&home).unwrap(); + assert!(outcome.get_bool("home_entry_removed")); + assert!(!paths::project_dir(&home, "shop", "app").exists()); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!( + intents(&open, stack).is_empty(), + "the admission was released" + ); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn a_live_operation_at_the_project_scope_is_busy_and_keeps_the_pending_sidecar() { + let Some((home, open, stack)) = shop("busy") else { + return; + }; + assert!(intents(&open, stack).is_empty()); + let holder = GcIntent::operation( + [7; 16], + GcIntentKind::Removal, + GcScopeKind::Project, + stack.as_slice(), + ) + .named(Some("app"), None); + with_service(&open, |service| { + service.store.history().gc_register(&holder).unwrap(); + Ok(()) + }) + .unwrap(); + + let refusal = remove(&home).expect_err("another removal holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + let pending = Intent::read(&home, "shop", "app") + .unwrap() + .expect("the pending sidecar stays"); + assert_eq!( + pending.state, + layerfs_store::project::delete::IntentState::Pending + ); + assert!(pending.op.is_some()); + assert_eq!(intents(&open, stack).len(), 1, "only the test's own holder"); + assert_eq!(intents(&open, stack)[0].op_id, [7; 16]); + + // The operation ends; the pending retry redos the marker, keeps the + // entry and releases its own admission. + release(&open, [7; 16]); + let outcome = remove(&home).unwrap(); + assert!(!outcome.get_bool("home_entry_removed")); + assert!(paths::project_dir(&home, "shop", "app").is_dir()); + assert_eq!( + Intent::read(&home, "shop", "app").unwrap().unwrap().state, + layerfs_store::project::delete::IntentState::Marked + ); + assert!(intents(&open, stack).is_empty()); + // The marker is confirmed: the next call finishes the entry. + assert!(remove(&home).unwrap().get_bool("home_entry_removed")); + assert_eq!(Intent::read(&home, "shop", "app").unwrap(), None); + assert!(intents(&open, stack).is_empty()); + let _ = std::fs::remove_dir_all(&home); +} + +#[test] +fn an_unnamed_admission_names_the_project_for_a_later_refusal() { + let Some((home, open, stack)) = shop("unnamed-admit") else { + return; + }; + // A `project.delete.admit` from a caller that does not send the name (an + // older control, or one that only knows the Stack): the worker fills the + // name from the Stack record, so the stored row still names the project. + let frame = DeleteAdmitFrame { + store: StoreName::parse("shop").unwrap(), + stack, + op: [7; 16], + project: None, + store_record: open.store_record.clone(), + }; + let answer = admit_project(&home, &frame).unwrap(); + assert_eq!(answer.get_text("kind"), Some("Admitted"), "{answer:?}"); + let held = intents(&open, stack); + assert_eq!(held.len(), 1); + assert_eq!( + held[0].project.as_deref(), + Some("app"), + "an unnamed caller still names the row from the Stack record" + ); + + // A later `project.delete` meets that live row: the `Busy` detail names + // the project, never the hex LayerStack id. + let refusal = remove(&home).expect_err("the live admission holds the project"); + assert_eq!(refusal.code, "Busy", "{refusal:?}"); + let entity = refusal.detail.get("entity").expect("entity"); + assert_eq!(entity.get_text("kind"), Some("project")); + assert_eq!(entity.get_text("name"), Some("app"), "{refusal:?}"); + assert_eq!(intents(&open, stack).len(), 1, "only the test's own holder"); + release(&open, [7; 16]); + let _ = std::fs::remove_dir_all(&home); +} diff --git a/control/crates/layerfs-store/tests/project_frames.rs b/control/crates/layerfs-store/tests/project_frames.rs index e200d7f041..2afe04cff9 100644 --- a/control/crates/layerfs-store/tests/project_frames.rs +++ b/control/crates/layerfs-store/tests/project_frames.rs @@ -3,7 +3,7 @@ //! refuses in the order it always did. use layerfs_bridge::control::SdkValue; use layerfs_home::names::StoreName; -use layerfs_store::project::frame::{InitFrame, RemoveFrame, StageFrame, StageStep}; +use layerfs_store::project::frame::{InitFrame, StageFrame, StageStep}; fn t(text: &str) -> SdkValue { SdkValue::text(text) @@ -45,26 +45,6 @@ fn each_frame_encodes_the_map_its_caller_built() { assert_eq!((decoded.name, decoded.timeout_ms), (init.name, Some(5))); assert_eq!(decoded.store_record, init.store_record); - let remove = RemoveFrame { - store: store(), - project: "web".to_owned(), - store_record: Some(record.clone()), - }; - let encoded = remove.encode(); - assert_eq!( - encoded, - map(vec![ - ("store", t("shop")), - ("project", t("web")), - ("store_record", record), - ]) - ); - let decoded = RemoveFrame::decode(&encoded).unwrap(); - assert_eq!( - (decoded.store, decoded.project), - (store(), "web".to_owned()) - ); - let stage = |step| StageFrame { store: "shop".to_owned(), name: "web".to_owned(), @@ -98,15 +78,6 @@ fn the_worker_refuses_a_frame_in_its_old_order() { let bare = InitFrame::decode(&map(vec![("store", t("shop")), ("staged", t(ID))])).unwrap(); assert_eq!((bare.name, bare.timeout_ms), (None, None)); - let remove = |fields| RemoveFrame::decode(&map(fields)).unwrap_err().message; - assert_eq!(remove(vec![]), "missing text argument `store`"); - // no project before a bad Store name - assert_eq!( - remove(vec![("store", t("../x"))]), - "missing text argument `project`" - ); - assert!(remove(vec![("store", t("../x")), ("project", t("web"))]).starts_with("store name")); - let stage = |fields| StageFrame::decode(map(fields)).unwrap_err().message; // a bad id before an unknown step assert_eq!( diff --git a/control/crates/layerfs-store/tests/project_list_frames.rs b/control/crates/layerfs-store/tests/project_list_frames.rs new file mode 100644 index 0000000000..344f1ee3f8 --- /dev/null +++ b/control/crates/layerfs-store/tests/project_list_frames.rs @@ -0,0 +1,122 @@ +//! The `project.list` worker frame and its reply (T4.5c): the map the +//! caller builds, the fields the worker decodes, and the filtered listing +//! it sends back. +use layerfs_bridge::control::SdkValue; +use layerfs_bridge::sdk_values::hex; +use layerfs_history::LayerStackId; +use layerfs_home::names::StoreName; +use layerfs_store::project::frame::{ListedProject, ListingFrame, ListingReply}; + +fn t(text: &str) -> SdkValue { + SdkValue::text(text) +} + +fn map(fields: Vec<(&str, SdkValue)>) -> SdkValue { + SdkValue::Map(fields.into_iter().map(|(n, v)| (n.to_owned(), v)).collect()) +} + +fn stack(body: u8) -> LayerStackId { + LayerStackId::from_authority([body; 16]) +} + +#[test] +fn the_frame_names_the_store_and_carries_its_record() { + let record = map(vec![("name", t("shop")), ("kind", t("sqlite"))]); + let frame = ListingFrame { + store: StoreName::parse("shop").unwrap(), + store_record: Some(record.clone()), + }; + let encoded = frame.encode(); + assert_eq!( + encoded, + map(vec![("store", t("shop")), ("store_record", record.clone())]) + ); + let decoded = ListingFrame::decode(&encoded).unwrap(); + assert_eq!(decoded.store, frame.store); + assert_eq!(decoded.store_record, Some(record)); +} + +#[test] +fn a_frame_with_only_the_store_decodes() { + let decoded = ListingFrame::decode(&map(vec![("store", t("shop"))])).unwrap(); + assert_eq!(decoded.store.as_str(), "shop"); + assert_eq!(decoded.store_record, None); +} + +#[test] +fn the_worker_refuses_a_bad_frame_in_order() { + let decode = |fields| ListingFrame::decode(&map(fields)).unwrap_err().message; + assert_eq!(decode(vec![]), "missing text argument `store`"); + assert!(decode(vec![("store", t("../x"))]).starts_with("store name")); +} + +#[test] +fn the_reply_round_trips_and_requires_listed() { + let reply = ListingReply { + projects: vec![ + ListedProject { + name: "app".to_owned(), + stack: stack(1), + }, + ListedProject { + name: "web".to_owned(), + stack: stack(2), + }, + ], + }; + let listed = |name: &str, stack: &LayerStackId| { + map(vec![ + ("name", t(name)), + ("stack_id", t(&hex(&stack.to_bytes()))), + ]) + }; + let encoded = reply.encode(); + assert_eq!( + encoded, + map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![listed("app", &stack(1)), listed("web", &stack(2))]), + ), + ]) + ); + assert_eq!(ListingReply::decode(&encoded).unwrap(), reply); + + let wrong_kind = map(vec![("kind", t("Removed"))]); + assert_eq!( + ListingReply::decode(&wrong_kind).unwrap_err().message, + "the worker project.list reply is not `Listed`" + ); + let no_list = map(vec![("kind", t("Listed"))]); + assert_eq!( + ListingReply::decode(&no_list).unwrap_err().message, + "reply has no `projects` list" + ); + let no_name = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("stack_id", t(&hex(&stack(3).to_bytes())))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&no_name).unwrap_err().message, + "reply project has no `name`" + ); + let bad_stack = map(vec![ + ("kind", t("Listed")), + ( + "projects", + SdkValue::List(vec![map(vec![("name", t("app")), ("stack_id", t("zz"))])]), + ), + ]); + assert_eq!( + ListingReply::decode(&bad_stack).unwrap_err().message, + "`stack_id` must be hex digits" + ); + let empty = ListingReply { + projects: Vec::new(), + }; + assert_eq!(ListingReply::decode(&empty.encode()).unwrap(), empty); +} diff --git a/core/crates/layerfs-content/src/object/admission.rs b/core/crates/layerfs-content/src/object/admission.rs index 2cc79c615e..285f8f7c14 100644 --- a/core/crates/layerfs-content/src/object/admission.rs +++ b/core/crates/layerfs-content/src/object/admission.rs @@ -36,7 +36,12 @@ pub(super) fn references( }) } -pub(super) fn decode_references( +/// Derives one object's direct logical references from its canonical body. +/// +/// Grammar and role checks stay; identity is **not** re-hashed, so a caller +/// that already authenticated the bytes (a mark resolves and authenticates each +/// chain once) pays one hash rather than two. +pub fn decode_references( role: ObjectRole, canonical: &[u8], policy: ConstructionPolicy, diff --git a/core/crates/layerfs-content/src/object/mod.rs b/core/crates/layerfs-content/src/object/mod.rs index a4efd72b55..8e74740fe5 100644 --- a/core/crates/layerfs-content/src/object/mod.rs +++ b/core/crates/layerfs-content/src/object/mod.rs @@ -14,6 +14,7 @@ mod predecessor; pub use crate::policy::{MAX_CANONICAL_OBJECT_BYTES, MAX_OBJECT_FIELD_BYTES}; pub use access::AuthenticatedObjects; +pub use admission::decode_references; pub use codec::{ canonical_len, decode_bytes_object, encode_bytes_object, encode_bytes_object_to, BYTES_KIND, HEADER_LEN, MAX_PAYLOAD_BYTES, OBJECT_MAGIC, VALUE_LEN_BYTES, diff --git a/core/crates/layerfs-content/src/object/output.rs b/core/crates/layerfs-content/src/object/output.rs index 9d3dda51ae..46d5ef330e 100644 --- a/core/crates/layerfs-content/src/object/output.rs +++ b/core/crates/layerfs-content/src/object/output.rs @@ -62,6 +62,18 @@ impl ObjectRole { } } + /// Whether a canonical body of this role names other objects. + /// + /// A role without logical references is fully described by its identity, + /// stored width and delta base, so a mark never decodes its payload. This + /// is what makes the object closure metadata-only for file payloads. + pub const fn has_references(self) -> bool { + !matches!( + self, + Self::WholeFile | Self::Chunk | Self::Symlink | Self::DirectoryLeaf + ) + } + /// Rebuilds a role from its persisted code. pub const fn from_code(code: u8) -> ContentResult { match code { diff --git a/core/crates/layerfs-history/src/contract/catalog.rs b/core/crates/layerfs-history/src/contract/catalog.rs index 9a39d81c1f..2136f3de71 100644 --- a/core/crates/layerfs-history/src/contract/catalog.rs +++ b/core/crates/layerfs-history/src/contract/catalog.rs @@ -20,9 +20,10 @@ use crate::error::HistoryResult; use crate::identity::{BranchId, CatalogId, CommitId, HistoryName, LayerId, LayerStackId, NodeId}; use crate::records::{ AddLayerOutcome, AddLayerRequest, BranchMergeRequest, BranchRecord, BranchSnapshot, - CommitHistoryRequest, CommitOutcome, CommitRecord, CommitRequest, ForkRequest, - LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, Reservation, - ReserveRequest, StackInitialization, + CommitHistoryRequest, CommitOutcome, CommitRecord, CommitRequest, ForkRequest, GcDeletion, + GcIntent, GcScopeKind, LayerHistoryRequest, LayerRecord, LayerStackRecord, Page, PageResult, + PathRemovalOutcome, PathRemovalRecord, PathRemovalRequest, Reservation, ReserveRequest, + StackInitialization, }; /// Checked inputs that establish one catalog binding. #[derive(Clone, Eq, PartialEq)] @@ -91,6 +92,17 @@ pub trait HistoryCatalog: Send + Sync { /// Stacks in name order. fn layer_stacks(&self, page: &Page) -> HistoryResult>; + /// The typed `NameTaken` state for a LayerStack (project) name, live or + /// deleted, read without resolving the Stack (spec 4.4). The default + /// reports none; a provider that keeps the collision query overrides it, + /// so `project.init` refuses a held name before it scans anything. + fn layer_stack_name_taken( + &self, + _name: &HistoryName, + ) -> HistoryResult> { + Ok(None) + } + /// The stack named `name`, if any. A provider answers through its name /// index; this default pages through the stacks in name order. fn layer_stack_named(&self, name: &HistoryName) -> HistoryResult> { @@ -113,6 +125,15 @@ pub trait HistoryCatalog: Send + Sync { /// One Branch by identity. fn branch(&self, id: BranchId) -> HistoryResult>; + /// One Branch by identity without the visibility filter (spec 4.2). Only + /// rendering and identity keep a retired Branch's name this way; every + /// caller-facing lookup resolves through [`HistoryCatalog::branch`]. A + /// provider that keeps retired rows overrides this; the default reads the + /// filtered row. + fn branch_any(&self, id: BranchId) -> HistoryResult> { + self.branch(id) + } + /// One Branch by its name in a stack, through the name index: the /// stack's other Branches are not read. fn branch_named( @@ -287,6 +308,200 @@ pub trait HistoryCatalog: Send + Sync { /// Consumes one checked half-open inode range for a scope. fn reserve_inodes(&self, request: &ReserveRequest) -> HistoryResult; + + /// Records a `branch.remove` marker. The default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it keeps no removal markers; an implementation returns its own + /// [`crate::error::HistoryError`] when the marker cannot be recorded. + fn mark_branch_removed(&self, _branch: BranchId, _removed_ms: i64) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported( + "branch removal marker", + )) + } + + /// Records a `project.delete` marker. The default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it keeps no deletion markers; an implementation returns its own + /// [`crate::error::HistoryError`] when the marker cannot be recorded. + fn mark_stack_deleted(&self, _stack: LayerStackId, _deleted_ms: i64) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported( + "project deletion marker", + )) + } + + /// The deleted_ms a project.delete marker recorded, when the stack is + /// retired. A same-home retry reads it by the stack id its durable intent + /// recorded before the marker; a live or absent stack has none. The + /// default refuses. + fn stack_deleted_ms(&self, _stack: LayerStackId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported( + "project deletion marker read", + )) + } + + /// Records a `project.remove_file` marker. The default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it keeps no path removal markers; an implementation returns its own + /// [`crate::error::HistoryError`] when the marker cannot be recorded. + fn mark_path_removed( + &self, + _stack: LayerStackId, + _path: &[u8], + _frontier_serial: u64, + _removed_ms: i64, + ) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported( + "path removal marker", + )) + } + + /// Applies one `project.remove_file` marker in one write transaction: + /// reads the Store high-water as the new frontier, returns `UpToDate` + /// when the path's marker already covers every project node, otherwise + /// advances the marker and the project epoch. The default refuses. + fn remove_path(&self, _request: &PathRemovalRequest) -> HistoryResult { + Err(crate::error::HistoryError::Unsupported("path removal")) + } + + /// Every `path_removed` marker of one Stack. The default refuses. + fn path_removals(&self, _stack: LayerStackId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported( + "path removal markers", + )) + } + + /// The `history_serial` a Commit or Layer records, if it exists. The + /// default refuses. + fn node_history_serial(&self, _node: NodeId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported( + "node history position", + )) + } + + /// The retained history roots of one Stack. The default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it retains no roots; an implementation returns its own + /// [`crate::error::HistoryError`] when the roots cannot be read. + fn retained_roots(&self, _stack: LayerStackId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported("retained roots")) + } + + /// The retained history roots of one Stack, for a caller that resolves + /// them separately. The default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it retains no roots; an implementation returns its own + /// [`crate::error::HistoryError`] when the roots cannot be read. + fn get_retained_roots(&self, _stack: LayerStackId) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported("retained roots")) + } + + /// Removes the unreachable history rows of one Stack, or reports how many + /// a dry run would remove. The mark that decides reachability runs in the + /// same write transaction as the removal (a dry run runs it too), so a + /// node created after an earlier mark is never collected. The returned + /// count covers that Stack's Commit, Layer, Branch and LayerStack rows; + /// the orphaned lineage rows the cleanup also drops are not included. The + /// default refuses. + /// + /// # Errors + /// + /// The default returns [`crate::error::HistoryError::Unsupported`] + /// because it removes nothing; an implementation returns its own + /// [`crate::error::HistoryError`] when the rows cannot be removed. + fn remove_unreachable(&self, _stack: LayerStackId, _dry_run: bool) -> HistoryResult { + Err(crate::error::HistoryError::Unsupported( + "unreachable history removal", + )) + } + + /// Registers one admission intent (§5) in its own write transaction. + /// + /// The default refuses: a provider without admission cannot make the + /// `MountsActive`/`Busy`/`Unresolved` promise. + fn gc_register(&self, _intent: &GcIntent) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// Releases one admission intent, deleting its row. + /// + /// The default refuses; a provider that never registered one is unaffected. + fn gc_release(&self, _op_id: &[u8; 16]) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// Marks one admission intent uncertain, so a later call reports + /// `Unresolved` until its owner settles it. + fn gc_mark_uncertain(&self, _op_id: &[u8; 16]) -> HistoryResult<()> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } + + /// `branch.remove`: admits at Branch scope and writes the marker in one + /// write transaction. + fn gc_remove_branch( + &self, + branch: BranchId, + removed_ms: i64, + _intent: &GcIntent, + ) -> HistoryResult<()> { + // A provider without admission still writes the marker; the Store + // service only adds admission where the provider keeps intents. + self.mark_branch_removed(branch, removed_ms) + } + + /// `project.delete`: admits at project scope and writes the marker in one + /// write transaction, preserving the same-home retry classification. + fn gc_delete_stack( + &self, + stack: LayerStackId, + deleted_ms: i64, + _intent: &GcIntent, + ) -> HistoryResult { + if let Some(deleted_ms) = self.stack_deleted_ms(stack)? { + return Ok(GcDeletion::Retired { deleted_ms }); + } + if self.layer_stack(stack)?.is_some() { + self.mark_stack_deleted(stack, deleted_ms)?; + return Ok(GcDeletion::Marked { deleted_ms }); + } + Ok(match self.stack_deleted_ms(stack)? { + Some(deleted_ms) => GcDeletion::Retired { deleted_ms }, + None => GcDeletion::Absent, + }) + } + + /// `project.remove_file`: admits at project scope and advances the marker + /// in one write transaction. + fn gc_remove_path( + &self, + request: &PathRemovalRequest, + _intent: &GcIntent, + ) -> HistoryResult { + self.remove_path(request) + } + + /// The live intents recorded at one scope, for `store.status` and tests. + fn gc_intents( + &self, + _scope_kind: GcScopeKind, + _scope_id: &[u8], + ) -> HistoryResult> { + Err(crate::error::HistoryError::Unsupported("GC admission")) + } } impl std::fmt::Debug for HistoryCatalogConfig { diff --git a/core/crates/layerfs-history/src/contract/error.rs b/core/crates/layerfs-history/src/contract/error.rs index 3cede4537b..53febbb686 100644 --- a/core/crates/layerfs-history/src/contract/error.rs +++ b/core/crates/layerfs-history/src/contract/error.rs @@ -7,6 +7,7 @@ //! that persistence did not answer and therefore proves nothing about rollback. use crate::identity::{CommitId, LayerId}; +use crate::records::{GcIntentKind, GcScopeKind}; /// A related record kind, used for a typed absence. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -25,6 +26,99 @@ pub enum Missing { Catalog, } +/// Which kind of record holds a name. +/// +/// A later release may add a holder kind, so the enum is non-exhaustive: a +/// caller matches it with a wildcard arm and never constructs it. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HolderKind { + /// A Branch. + Branch, + /// A project (LayerStack). + Project, +} + +/// State of the record that holds a name. +/// +/// A later release may add a state, so the enum is non-exhaustive: a +/// caller matches it with a wildcard arm and never constructs it. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HolderState { + /// The record is live. + Live, + /// The record is retired by `branch.remove`. + Retired, + /// The project is deleted. + Deleted, +} + +/// The record that holds a name. +/// +/// A later release may add a field, so the struct is non-exhaustive: +/// a caller reads its fields, and construction belongs to this crate. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NameHolder { + /// Record kind. + pub kind: HolderKind, + /// Name held. + pub name: String, + /// Holder state. + pub state: HolderState, +} + +/// Where a name is held. +/// +/// A later release may add a source, so the enum is non-exhaustive: a +/// caller matches it with a wildcard arm and never constructs it. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NameSource { + /// The Store's own records. + Store, + /// A home project record. + Home, +} + +/// Context for a `NameTaken` refusal. +/// +/// A later release may add a field, so the struct is non-exhaustive: +/// a caller reads its fields, and construction belongs to this crate. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NameTakenState { + /// The holder. + pub holder: NameHolder, + /// Where it is held. + pub source: NameSource, +} + +/// One hidden path. +/// +/// A later release may add a field, so the struct is non-exhaustive: +/// a caller reads its fields, and construction belongs to this crate. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RemovedPath { + /// The request path. + pub path: String, + /// The governing frontier serial. + pub frontier_serial: u64, +} + +/// Context for a `Removed` refusal. +/// +/// A later release may add a field, so the struct is non-exhaustive: +/// a caller reads its fields, and construction belongs to this crate. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RemovedState { + /// The hidden paths. + pub paths: Vec, +} + /// Expected versus observed Branch publication state. /// /// Both sides are reported, so a caller can distinguish a moved head from a @@ -50,8 +144,24 @@ pub enum HistoryError { Missing(Missing), /// A required capability is not provided by this build or provider. Unsupported(&'static str), + /// A name is held by a live, retired or deleted record. + NameTaken(Box), + /// A path is hidden by `remove_file`. + Removed(Box), + /// A staged name or artifact already exists and no record names it. + Exists(&'static str), + /// A required marker or fence row is missing or contradicts the request. + Conflict(&'static str), + /// The Store was replaced or moved under this handle. + StaleStore, /// Immediate metadata admission was refused; the caller may retry later. Busy, + /// A mount or Sandbox holds the scope; the holders are the ones observed. + MountsActive(Box), + /// A live operation holds the scope; its entity, phase and op are reported. + BusyOp(Box), + /// A dead or uncertain owner's operation holds the scope. + Unresolved(Box), /// The content authority this catalog is bound to is not available. OwnershipUnavailable, /// A declared bound, count or counter is exhausted. @@ -85,15 +195,38 @@ pub enum HistoryError { /// Persistence did not answer; the outcome of the attempted statement set /// is unknown and nothing may be replayed, discarded or rolled back on a guess. UnknownOutcome, + /// The same unknown outcome, named by the service or control call that + /// held the operation: spec §10 A6 reports `{entity: {kind, name}, phase, + /// op}` and leaves `phase` and `op` null when no durable operation is + /// known. Persistence itself constructs the bare [`Self::UnknownOutcome`]; + /// a call that knows its entity attaches this context at the boundary. + UnknownOutcomeOp(Box), } impl HistoryError { /// True when the attempted mutation's persistence outcome is unknown. + #[must_use] pub fn unknown(&self) -> bool { - matches!(self.cause(), Self::UnknownOutcome) + matches!( + self.cause(), + Self::UnknownOutcome | Self::UnknownOutcomeOp(_) + ) + } + + /// Names the operation this call held when persistence did not answer, + /// for the §10 A6 `Unknown` detail. Any other failure is returned + /// unchanged, so a call can attach its context to a whole `Result` + /// without rewriting a definite refusal. + #[must_use] + pub fn with_unknown(self, state: UnknownOutcomeState) -> Self { + match self { + Self::UnknownOutcome => Self::UnknownOutcomeOp(Box::new(state)), + other => other, + } } /// The typed failure itself; kept for callers that match through it. + #[must_use] pub fn cause(&self) -> &Self { self } @@ -105,7 +238,31 @@ impl std::fmt::Display for HistoryError { Self::InvalidInput(what) => write!(formatter, "invalid input: {what}"), Self::Missing(what) => write!(formatter, "missing {what:?}"), Self::Unsupported(what) => write!(formatter, "unsupported: {what}"), + Self::NameTaken(state) => write!(formatter, "name taken: {:?}", state.holder), + Self::Removed(state) => write!(formatter, "removed: {} paths", state.paths.len()), + Self::Exists(what) => write!(formatter, "exists: {what}"), + Self::Conflict(what) => write!(formatter, "conflict: {what}"), + Self::StaleStore => formatter.write_str("stale Store"), Self::Busy => formatter.write_str("busy"), + Self::MountsActive(state) => { + write!( + formatter, + "mounts active: {} holder(s)", + state.holders.len() + ) + } + Self::BusyOp(state) => write!( + formatter, + "busy: {} at {}", + state.kind.as_str(), + state.entity + ), + Self::Unresolved(state) => write!( + formatter, + "unresolved: {} at {}", + state.kind.as_str(), + state.entity + ), Self::OwnershipUnavailable => formatter.write_str("ownership unavailable"), Self::Capacity(what) => write!(formatter, "capacity: {what}"), Self::Integrity(what) => write!(formatter, "integrity: {what}"), @@ -119,7 +276,9 @@ impl std::fmt::Display for HistoryError { write!(formatter, "stack moved: {expected} -> {actual}") } Self::ContinuityUnavailable => formatter.write_str("continuity unavailable"), - Self::UnknownOutcome => formatter.write_str("unknown outcome"), + Self::UnknownOutcome | Self::UnknownOutcomeOp(_) => { + formatter.write_str("unknown outcome") + } } } } @@ -128,3 +287,188 @@ impl std::error::Error for HistoryError {} /// Result of one history operation. pub type HistoryResult = Result; + +impl NameHolder { + /// Builds a holder for a typed `NameTaken` context. + pub fn new(kind: HolderKind, name: impl Into, state: HolderState) -> Self { + Self { + kind, + name: name.into(), + state, + } + } +} + +impl NameTakenState { + /// Builds the context of a `NameTaken` refusal. + pub fn new(holder: NameHolder, source: NameSource) -> Self { + Self { holder, source } + } +} + +impl RemovedPath { + /// Builds one hidden path for a `Removed` refusal. + pub fn new(path: impl Into, frontier_serial: u64) -> Self { + Self { + path: path.into(), + frontier_serial, + } + } +} + +impl RemovedState { + /// Builds the context of a `Removed` refusal. + pub fn new(paths: Vec) -> Self { + Self { paths } + } +} + +/// One admission holder: a live mount or operation in the scope. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GcHolder { + /// What kind of holder it is (`mount`, `init`, `merge`, ...). + pub kind: GcIntentKind, + /// The project, Branch or Store the holder is on. + pub entity: String, + /// The holder's Workspace, when it has one. + pub workspace: Option, + /// The holder's Sandbox, when it has one. + pub sandbox: Option, + /// The holder's machine, when it has one. + pub machine: Option, + /// The holder operation identity, hex encoded. + pub op: String, +} + +impl GcHolder { + /// Builds one holder report. + #[must_use] + pub fn new(kind: GcIntentKind, entity: impl Into, op: impl Into) -> Self { + Self { + kind, + entity: entity.into(), + workspace: None, + sandbox: None, + machine: None, + op: op.into(), + } + } +} + +/// The holders that refuse a removal or compaction with `MountsActive`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MountHolders { + /// Every live or uncertain holder observed in the scope. + pub holders: Vec, +} + +impl MountHolders { + /// Builds the refusal context. + #[must_use] + pub fn new(holders: Vec) -> Self { + Self { holders } + } +} + +/// The live or uncertain operation that refuses a call with `Busy` or +/// `Unresolved`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpConflict { + /// The conflicting operation kind. + pub kind: GcIntentKind, + /// The resource scope kind the operation holds: store, project or + /// branch. This is the entity.kind the wire detail reports; the + /// operation kind is a separate fact. + pub scope_kind: GcScopeKind, + /// The project, Branch or Store it is on. + pub entity: String, + /// Its phase, when recorded. + pub phase: Option, + /// Its operation identity, hex encoded. + pub op: String, + /// Whether its owner is uncertain. + pub uncertain: bool, + /// The PostgreSQL holder's backend pid, when the refusal came from a lock + /// or a recorded owner; `None` on SQLite and on an unknown holder. + pub pid: Option, + /// The PostgreSQL holder's `client_addr`, when the server reports one. + pub client_addr: Option, + /// The PostgreSQL holder's `backend_start`. + pub backend_start: Option, +} + +impl OpConflict { + /// Builds the refusal context. + #[must_use] + pub fn new( + kind: GcIntentKind, + scope_kind: GcScopeKind, + entity: impl Into, + phase: Option, + op: impl Into, + uncertain: bool, + ) -> Self { + Self { + kind, + scope_kind, + entity: entity.into(), + phase, + op: op.into(), + uncertain, + pid: None, + client_addr: None, + backend_start: None, + } + } + + /// Adds the PostgreSQL holder a `pg_locks`/`pg_stat_activity` read names. + #[must_use] + pub fn with_backend( + mut self, + pid: Option, + client_addr: Option, + backend_start: Option, + ) -> Self { + self.pid = pid; + self.client_addr = client_addr; + self.backend_start = backend_start; + self + } +} + +/// The operation a call held when persistence did not answer (spec §10 A6). +/// +/// `phase` and `op` are `None` when no durable operation is known; the wire +/// detail then carries them as null. A later release may add a field, so the +/// struct is non-exhaustive: callers read it, and construction belongs to +/// this crate. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UnknownOutcomeState { + /// The resource scope kind the call names: store, project or Branch. + pub scope_kind: GcScopeKind, + /// The Store, project or Branch name. + pub entity: String, + /// The phase the operation was in, when known. + pub phase: Option, + /// The durable operation identity, hex encoded, when one is known. + pub op: Option, +} + +impl UnknownOutcomeState { + /// Builds the context of an `UnknownOutcome` refusal. + #[must_use] + pub fn new( + scope_kind: GcScopeKind, + entity: impl Into, + phase: Option, + op: Option, + ) -> Self { + Self { + scope_kind, + entity: entity.into(), + phase, + op, + } + } +} diff --git a/core/crates/layerfs-history/src/contract/identity.rs b/core/crates/layerfs-history/src/contract/identity.rs index d3234b1d6e..443e9588b4 100644 --- a/core/crates/layerfs-history/src/contract/identity.rs +++ b/core/crates/layerfs-history/src/contract/identity.rs @@ -264,7 +264,8 @@ authority_identity!(BranchId, BRANCH_BYTES, BRANCH_TAG); impl BranchId { /// The identity the SDK gives a Branch it creates by name. Branch names - /// are unique in their stack and are never reused, so the name decides. + /// are unique in their stack and are never reused while their rows exist, + /// so the name decides. pub fn named(stack: LayerStackId, name: &HistoryName) -> Self { let mut hasher = blake3::Hasher::new(); hasher.update(b"layerfs/sdk-branch/v1\0"); diff --git a/core/crates/layerfs-history/src/contract/mod.rs b/core/crates/layerfs-history/src/contract/mod.rs index 37313bcbc2..520aebbe72 100644 --- a/core/crates/layerfs-history/src/contract/mod.rs +++ b/core/crates/layerfs-history/src/contract/mod.rs @@ -2,5 +2,6 @@ pub mod catalog; pub mod error; pub mod identity; +pub mod path; pub mod query; pub mod records; diff --git a/core/crates/layerfs-history/src/contract/path.rs b/core/crates/layerfs-history/src/contract/path.rs new file mode 100644 index 0000000000..2e2140ca8f --- /dev/null +++ b/core/crates/layerfs-history/src/contract/path.rs @@ -0,0 +1,102 @@ +//! The `project.remove_file` path grammar and its keys. +//! +//! The call accepts an absolute POSIX path and stores it as the normalized +//! relative path (`/a` and `a` are the same). A path is refused when it is +//! empty, `/`, `//a`, has an empty, `.` or `..` component, a trailing `/`, a +//! `\` or NUL byte, is longer than 4096 bytes, or has more than 256 +//! components. The stored key is `BLAKE3(normalized path)`; the trace key is +//! keyed by the Store's `trace_salt` over the stack and the normalized path. + +use crate::identity::LayerStackId; + +/// The most bytes one normalized path may have. +pub const MAXIMUM_PATH_BYTES: usize = 4096; +/// The most components one normalized path may have. +pub const MAXIMUM_PATH_COMPONENTS: usize = 256; + +/// A checked, normalized relative path. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NormalizedPath { + relative: String, +} + +impl NormalizedPath { + /// Checks `text` and returns its normalized relative path. + /// + /// One leading `/` is stripped; everything after it is the relative path. + /// The returned value never starts or ends with `/` and never holds an + /// empty, `.` or `..` component. + pub fn new(text: &str) -> Result { + if text.is_empty() { + return Err("path is empty"); + } + if text.as_bytes().contains(&0) { + return Err("path contains a NUL byte"); + } + if text.contains('\\') { + return Err("path contains a backslash"); + } + let relative = text.strip_prefix('/').unwrap_or(text); + if relative.is_empty() { + return Err("path is the root"); + } + if relative.starts_with('/') { + return Err("path has an empty component"); + } + let mut components = 0usize; + for component in relative.split('/') { + if component.is_empty() { + return Err("path has an empty component"); + } + if component == "." || component == ".." { + return Err("path has a dot component"); + } + components += 1; + } + if components > MAXIMUM_PATH_COMPONENTS { + return Err("path has too many components"); + } + if relative.len() > MAXIMUM_PATH_BYTES { + return Err("path is too long"); + } + Ok(Self { + relative: relative.to_owned(), + }) + } + + /// The normalized relative path, without a leading `/`. + pub fn as_str(&self) -> &str { + &self.relative + } + + /// The normalized path bytes as stored in the marker. + pub fn as_bytes(&self) -> &[u8] { + self.relative.as_bytes() + } + + /// `BLAKE3(normalized path)`, the marker key. + pub fn key(&self) -> [u8; 32] { + path_key(self.as_bytes()) + } +} + +/// `BLAKE3(path)` over already normalized path bytes, the marker key. +pub fn path_key(path: &[u8]) -> [u8; 32] { + *blake3::hash(path).as_bytes() +} + +/// True when `path` is `marker` or lies below it at a `/` boundary +/// (`p == m OR p begins m ++ b"/"`), compared over bytes, never `||`/`LIKE`. +pub fn governs(marker: &[u8], path: &[u8]) -> bool { + path == marker + || (path.len() > marker.len() && path.starts_with(marker) && path[marker.len()] == b'/') +} + +/// The key a trace hashes for one stack and path: keyed by the Store's +/// 32-byte `trace_salt` over the stack identity and the normalized path. +pub fn trace_key(salt: &[u8; 32], stack: LayerStackId, path: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new_keyed(salt); + hasher.update(stack.as_slice()); + hasher.update(path); + *hasher.finalize().as_bytes() +} diff --git a/core/crates/layerfs-history/src/contract/records.rs b/core/crates/layerfs-history/src/contract/records.rs index 5027cc8d47..a9b05bf1ff 100644 --- a/core/crates/layerfs-history/src/contract/records.rs +++ b/core/crates/layerfs-history/src/contract/records.rs @@ -637,3 +637,273 @@ pub struct InitStatistics { /// Init's wall time in milliseconds, up to the first Branch. pub millis: u64, } + +/// One `path_removed` marker as stored for a project. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalRecord { + /// The normalized relative path the marker names. + pub path: Vec, + /// The Store high-water when the marker last advanced. + pub frontier_serial: u64, + /// When the marker last advanced, in milliseconds since the epoch. + pub removed_ms: i64, +} + +/// Whether a `project.remove_file` call advanced the marker or found it current. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PathRemovalKind { + /// The marker advanced to a new frontier. + Removed, + /// No project node is above the marker's frontier; the stored values are returned. + UpToDate, +} + +/// One `project.remove_file` request: the project, the normalized path and the +/// time the caller observed. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalRequest { + /// Project (LayerStack) the path is removed in. + pub stack: LayerStackId, + /// The normalized relative path bytes; `BLAKE3` keys the marker. + pub path: Vec, + /// When the call was made, in milliseconds since the epoch. + pub removed_ms: i64, +} + +/// One `project.remove_file` result: the stored marker values. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRemovalOutcome { + /// Whether the call advanced the marker or found it current. + pub kind: PathRemovalKind, + /// Project (LayerStack) the marker belongs to. + pub stack: LayerStackId, + /// The stored normalized relative path. + pub path: Vec, + /// The stored frontier serial. + pub frontier_serial: u64, + /// The stored `removed_ms`. + pub removed_ms: i64, +} + +/// One kind of operation admitted through the Store's `gc_intent` table (§5.1). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcIntentKind { + /// A mounted or mounting Workspace. + Mount, + /// A project Init. + Init, + /// A Branch merge. + Merge, + /// A soft removal (`branch.remove`, `project.delete`, `project.remove_file`). + Removal, + /// A Store compaction. + Compaction, +} + +impl GcIntentKind { + /// The `gc_intent.kind` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Mount => "mount", + Self::Init => "init", + Self::Merge => "merge", + Self::Removal => "removal", + Self::Compaction => "compaction", + } + } + + /// True when the operation shares its scopes instead of excluding them. + #[must_use] + pub const fn shared(self) -> bool { + matches!(self, Self::Mount | Self::Init | Self::Merge) + } +} + +/// The scope an admission intent is taken at. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcScopeKind { + /// The whole Store (compaction). + Store, + /// One project (Init, merge, `project.delete`, `remove_file`). + Project, + /// One Branch (mount, `branch.remove`). + Branch, +} + +impl GcScopeKind { + /// The `gc_intent.scope_kind` text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Store => "store", + Self::Project => "project", + Self::Branch => "branch", + } + } +} + +/// One durable admission row (§5.1), or the operation row a caller writes +/// before touching a Store. Ids are typed per kind: 32 bytes for the Store +/// catalog, 17 for a project or Branch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GcIntent { + /// 16-byte operation identity (`gc_intent.op_id`). + pub op_id: [u8; 16], + /// Operation kind. + pub kind: GcIntentKind, + /// Scope the intent is recorded at. + pub scope_kind: GcScopeKind, + /// Typed scope identity. + pub scope_id: Vec, + /// The owning project identity, when the scope is a Branch. It is not a + /// stored column: it lets the PostgreSQL key derivation name the project + /// scope of a Branch intent. + pub project_id: Option>, + /// Mounting Workspace identity, 32 bytes. + pub workspace: Option<[u8; 32]>, + /// Sandbox name, when the caller knows one. + pub sandbox: Option, + /// Machine name, when the caller knows one. + pub machine: Option, + /// Project name, when the caller knows it. + pub project: Option, + /// Branch name, when the caller knows it. + pub branch: Option, + /// Owning runtime token. + pub owner_runtime: Option, + /// Owning boot token; `UNIQUE(scope_kind, scope_id, owner_boot)`. + pub owner_boot: Option, + /// Operation phase, for a holder report. + pub phase: Option, + /// True when the owner is uncertain and only `Unresolved` may report it. + pub uncertain: bool, +} + +impl GcIntent { + /// A shared (`mount`, `init`, `merge`) intent at one scope. + #[must_use] + pub fn shared( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self::new(op_id, kind, scope_kind, scope_id) + } + + /// An exclusive (`removal`, `compaction`) intent at one scope. + #[must_use] + pub fn operation( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self::new(op_id, kind, scope_kind, scope_id) + } + + fn new( + op_id: [u8; 16], + kind: GcIntentKind, + scope_kind: GcScopeKind, + scope_id: impl Into>, + ) -> Self { + Self { + op_id, + kind, + scope_kind, + scope_id: scope_id.into(), + project_id: None, + workspace: None, + sandbox: None, + machine: None, + project: None, + branch: None, + owner_runtime: None, + owner_boot: None, + phase: None, + uncertain: false, + } + } + + /// Records the owning project identity of a Branch-scoped intent. + #[must_use] + pub fn in_project(mut self, project: impl Into>) -> Self { + self.project_id = Some(project.into()); + self + } + + /// Fills the project and Branch names a holder report shows. + #[must_use] + pub fn named(mut self, project: Option<&str>, branch: Option<&str>) -> Self { + self.project = project.map(str::to_owned); + self.branch = branch.map(str::to_owned); + self + } + + /// Fills the Workspace, Sandbox and machine a holder report shows. + #[must_use] + pub fn placed( + mut self, + workspace: Option<[u8; 32]>, + sandbox: Option<&str>, + machine: Option<&str>, + ) -> Self { + self.workspace = workspace; + self.sandbox = sandbox.map(str::to_owned); + self.machine = machine.map(str::to_owned); + self + } + + /// Fills the owner tokens. + #[must_use] + pub fn owned(mut self, runtime: Option<&str>, boot: Option<&str>) -> Self { + self.owner_runtime = runtime.map(str::to_owned); + self.owner_boot = boot.map(str::to_owned); + self + } + + /// Records the phase a holder report shows. + #[must_use] + pub fn with_phase(mut self, phase: Option<&str>) -> Self { + self.phase = phase.map(str::to_owned); + self + } + + /// Marks the owner uncertain: only `Unresolved` may report it. + #[must_use] + pub fn uncertain(mut self) -> Self { + self.uncertain = true; + self + } +} + +/// The outcome of the Store side of `project.delete`: a marker already there, +/// a marker written now, or neither a live stack nor a marker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GcDeletion { + /// The marker was already recorded; its `deleted_ms` is reused. + Retired { + /// The `deleted_ms` the marker holds. + deleted_ms: i64, + }, + /// A live stack was marked with the caller's `deleted_ms`. + Marked { + /// The `deleted_ms` written. + deleted_ms: i64, + }, + /// Neither a live stack nor a marker: the caller removes its stale entry. + Absent, +} + +impl GcDeletion { + /// The `deleted_ms` a `project.delete` result reports. + #[must_use] + pub fn deleted_ms(self, requested: i64) -> i64 { + match self { + Self::Retired { deleted_ms } | Self::Marked { deleted_ms } => deleted_ms, + Self::Absent => requested, + } + } +} diff --git a/core/crates/layerfs-history/src/lib.rs b/core/crates/layerfs-history/src/lib.rs index 0134a0e65d..92de00fc64 100644 --- a/core/crates/layerfs-history/src/lib.rs +++ b/core/crates/layerfs-history/src/lib.rs @@ -21,19 +21,24 @@ mod contract; pub use contract::catalog; pub use contract::error; pub use contract::identity; +pub use contract::path; pub use contract::query; pub use contract::records; pub use catalog::{HistoryCatalog, HistoryCatalogConfig, NamedSnapshot, PublishLane}; -pub use error::{HistoryError, HistoryResult}; +pub use error::{ + GcHolder, HistoryError, HistoryResult, MountHolders, OpConflict, UnknownOutcomeState, +}; pub use identity::{ BranchId, CatalogId, CommitId, HistoryName, LayerId, LayerStackId, NodeId, WorkspaceId, }; pub use records::{ AddLayerOutcome, AddLayerRequest, BindingHint, BranchMergeRequest, BranchRecord, BranchSnapshot, CommitHistoryRequest, CommitMeta, CommitOutcome, CommitRecord, CommitRequest, - CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, InitStatistics, - LayerHistoryRequest, LayerRecord, LayerStackRecord, MergeCommit, MergeHead, MergeSide, Page, - PageResult, RecordKind, Reservation, ReserveRequest, StackInitialization, StoreStatistics, - FILE_DIGEST_WINDOW, MAXIMUM_MESSAGE_BYTES, MAXIMUM_PAGE_RECORDS, + CommitStatistics, CommitVia, FileDigest, ForkRequest, ForkSource, GcDeletion, GcIntent, + GcIntentKind, GcScopeKind, InitStatistics, LayerHistoryRequest, LayerRecord, LayerStackRecord, + MergeCommit, MergeHead, MergeSide, Page, PageResult, PathRemovalKind, PathRemovalOutcome, + PathRemovalRecord, PathRemovalRequest, RecordKind, Reservation, ReserveRequest, + StackInitialization, StoreStatistics, FILE_DIGEST_WINDOW, MAXIMUM_MESSAGE_BYTES, + MAXIMUM_PAGE_RECORDS, }; diff --git a/core/crates/layerfs-history/tests/contract_constructors.rs b/core/crates/layerfs-history/tests/contract_constructors.rs new file mode 100644 index 0000000000..94625590a7 --- /dev/null +++ b/core/crates/layerfs-history/tests/contract_constructors.rs @@ -0,0 +1,70 @@ +//! A provider outside this crate builds each refusal context through the +//! public constructors, and readers still reach the public fields. + +use layerfs_history::error::{ + HolderKind, HolderState, NameHolder, NameSource, NameTakenState, RemovedPath, RemovedState, +}; +use layerfs_history::HistoryError; + +#[test] +fn name_taken_context_is_constructed_and_read() { + let holder = NameHolder::new(HolderKind::Branch, "feature", HolderState::Retired); + assert_eq!(holder.kind, HolderKind::Branch); + assert_eq!(holder.name, "feature"); + assert_eq!(holder.state, HolderState::Retired); + + let state = NameTakenState::new(holder, NameSource::Home); + assert_eq!(state.source, NameSource::Home); + assert_eq!(state.holder.name, "feature"); + + let error = HistoryError::NameTaken(Box::new(state)); + match error.cause() { + HistoryError::NameTaken(state) => { + assert_eq!(state.holder.kind, HolderKind::Branch); + assert_eq!(state.holder.state, HolderState::Retired); + assert_eq!(state.source, NameSource::Home); + } + other => panic!("expected NameTaken, got {other:?}"), + } +} + +#[test] +fn removed_context_is_constructed_and_read() { + let path = RemovedPath::new("/gone.txt", 42); + assert_eq!(path.path, "/gone.txt"); + assert_eq!(path.frontier_serial, 42); + + let state = RemovedState::new(vec![path, RemovedPath::new("/other.txt", 7)]); + assert_eq!(state.paths.len(), 2); + assert_eq!(state.paths[1].path, "/other.txt"); + + let error = HistoryError::Removed(Box::new(state)); + match error.cause() { + HistoryError::Removed(state) => { + assert_eq!(state.paths[0].path, "/gone.txt"); + assert_eq!(state.paths[0].frontier_serial, 42); + assert_eq!(state.paths[1].frontier_serial, 7); + } + other => panic!("expected Removed, got {other:?}"), + } +} + +#[test] +fn every_holder_kind_and_state_builds() { + for kind in [HolderKind::Branch, HolderKind::Project] { + for state in [ + HolderState::Live, + HolderState::Retired, + HolderState::Deleted, + ] { + let holder = NameHolder::new(kind, "name", state); + assert_eq!(holder.kind, kind); + assert_eq!(holder.state, state); + } + } + for source in [NameSource::Store, NameSource::Home] { + let holder = NameHolder::new(HolderKind::Project, "project", HolderState::Live); + let state = NameTakenState::new(holder, source); + assert_eq!(state.source, source); + } +} diff --git a/core/crates/layerfs-history/tests/contract_paths.rs b/core/crates/layerfs-history/tests/contract_paths.rs new file mode 100644 index 0000000000..f93c4905d8 --- /dev/null +++ b/core/crates/layerfs-history/tests/contract_paths.rs @@ -0,0 +1,55 @@ +//! The `project.remove_file` path grammar and its keys, read through the +//! public contract. Moved out of `contract/path.rs`: product source holds no +//! tests (the product-boundary guard). + +use layerfs_history::path::{ + governs, path_key, trace_key, NormalizedPath, MAXIMUM_PATH_BYTES, MAXIMUM_PATH_COMPONENTS, +}; +use layerfs_history::LayerStackId; + +#[test] +fn accepts_one_leading_slash_and_relative_paths() { + assert_eq!(NormalizedPath::new("/a").unwrap().as_str(), "a"); + assert_eq!(NormalizedPath::new("a").unwrap().as_str(), "a"); + assert_eq!(NormalizedPath::new("/a/b").unwrap().as_str(), "a/b"); + assert_eq!(NormalizedPath::new("a-b_c.d").unwrap().as_str(), "a-b_c.d"); +} + +#[test] +fn refuses_the_bad_grammar() { + for text in [ + "", "/", "//a", "a//b", "a/", "./a", "a/./b", ".", "..", "/./a", "a\\b", "a\0b", + ] { + assert!(NormalizedPath::new(text).is_err(), "accepted {text:?}"); + } +} + +#[test] +fn refuses_long_paths_and_many_components() { + assert!(NormalizedPath::new(&"a".repeat(MAXIMUM_PATH_BYTES)).is_ok()); + assert!(NormalizedPath::new(&"a".repeat(MAXIMUM_PATH_BYTES + 1)).is_err()); + let many = vec!["a"; MAXIMUM_PATH_COMPONENTS].join("/"); + assert!(NormalizedPath::new(&many).is_ok()); + let more = vec!["a"; MAXIMUM_PATH_COMPONENTS + 1].join("/"); + assert!(NormalizedPath::new(&more).is_err()); +} + +#[test] +fn subtree_matches_only_at_a_slash_boundary() { + assert!(governs(b"a", b"a")); + assert!(governs(b"a", b"a/b")); + assert!(!governs(b"a", b"ab")); + assert!(!governs(b"ab", b"a")); + assert!(governs(b"a/b", b"a/b/c")); + assert!(!governs(b"a/b", b"a/bc")); +} + +#[test] +fn keys_are_stable_and_salted_traces_differ() { + let path = NormalizedPath::new("/a/b").unwrap(); + assert_eq!(path.key(), path_key(b"a/b")); + let stack = LayerStackId::from_authority([7; 16]); + let first = trace_key(&[1; 32], stack, path.as_bytes()); + let second = trace_key(&[2; 32], stack, path.as_bytes()); + assert_ne!(first, second); +} diff --git a/core/crates/layerfs-storage/src/encoding/decode.rs b/core/crates/layerfs-storage/src/encoding/decode.rs index 3af347bd02..e0d1ca07b9 100644 --- a/core/crates/layerfs-storage/src/encoding/decode.rs +++ b/core/crates/layerfs-storage/src/encoding/decode.rs @@ -277,6 +277,24 @@ fn decode_payload( } } +/// Number of framed records in one group body, with the group directory +/// validated exactly as [`framed_record`] validates it. +pub fn framed_record_count(group: &[u8]) -> StorageResult { + if group.len() < 4 { + return Err(StorageError::Integrity("group framing")); + } + let count = u32::from_le_bytes( + group[..4] + .try_into() + .map_err(|_| StorageError::Integrity("group record count"))?, + ) as usize; + let ends = group + .get(4..4 + 4 * count) + .ok_or(StorageError::Integrity("group record directory"))?; + record_range(count, ends, group.len(), 0)?; + Ok(count) +} + /// Extracts one framed record from a group body. pub fn framed_record(group: &[u8], ordinal: usize) -> StorageResult<&[u8]> { if group.len() < 4 { diff --git a/core/crates/layerfs-storage/src/encoding/mod.rs b/core/crates/layerfs-storage/src/encoding/mod.rs index e10b0302e0..b33bceb546 100644 --- a/core/crates/layerfs-storage/src/encoding/mod.rs +++ b/core/crates/layerfs-storage/src/encoding/mod.rs @@ -23,7 +23,7 @@ pub use codec::{ CodecProfile, CompressionWorkspace, DecompressionWorkspace, DECODE_WORKSPACE_BYTES, ENCODE_WORKSPACE_BYTES, GROUP_LIMIT, }; -pub use decode::{framed_record, GroupCache}; +pub use decode::{framed_record, framed_record_count, GroupCache}; pub use full::{encode_full, encode_prefix, lane_body_limit, raw_payload, EncodedRecord}; pub use pack_cache::{PackCache, PackCacheWork}; pub use pack_units::{GroupSlice, PackAcquisition, PackUnit}; diff --git a/core/crates/layerfs-storage/src/store/error.rs b/core/crates/layerfs-storage/src/store/error.rs index 21f324b814..8ea58ccc76 100644 --- a/core/crates/layerfs-storage/src/store/error.rs +++ b/core/crates/layerfs-storage/src/store/error.rs @@ -95,10 +95,14 @@ impl StorageError { } } - /// True when persistence definitely refused the write for lack of space, - /// such as a full Store disk. Nothing was written by the refused attempt. + /// True when persistence definitely refused the write for lack of capacity: + /// a full Store disk, or a declared bound ([`Self::CapacityExceeded`]) that + /// refused the offered value. Nothing was written by the refused attempt. + /// A declared bound stays a provider/input failure, never disk exhaustion + /// (`admission-and-persistence.md` §10). pub fn is_capacity_refusal(&self) -> bool { match self { + Self::CapacityExceeded { .. } => true, Self::Io(error) => matches!( error .get_ref() diff --git a/core/crates/layerfs-storage/src/store/source.rs b/core/crates/layerfs-storage/src/store/source.rs index cf15de9075..88418224f3 100644 --- a/core/crates/layerfs-storage/src/store/source.rs +++ b/core/crates/layerfs-storage/src/store/source.rs @@ -21,6 +21,12 @@ pub trait Source { } /// Complete declared pack bytes, without capacity padding. fn pack_bytes(&self, pack_id: i64) -> StorageResult>; + /// The bounded control/directory prefix of one pack, with its declared + /// length. A metadata-only scan reads this instead of the whole body; a + /// source that cannot answer returns `None` and the caller reads the body. + fn pack_prefix(&self, _pack_id: i64, _limit: usize) -> StorageResult)>> { + Ok(None) + } /// Acquires the complete encoded groups needed by this bounded physical /// demand. `whole_for_reuse` requests whole materialization before I/O when /// this owner already retained another group. Custom sources use whole bodies. diff --git a/core/crates/layerfs-storage/tests/capacity_refusal.rs b/core/crates/layerfs-storage/tests/capacity_refusal.rs index 705733a692..c0e1a6d255 100644 --- a/core/crates/layerfs-storage/tests/capacity_refusal.rs +++ b/core/crates/layerfs-storage/tests/capacity_refusal.rs @@ -1,4 +1,5 @@ -//! A definite persistence refusal for lack of space is recognised (#379). +//! A definite persistence refusal for lack of capacity is recognised +//! (#379, #42 §10). use layerfs_storage::{port::PersistenceError, StorageError}; fn refused(status: &str) -> StorageError { @@ -15,8 +16,9 @@ fn only_a_definite_capacity_refusal_counts() { assert!(!StorageError::from(PersistenceError::Uncertain).is_capacity_refusal()); assert!(!StorageError::from(PersistenceError::Busy).is_capacity_refusal()); assert!(!StorageError::from(PersistenceError::Malformed).is_capacity_refusal()); - // A declared format bound is not a full disk. - assert!(!StorageError::CapacityExceeded { + // A declared bound refused the offered value before any write, so it is a + // capacity refusal too (§10), even though it is not a full disk. + assert!(StorageError::CapacityExceeded { what: "pack.assembled_length", limit: 1, actual: 2, diff --git a/core/docs/issues/project-41/LayerFS-GC-spec.md b/core/docs/issues/project-41/LayerFS-GC-spec.md index e54ced2101..65abed9252 100644 --- a/core/docs/issues/project-41/LayerFS-GC-spec.md +++ b/core/docs/issues/project-41/LayerFS-GC-spec.md @@ -1,26 +1,31 @@ # LayerFS garbage collection: soft removal and Store compaction -> **Status:** Proposal; target LayerFS 2.0.0; not a released contract. +> **Status:** **Frozen for layerfs-project#41 (LayerFS 2.0.0) on 2026-11-15**, approved by the owner on layerfs-project#42. An amendment needs a new #42 decision entry. > **Authority.** This is the only GC design document in the tree. It replaces the > six-file set merged by #658 (removed by #671; last text in the #677 branch). -> The contract it states is the frozen set of layerfs-project#42 (items 1–13, -> 2026-11-13) as amended by #42 items 14–26 (2026-11-14) and 27–44 -> (2026-11-15, implementation choices). Where this file and an older document -> disagree, this file wins; a change to a contract sentence needs a new +> The contract it states is the frozen set of layerfs-project#42: S1–S11 +> and items 1–42 (2026-11-13 to 2026-11-15). Items 43–44 are recorded on +> layerfs-project#42 and are not contract text. Where this file and an older +> document disagree, this file wins; a change to a contract sentence needs a new > decision entry on #42 first. > +> **Sign-off.** S1–S11, items 1–42 and the amendments below were approved by +> the owner on layerfs-project#42; this file is the frozen text. The decision +> table indexes every decision that carries contract text here; the complete +> log is layerfs-project#42. +> > **Scope.** layerfs-project#41 and its sub-issues #42–#52. The durability > profile (#155 §7) is outside this contract (#42 item 26). GC is developed > and measured under the Disposable profile (#42 item 42). > > **Source.** Code references are to `layerfs_v2_0_0` at `e8fa0a7e00`; those -> added by items 27–44 were checked at `8cfb6e2734`. +> added by items 27–42 were checked at `8cfb6e2734`. ## 1. Requirements (owner policy, #41) -- Four operations, on SQLite and PostgreSQL: `branch.remove(name)`, - `project.delete(name)`, `project.remove_file(project, path)` and +- Four operations, on SQLite and PostgreSQL: `branch.remove(project, branch)`, + `project.delete(project)`, `project.remove_file(project, path)` and `store.compact(name)`. Each takes only a resource identifier: no confirm, force, dry-run, stop-Sandboxes, Store, URL or volume option. - The three removals are logical. They change visibility and retention and @@ -44,15 +49,21 @@ |---|---|---| | S1 | Reachability is computed per Store, over every project in it | §6 | | S2 | Exclusion reuses the Store lock, lease and op record; PostgreSQL adds advisory locks (#42 items 20, 33, 34) | §5 | +| S3 | Superseded by S5/§7–§8: the rebuild is its own module, not the transfer `Stage`; no transfer-intent sidecar (the rebuild artifacts of §7/§9 are separate) | §7, §8 | | S4 | No object age; collection depends only on reachability | §6 | | S5 | The unit that frees bytes is the pack. A rebuild copies only retained rows into a fresh Store; a published `pack` row is never updated or deleted in place | §7, §8 | | S6 | A SQLite Store compacts on the machine where it lives (`place::plan`, `control/crates/layerfs-store/src/place.rs:40-49`) | §7 | +| S7 | Report the three byte definitions, peak temporary space and residue; estimates carry `estimate: true`; a capped report names its cap | §11 | +| S8 | One fence: `(dev, ino)` per transaction on SQLite, the incarnation join on PostgreSQL; both `StaleStore` | §10 | +| S9 | Install is binary; `live_heads_digest` matches before and after | §7, §8 | | S10 | Logical removal never rewrites history or changes an ID | §4 | -| S11 | Compaction is exclusive over the whole Store; reads are unavailable during it | §5 | +| S11 | Compaction is exclusive against mounts and long-lived users; on PostgreSQL one-shot reads/writes may run until the swap, and a write that spans it makes the swap abandon; SQLite excludes all readers | §5 | +| 6 | Marker and position tables are added to 2.0 Stores at open by an additive step; 1.0 Stores are converted offline | §4.1 | | 7 | Removal markers live in marker tables; `branch` and `layer_stack` stay append-only | §4 | -| 14 | Compaction collects history rows no retained root reaches; Commits of a removed Branch that nothing retains become `NotFound` | §6.1 | +| 12a | PostgreSQL advisory key is `(kind, id)` over the Store, project and Branch scopes | §5.2 | +| 14 | Compaction collects history rows no retained root reaches; Commits of a removed Branch that nothing retains become `NoSuchCommit` | §6.1 | | 15 | A name held by a removed Branch or deleted project returns `NameTaken` until compaction removes the row; then it is free. `NameRetired` is not added | §4.4 | -| 16 | `remove_file` hides a path across the project's history at once and lets compaction reclaim what only that path reached; reads through it return `Removed` (details: item 35) | §4.3, §6.2 | +| 16 | `remove_file` hides a path across the project's history at once; 2.0 is visibility-only and frees no bytes for it (details: item 35) | §4.3, §6.2 | | 17 | `store.compact` has two paths, `none` and `rebuild`; there is no light path. C3 has no caller in 2.0 | §3 | | 18 | A leftover is judged by the Store generation; no `OrphanArtifact`, no leftover cap | §9 | | 19 | API codes follow #41 (`MountsActive`, `Busy`); one `StaleStore` for both backends | §10 | @@ -60,6 +71,7 @@ | 21 | PostgreSQL compaction builds in batches, then swaps two tables in one short transaction | §8 | | 22 | SQLite compaction is its own module: the walk is fused with the copy, the visited set is a temp-file table, verification is cheap by default | §6.3, §7 | | 23 | Removed: the `pinned` set, the per-transaction SQLite generation check, the capacity pre-check, the extra report fields | §6, §7, §11 | +| 26 | The durability profile is outside this contract | header | | 27 | One visit level: anything reached by a logical or base edge is kept with all its references | §6.2 | | 28 | `merged_from_id` and `displaced_id` are history edges | §6.1 | | 29 | A retired Branch is collected as a unit; the history closure is a fixpoint | §6.1 | @@ -68,13 +80,14 @@ | 32 | `project.remove` is retired; compaction reuses its removal statements | §4, §7, §8 | | 33 | Admission at project and Branch scope (project op, intent record before bind, PostgreSQL keys at three scopes) | §5 | | 34 | PostgreSQL presence locks only on long-lived sessions; SDK calls keep working through transaction poolers | §5.2 | -| 35 | `remove_file` marker with a frontier; whiteout at mount; object-ID reads not path-aware; purge best effort | §4.3 | +| 35 | `remove_file` marker with a Store history-serial frontier; every tree-building operation materializes the whiteout, not only mounts; a republished Layer keeps its source Commit's serial; object-ID reads not path-aware; 2.0 frees no bytes (visibility only) | §4.3 | | 36 | No format change for the PostgreSQL mark until #52 measures it | §6.2 | | 37 | All `inode_value_block` rows are kept in 2.0 | §6.4 | | 38 | The rebuild clears `binding_hint_block` | §6.4 | | 39 | All four calls are admin-profile and refused under a folder scope | §10 | | 40 | Compaction writes its phase into the op; `store.status` shows it | §11 | | 41 | A started rebuild completes | §3 | +| 42 | GC is developed and measured under the Disposable profile | header | Out of this release: object age (Q3), repacking partly dead packs (reported, not performed), in-place deletion of packs on either backend, a read-available @@ -87,11 +100,17 @@ compaction window (A'), pruning (#46). | `none` | Bytes in wholly dead packs are below **10%** of the Store's bytes | Mark only; nothing is written; the report gives the reclaimable bytes | unchanged | | `rebuild` | 10% or more | SQLite §7, PostgreSQL §8 | `G → G+1` | -The 10% value is a starting point, set after the first measurement (#52). +The numerator is `COALESCE(SUM(pack.length), 0)` over wholly dead packs and +the denominator `COALESCE(SUM(pack.length), 0)` over all packs, on both +backends; the test is `rebuild iff total > 0 AND dead * 10 >= total` (no +truncating integer division). The 10% value is frozen at 10%; a change needs a +#42 entry. #52 measures it. A Store with no packs, or packs summing to 0, is +`none` with `deadBytesBound = 0` and `zeroReason: "empty-store"`. Below it a rebuild would copy at least nine live bytes for each byte freed. The choice is made on the bound known after the metadata walk; a rebuild that has started completes even if the refined share falls below 10%, and the -report gives both numbers (#42 item 41). +report gives both numbers as `deadBytesBound` and `deadBytesRefined` (§11; +#42 item 41). There is no path that deletes rows in place: it would free no payload, touch most index pages of the hash-keyed tables and need its own deletion contract. @@ -99,39 +118,117 @@ most index pages of the hash-keyed tables and need its own deletion contract. All three removals write one marker row in one short write transaction. No pack, object, Commit or Layer row changes, and no ID changes. They do not -bump the Store generation. +bump the Store generation, but they do bump the project's `epoch` in +`project_epoch` (§5), which a concurrent writer checks before it commits. | Call | Marker table | Effect | |---|---|---| -| `branch.remove(name)` | `branch_retired(branch_id, removed_ms)` | Hidden from listings and name resolution; fork, merge, publish and mount refused; `main` → `ReservedName` | -| `project.delete(name)` | `layer_stack_retired(layer_stack_id, deleted_ms)` | Project hidden; its home entry is removed; its data stays until compaction; the Store stays addressable | -| `project.remove_file(project, path)` | `path_removed(layer_stack_id, path, frontier, removed_ms)` | §4.3 | +| `branch.remove(project, branch)` | `branch_retired(branch_id BLOB NOT NULL PRIMARY KEY REFERENCES branch(branch_id) ON DELETE CASCADE, removed_ms INTEGER NOT NULL) STRICT` | Hidden from listings and name resolution; fork, merge, publish and mount refused; `main` → `ReservedName` | +| `project.delete(project)` | `layer_stack_retired(layer_stack_id, deleted_ms)` | Project hidden; its home entry is removed; its data stays until compaction; the Store stays addressable | +| `project.remove_file(project, path)` | `path_removed(layer_stack_id BLOB NOT NULL, path_key BLOB/bytea NOT NULL, path BLOB/bytea NOT NULL, frontier_serial INTEGER NOT NULL, removed_ms INTEGER NOT NULL) STRICT`, `PRIMARY KEY(layer_stack_id, path_key)`, `REFERENCES layer_stack ON DELETE CASCADE`; `path_key = BLAKE3(normalized path)` (a 4096-byte `bytea` key cannot fit a PG btree), the stored `path` is compared on collision; `frontier_serial` is the marker frontier (§4.3) | §4.3 | `project.delete` is the only project removal: the hard-deleting -`project.remove` is retired, and compaction reuses its `removal_*` statements -(`engine/crates/layerfs-persistence/src/history/removal.rs`) to drop a deleted -project's history (#42 item 32). +`project.remove` is removed from the call table (an old client that sends it +gets `InvalidInput` "no call named `project.remove`", never a silent hard +delete), and its module is deleted; compaction reuses its `removal_*` +statements (`engine/crates/layerfs-persistence/src/history/removal.rs`) to +drop a deleted project's history (#42 item 32). Tests that call +`project.remove` migrate to `project.delete`. Retirement also covers the +worker op/route for `project.remove`, the remote-SQLite route for +`project.delete`, the legacy own-project identity save (project name == Store +name) before the home entry is removed, the call-count and MCP-profile +generators, and the PostgreSQL path. The worker (the Store's home) is named +`project.delete`; the frame is `{store, stack_id, deleted_ms?}` (the caller +writes `deleted_ms` to the sidecar before the call and resends it on +retries). The worker saves the legacy own-project identity, writes the +marker, removes its own entry and replies +`{kind: "Removed", project, stack_id, deleted_ms}`; the caller removes only +its own entry and echoes `home_entry_removed`. A lost reply is `Unknown`, +keeps the sidecar, and the retry resolves retired/live/absent; the old +`project.remove` worker arm is deleted. + +`project.delete` commits the Store marker first and removes the caller's home +entry after it. The marker is the commit point. If the home removal fails the +call returns success with `home_entry_removed: false` and the stored +`deleted_ms`; a retry from the same home is idempotent: it may read the +retired stack allow-listed, reuses `deleted_ms`, finishes the entry removal +and returns `home_entry_removed: true`. The durable delete intent +`{stack_id, deleted_ms, attempt_ms}` lives outside the removed entry (or the +entry is **not** renamed; one sidecar file at +`/store//delete/.json` with +`{store, project, stack_id, deleted_ms, attempt_ms, state}` is written +(temp + fsync + rename + dir fsync) after the identity save and before the +marker, with `pending|marked|done` updated atomically and unlinked after the +entry removal; the retry +re-reads the Store by the recorded stack id and compares it with the home +record, discarding a mismatch without touching a re-created project's entry; +a retired row reuses `deleted_ms` and finishes the entry; a live row with no +marker redoes the marker with the same `deleted_ms` and does **not** remove +the entry; an absent stack row finishes the entry. It never resolves a name. If the marker fails nothing is +removed. The legacy own-project identity save (project name == Store name) +runs before the marker transaction, is idempotent, and a failure refuses with +nothing removed. A different home's stale entry is not rewritten; the +`project.list`/`project.get` intersection in §4.2 hides it. ### 4.1 Representation -The marker tables are added at open by an additive step under the current -schema identity (#42 item 6). On SQLite this is one more `add_*` step, as -`add_file_digests` was; the existing steps, the table list and +The marker and intent tables are added at open by an additive step under +the current schema identity (#42 item 6). On SQLite this is one more `add_*` +step, as `add_file_digests` was; the existing steps, the table list and `tools/store_blocks.py` are updated with it. PostgreSQL has no such step today (its open check requires the exact table set and recorded definition, `backend/postgresql/schema.rs:158-252`), so new Stores get the tables in `schema.sql` and an additive step at open creates them and rewrites the recorded definition for older Stores (#42 item 31). 1.0 Stores are converted -offline. `branch` and `layer_stack` keep their full unique name indexes. +offline. The same step adds the position state: `history_meta.next_history_serial` +(the next serial to assign; never lowered), a `history_serial` column on +`commit` and `layer` (a Layer that republishes a Commit carries that +Commit's serial; every other Commit and Layer takes the next value in its +creation transaction), `project_epoch(layer_stack_id PRIMARY KEY, epoch)` +(one row created with the project, bumped in the same transaction as every +marker write, copied by a rebuild, collected with the project), and two +admission tables: `gc_intent(...)` (§5.1) and `gc_op(...)`. The same step +adds `store_policy.trace_salt` last, written as 32 random bytes by the +Store-creation INSERT (`randomblob(32)`/OS RNG) and backfilled for existing +rows; `store_policy` copies use an explicit column list. `gc_intent` is +the shared Store-side admission table (created empty, never copied); `gc_op` +is PostgreSQL's single compaction row (created by PG's additive step, +untouched by the swap, cleared on settle) with columns `generation`, +`phase`, `scratch_schema`, `mark_serial`, `mark_pack_id`, `mark_digest`, +`mark_totals`, `mark_stamp` (`MAX(content_signature.stamp)`), `started_ms`, +`dead_canonical`, `dead_stored`, `dead_metadata`, +`dead_logical`, `totals_applied`, `artifact`, `pid`, `backend_start`, +`incarnation`. The `mark_*` columns are the first-statement tuple; the raw +head rows are hashed by the client. A GC-only `gc_dead(kind, id)` table (same additive step, never +copied, truncated on settle) carries the full dead set; a single `gc_op` row +cannot. SQLite keeps its phase in the +home/sidecar op and never writes the sealed DB, relying on the deterministic +artifact names for cross-machine recovery. The backfill runs on **every** +Store the step upgrades, including an existing 2.0 Store: Commits first by +`(created_ms, stack, id)`; then per Stack the genesis Layer first, a +republishing Layer takes its source Commit's serial, the rest by +`(source Commit created_ms, stack, id)`; `project_epoch(layer_stack_id, 0)` +for every existing stack; and +`next_history_serial = MAX(existing next_history_serial, +COALESCE(MAX(history_serial), 0) + 1)`; a Store with no stack/history keeps +its existing value. `branch` and `layer_stack` keep +their full unique name indexes. ### 4.2 Visibility filter No views: views count as schema on both backends. Every statement that lists or resolves a Branch or project carries a `NOT EXISTS` filter against the marker tables, and a guard test fails on any statement that reads `branch` or -`layer_stack` without it unless the statement is on an allow-list (the name -check and the fork/merge identity checks read retired rows on purpose). A -statement that bypasses the filter would resurrect a removed Branch. The +`layer_stack` without it unless the statement is on an allow-list: the +name-collision check (it must read the retired holder to return `NameTaken` +with its state), the fork/merge identity checks, compaction's mark/rebuild +statements, the `project.delete` retry, and two named by-id statements: +`branch_branch_by_id_any` (render/identity for `layer.list`/`layer.get`/ +`saved`/operator inspect) and `layerstack_stack_by_id_retired` (the retry's +live/absent read by the caller's recorded stack id). The ordinary +`branch_branch_by_id`/`layerstack_stack_by_id` stay filtered. Every +other statement that reads `branch` or `layer_stack` carries the filter; a new +allow-list entry needs a comment and a test. A statement that bypasses the filter would resurrect a removed Branch. The minimum is the Branch and project listings and name lookups (`branch_snapshot`, `branch_snapshot_named`, `layerstack_stack_by_name`), fork, merge, publish and mount resolution, and the project listing @@ -139,36 +236,134 @@ fork, merge, publish and mount resolution, and the project listing `identity::list_projects` and, since #678, the record rebuild in `backup.restore` (`control/crates/layerfs-store/src/backup/restore.rs`), so an unfiltered listing would write records for deleted projects back into a -home. +home. `project.list` and `project.get` intersect the home entries with this +filtered Store listing before paging: a stale entry (a different home, or a +failed home-entry removal) is absent from `project.list` and answers +`NoSuchProject` from `project.get`. The join is on the stack id recorded in +the home record: `manifest.stack` for a local SQLite record, +`record.stack` for PG/worker records; a missing stack is filled from the +filtered listing by name. The all-Stores listing omits an unreadable Store; +`project.list(store)` propagates `NoSuchStore`/`Busy`/`Unknown`. a Store that cannot be listed is omitted by `project.list` +and returns its own `NoSuchStore`/`Busy` from `project.get`, never the +unfiltered record. A hidden project is `NoSuchProject` for +the four calls, except `project.delete`'s same-home retry, which resolves the +caller's record by stack id and allow-lists its retired stack. ### 4.3 `remove_file` -- The marker names a path in one project and a **frontier**: the stack's - history serial when the marker was written. It applies to the Commits and - Layers at or below the frontier, the same way before and after any - compaction (#42 item 35): - - a read that resolves through the path returns `Removed`; - - a listing omits the entry and says that entries were removed (shape owned - by #47/#51); - - a mount of a Branch whose head is at or below the frontier applies a - whiteout for the path, so the next Commit records a real deletion. -- Commits after the frontier never inherit the path, and creating a file at - the same path later is an ordinary write that stays visible. -- Reads by object ID are not path-aware: they return the bytes until - compaction collects them, then `NotFound`; never different bytes. -- History rows and IDs are unchanged. Because the answer is `Removed` or - `NotFound`, not different bytes, an old ID never silently resolves to - changed contents. -- The call reports no freed bytes; compaction reports what it reclaimed (§6.2). -- The path is scoped to one project. The same bytes reached from another - project, another path, or as a delta base of a live object stay. -- Physical removal is best effort, and the documentation says so: a pack - shared with live objects is not repacked, and below the threshold - compaction writes nothing. A guaranteed purge (a targeted repack) is a - follow-up, not part of 2.0.0. -- Traces record a hash of the path, not the path. -- If #47 cannot deliver the path-aware walk in 2.0.0, the call ships as - visibility only and its documentation says plainly that it frees nothing. +- The marker names a path in one project and a **frontier**: the current + Store high-water, `next_history_serial - 1`, read under the Store write + lane in the same transaction as the marker; a node created concurrently is + ordered by its serial against that value, never by a later read. The first project's genesis Layer takes serial 1, so its first frontier + is 1; a Store with no Stack has none and `remove_file` is `NoSuchProject`. Every + Commit gets the next Store serial; a Layer that republishes a Commit + carries that Commit's serial, so `branch.publish` does not move a tree's + position, and every other Layer gets the next serial. A Commit or Layer is + *at or below* the frontier when its serial is at most the frontier; the + serial also covers Commits on unmerged Branches (#42 item 35). It applies + to the Commits and Layers at or below the frontier, before and after any + compaction: + - a read whose resolved tree actually contains the path (or a hidden + directory ancestor for an existing descendant) returns `Removed`; an + at-or-below node without the entry is `NotFound`/absent, and a + never-existing path hides nothing; + - a listing omits the entry and reports one top-level `removed_count` over + the queried subtree (present at 0, filtered before paging, no page slot or + admission cost); + - a mount of a Branch whose effective tree (its head Commit, else its base + Layer) is at or below the frontier applies a whiteout for the path, so + the next Commit records a real deletion. +- **Every operation that builds a new tree from a node at or below the + frontier materializes the whiteout**: a workspace Commit, a + non-fast-forward `branch.merge`, every one-shot tree creator (`file.write`, + `edit`, `apply`, `rm`, `mkdir`, `mv`, `cp`), and any server-side + construction. This tree/whiteout walk is mandatory in 2.0 and + distinct from the byte-freeing GC walk (§6.2). A fast-forward merge creates + no tree and is covered by the node it points at. `branch.publish` of a + Commit at or below the frontier republishes that tree as a Layer at the + same position: the marker hides the path and a mount of the Branch that + carries it whiteouts, so publish needs no whiteout of its own. A merge that + cannot materialize the whiteout is refused `Removed` with + `detail {paths: [...]}`. +- A path explicitly re-created after the frontier is not a whiteout: the + marker hides only content inherited from nodes at or below the frontier. + In a merge of a head at or below the frontier with a head whose path was + re-created after it, the re-creation wins; the whiteout removes the + inherited side and suppresses any conflict from that side. An ordinary edit + conflict is reported only when both sides are post-frontier (the marker does + not apply) or on other paths. +- One marker per project and path: a repeat `remove_file` writes the current + Store high-water (`next_history_serial - 1`) as the new frontier, so + everything at or below is hidden again. It is `UpToDate` iff no project + node has a serial greater than **this path's** marker frontier (a missing + marker is -1); a commit in another project changes + `next_history_serial - 1` but neither rewrites the marker nor advances its + frontier. A path that never existed is accepted and + hides nothing. The newest frontier subsumes the older ones. +- Every marker write that advances the frontier bumps the project's `epoch` + in the same transaction; an `UpToDate` reply does not. +- Commits after the frontier never inherit the path; creating a file at the + same path later is an ordinary visible write. +- A path is **hidden at node N** only when the entry exists at N and the + marker's frontier (the maximum over the path and its `/`-boundary + ancestors) is at least N's `history_serial`; a marked-but-absent entry is + absent/`NotFound`, and a marker on a never-existing path hides nothing. +- Reads by object ID are not path-aware: bytes while retained, then + `NotFound` once compaction collects it, never different bytes. +- The result is `{kind: "Removed", project, path, frontier_serial, + removed_ms}` or `{kind: "UpToDate", project, path, frontier_serial, + removed_ms}` (the stored values). Every refusal carries one detail shape: + `{paths: [{path, frontier_serial}]}` (one element for read/stat; the + governing maximum frontier and the request path). `file.read`/`file.stat` + refuse `Removed` with it. `file.list`/`file.glob` omit hidden entries and + report one top-level `removed_count` over the queried subtree (a hidden + target path is `Removed`; a marked-but-absent path is not counted). + `file.diff` has three states per side (visible/hidden/absent); hidden + sides never refuse: left visible + right hidden → `changeKind` `Removed`; + left hidden + right visible → `Added`; both hidden → `Removed` with both + side objects; a hidden side object is + `{removed: true, path, frontier_serial}`. A hidden side beats + `allow_missing`; only a genuinely absent side is `NoSuchPath` (with + `allow_missing: false`). `branch.diff`/`commit.diff` apply the same + per-side rule to their historical trees, so a removed path is absent, not a + spurious change. `file.revision` refuses `Removed` + when the anchor is hidden; otherwise it walks and **stops** at the first + hidden at-or-below version (never skips past it), returning only visible + versions. `file.blame` stops at the hidden version and attributes the + remaining lines at the tip. `file.grep`/`file.blame` omit hidden + hits. `project.export` omits it; a cp-from source is `Removed`; `file.edit` + is `Removed`; `file.mv` from a hidden path is `NotFound` and to a hidden + path is a visible re-creation; `mkdir`/`cp`/`apply` follow the same + construction rule; `file.rm` of a hidden path is `NotFound`; a + post-frontier write is the visible re-creation. +- The call accepts an absolute POSIX path and converts it to the Store's + normalized relative components (strip one leading `/`; `/a` and `a` are the + same). It refuses `InvalidInput` for empty, `/`, `//a`, `a//b`, `a/`, `./a`, + `a/./b`, `.`/`..` components, `\`, NUL, >4096 bytes or >256 components. A + directory argument hides its subtree only at a `/` boundary + (`p = m OR p begins m ++ b'/'`, computed as bytes in Rust, never `||` or + `LIKE` on a BLOB). A child of a removed **file** is `NotFound`; a read of + an existing descendant under a hidden directory is `Removed` with the + governing maximum frontier over the path and its ancestors (using + `history_serial`); an absent descendant is `NotFound`. An intermediate symlink component is `InvalidInput` (not followed); + a symlink leaf is hidden as an entry. Other hardlink names stay visible and + nothing is freed. +- History rows and IDs are unchanged. The trace key is BLAKE3(per-Store salt, + layer_stack_id, normalized relative path); the salt is a 32-byte + `store_policy.trace_salt` (random per Store, created by the additive step, + backfilled for converted Stores, copied by a rebuild). Every path argument + (`path`, `left_path`, `right_path`, `from_path`, `to_path`) is redacted + per call: the log records the normalized length as ``, never the + text; the daemon (native/worker) or the PG server-side call writes the keyed + hash, and the path argument names (`path`, `left_path`, `right_path`, + `from_path`, `to_path`, `path_prefix`, `pattern`) are in the per-call + redaction set. + Refusal details/messages are not traced; the `Removed` detail echoes the + request path, and raw paths never enter traces. +- The call frees no bytes in 2.0 (visibility only, §6.2); the path is scoped + to one project, so bytes reached elsewhere or as a live delta base stay. +- Physical removal, a path-aware GC walk and a guaranteed purge are follow-ups + that amend this contract before they ship. ### 4.4 Names @@ -177,12 +372,47 @@ home. name is never reused while its old rows exist. So: - A create or fork whose name a live row holds returns `NameTaken` (existing: - `engine/crates/layerfs-store-service/src/history/branch.rs:232,397`). + `engine/crates/layerfs-store-service/src/history/branch.rs:232,397`) with + `detail {holder: {kind: "branch"|"project", name, state: "live"}, + source: "store"}`. The detail rides + `HistoryError::NameTaken(Box)` (Holder/Source defined in + `layerfs-history`, boxed like `HeadMoved`); `lookup.rs` has an explicit arm + that projects `{holder, source}`. `source` is `store` only; a home-record + collision is a service `refuse("NameTaken", ..., {holder, source:"home"})`. + The `branch.rs:230-234,395-399` Null pre-checks are deleted, so every site + reaches the typed error. The branch query joins `branch_retired`; the + project query joins `layer_stack_retired`. - A name held by a removed Branch or deleted project also returns - `NameTaken`, with the holder's state in the detail. + `NameTaken`, with `detail {holder: {kind: "branch"|"project", name, + state: "live"|"retired"|"deleted"}, source: "store"}`; `source: "home"` is + used only for a project's home record. The `BranchId::named` identity check + and the name-collision statements both return this detail; the + `_ => "Failed"` arm is not allowed to swallow them. +- `branch_retired(branch_id BLOB NOT NULL PRIMARY KEY REFERENCES + branch(branch_id) ON DELETE CASCADE, removed_ms INTEGER NOT NULL) STRICT` + (same for `layer_stack_retired`), so a second remove cannot add a second + marker and a + collected base row takes its marker with it, on both backends. The write is + `INSERT ... ON CONFLICT(branch_id) DO NOTHING` with an affected-row check; + only an insert bumps `epoch`. A concurrent remove is `Busy` (phase + `"retire"`, no op), a repeat is `NoSuchBranch`; `project.delete`'s repeat is + the idempotent success in §4. +- `branch.fork` disambiguates by syntax: `main`, a valid HistoryName (a + Branch or `NoSuchBranch`, checked before `args::node`), or 66-hex (a Commit + or a Layer; a missing Layer is `NotFound`, a collected Commit + `NoSuchCommit`). A valid name is never retried as a node id; a Commit fork + keeps the retired row/name through `forked_from_id` and `commit.branch_id`, + and a Layer fork through `forked_from_id` and `layer.source_branch_id`. + A mount of a retired Branch is `NoSuchBranch`. For `merge`, a retired + **target** name is `NameTaken` while a retired source is `NoSuchBranch`; for + `publish`, a retired Branch is `NoSuchBranch` and a collected node is + `NoSuchCommit`. - Compaction removes a retired `branch` or `layer_stack` row once no retained row references it (`layer.source_branch_id`, `commit.branch_id` and the other history edges, `engine/crates/layerfs-persistence/sql/sqlite/history.sql:62-65`). + A Branch published by a retained Layer is pinned by that foreign key, and a + Branch named by a retained `commit.branch_id` is kept by the history + closure; only a Branch with neither reference is freed. The name is then free, and no retained row carries the old ID. The comment on `BranchId::named` ("never reused") is updated by #44 to say "never reused while its rows exist". @@ -193,54 +423,105 @@ name is never reused while its old rows exist. So: | Operation | Refused while | Code | |---|---|---| -| `branch.remove` | a mount or Sandbox of that Branch exists | `MountsActive` | -| `project.delete`, `project.remove_file` | any mount of the project (any Branch), or another user of the project | `MountsActive`, `Busy` | -| `store.compact` | any mount, client or operation on the Store, or unfinished Init/merge rows (#155 D-6) | `MountsActive`, `Busy` | - -Mounts take a Branch; there is no mount of a historical Commit. A Sandbox on a -machine that cannot be reached is `Busy`: its inactivity cannot be proven. - -The check and the exclusion are one atomic step: a new conflicting user is -refused from the check until the operation completes. +| `branch.remove` | a mount or Sandbox of that Branch, an unfinished Init/merge row names it, or another GC op at Branch scope | `MountsActive`, `Busy` | +| `project.delete`, `project.remove_file` | any mount of the project, another user of the project, an unfinished Init/merge row names it, or another GC op at project scope | `MountsActive`, `Busy` | +| `store.compact` | any mount, long-lived client or operation on the Store, or unfinished Init/merge rows | `MountsActive`, `Busy` | + +Mounts take a Branch; there is no mount of a historical Commit. All four +calls resolve their resource through the filtered statements (§4.2); a hidden +Branch is `NoSuchBranch`, a hidden project `NoSuchProject`, and a collected +Commit `NoSuchCommit` (the latter only on merge/publish), never a second +marker. The one exception is `project.delete`'s same-home retry, which +resolves the caller's recorded stack id under §4.2. + +`branch.merge` and `branch.publish` take no mount and are not in the table. A +non-fast-forward merge materializes the `remove_file` whiteout in its result +(§4.3) or is refused `Removed`; a publish of a Commit at or below the frontier +yields a Layer at the same position, which the marker hides. + +Every operation that creates a Commit or Layer, or writes under a project, +reads `project_epoch.epoch` at start and re-checks it before commit: +- SQLite: inside the one `BEGIN IMMEDIATE` transaction that also writes the + Commit/Layer, so a removal cannot interleave. +- PostgreSQL: the writer's last statement before commit is + `SELECT epoch FROM project_epoch WHERE layer_stack_id = $1 FOR SHARE` + (it blocks the marker's non-key `UPDATE`); a plain unlocked read is not + enough under READ COMMITTED. The marker transaction itself runs + `UPDATE project_epoch SET epoch = epoch + 1`, then reads the frontier + `next_history_serial - 1`, then inserts the marker. A missing epoch row + (post-cleanup project delete) is `Conflict`, never "unchanged". ### 5.1 SQLite -- **Store scope:** the existing Store lock, lease and op record - (`control/crates/layerfs-store/src/lease.rs`, `record.rs`, - `control/crates/layerfs-home/src/op.rs`). A live lease, reported by the - lease code as `InUse`, is returned as `Busy` at the API. -- **Project scope:** a project-level `op` on the project record, with the - same begin/settle pattern as the Store op - (`control/crates/layerfs-store/src/sqlite/family.rs:161-200`); Sandbox - create and mount check it (#42 item 33). -- **Branch scope:** a mount writes its Workspace intent record under the home - lock *before* bind, and removes it if bind fails. A removal checks the - records and writes its marker under the same lock, so a mount and a removal - can never both succeed; a bind that starts after the marker is refused by - the filtered statements (§4.2). +Admission is one Store-DB table and two `BEGIN IMMEDIATE` transactions +(SQLite 3.53.2 `lang_transaction`; a locked writer is `Busy`). + +`gc_intent(op_id PRIMARY KEY, kind, scope_kind, scope_id, workspace, sandbox, +machine, project, branch, owner_runtime, owner_boot, state, uncertain, phase, +generation, artifact, high_water, evidence, updated_ms)`, `STRICT`, with +`UNIQUE(scope_kind, scope_id, owner_boot)`; `kind` is `mount`, `init`, +`merge`, `removal` or `compaction`, `state` is `live`, `uncertain` or `done`, +`scope_kind` is `store`/`project`/`branch`, and the ids are typed per kind +(17-byte project/Branch, 33-byte commit/layer, 32-byte catalog id). The +migration step that adds it is the same additive step as §4.1. + +- A **mount** transaction checks that no operation op is running at every + scope it names (Store, project, Branch), inserts its intent and commits. A + **removal/compaction** transaction scans the intents in its scope, writes + its marker or op row, and commits. A live mount → `MountsActive`; a live + operation → `Busy`; a provably dead, uncertain op → `Unresolved`; a + provably dead, certain op is adopted or ignored per its `kind`. +- Compaction writes its `gc_intent` operation row **before** its first + checkpoint; the row lives in `.compact` during build/verify, moves + to the live Store after install, and is cleared at settle. `none` writes + and clears the same row (its only write); "nothing is written" in §3 means + no pack/object/history/generation change. +- Intent lifetime: the row is deleted in the `workspace.unmount` + transaction, on `sandbox.close`, daemon exit, machine stop, and in the + mount's error path if bind fails after the intent commit. A failed delete + is reported; a provably dead owner's row is ignored, never trusted. +- `machine_stopped` is provider evidence written into the intent/op; a live + but unreachable Sandbox is `Busy`; a removed machine record alone is not + proof. +- **Project scope:** `project.delete`/`remove_file` take a project op and + scan the intents naming the project. A mount is excluded by its own + intent, so several mounts of one project proceed. `project.delete` holds + its op through the marker transaction only; the retired-row entry-removal + retry needs no op, the live-row retry takes one and re-scans. +- **Branch scope:** the intent carries the Branch; a removal checks the + intents and writes its marker in the same transaction, so a mount and a + removal can never both succeed; a bind after the marker is refused by the + filtered statements (§4.2). +- `Unresolved` carries `{entity: {kind, name}, phase, op}`; a non-owning call + observes it, the owning call adopts/clears it. A `kind`-by-owner-state + matrix governs {mount, removal, compaction} × {live, unreachable, + dead-certain, dead-uncertain}. ### 5.2 PostgreSQL Leases are files in one home and cannot see another machine. So (#42 items 33, 34): -- **Presence.** Advisory keys at three scopes, `key(scope) = - blake3("layerfs/advisory/v1\0" ‖ catalog_id ‖ scope)[..8]` with scope the - Store, a project or a Branch (#42 item 12a, extended). Long-lived sessions - hold them shared on their own connection: the daemon for each mount (Store, - project and Branch keys), and compaction's own connection. A removal tries - the exclusive project or Branch key; compaction tries the exclusive Store - key. Failure is `Busy` or `MountsActive`, naming the holders (`pid`, - `client_addr`, `backend_start`). This makes an idle remote daemon with a - mount visible. -- **One-shot SDK calls** hold no presence lock. They rely on the fence below, - and on the swap's `ACCESS EXCLUSIVE NOWAIT` (§8): a transaction still open - at swap time makes the swap abandon. +- **Presence.** Advisory keys at three scopes, `key(kind, id) = + blake3("layerfs/advisory/v1\0" ‖ catalog_id ‖ kind ‖ id)[..8]` with kind 1 + the Store (32-byte `catalog_id`), 2 a project (17) and 3 a Branch (17); the + 8-byte truncation can cause a false `Busy` or a misclassified holder, never + a missed exclusion. A mount holds shared kind 1/2/3 on one dedicated + session for the daemon's life, refcounted per `(session, key)` and acquired + all-or-none with `pg_try_advisory_lock[_shared]`; a failed acquisition + unlocks what it took. Compaction holds only the exclusive kind 1 key, taken + before Phase M and held through Phase C/settle on that same pinned session. +- **Removals** take transaction-scoped locks in the marker transaction: + `branch.remove` = shared kind 1 + shared kind 2 + exclusive kind 3; + `project.delete`/`remove_file` = shared kind 1 + exclusive kind 2; order + Store → project → Branch. A failed shared attempt is `MountsActive`, a + failed exclusive attempt `Busy`; `pg_locks.mode` names the holder. +- **One-shot SDK calls** hold no presence lock. They rely on the + `project_epoch` `FOR SHARE` check (§5), the fence below, and the swap's + `ACCESS EXCLUSIVE NOWAIT` (§8). The four GC calls are not one-shot. - **Fence.** The generation check rides in the statements a client already sends: in the `BEGIN` batch of a transaction, and as a joined column of - `history_meta.catalog_incarnation` in the reads that run without a - transaction (`Mode::Immutable`/`Statement` skip `BEGIN`: - `engine/crates/layerfs-persistence/src/backend/postgresql/session/transaction.rs:263-268`). + `history_meta.catalog_incarnation` in reads without a transaction. No extra round trip. A mismatch is `StaleStore`. - **Connections.** Mounts and compaction need a direct or session-pooled connection; transaction pooling cannot hold a session lock and is @@ -248,7 +529,9 @@ Leases are files in one home and cannot see another machine. So (#42 items poolers. - **Dead clients.** A client whose TCP connection died holds its lock until the server notices. The guide documents `tcp_keepalives_idle`, - `client_connection_check_interval` and `pg_terminate_backend`. + `client_connection_check_interval` and `pg_terminate_backend`; a GC call's + lost reply is resolved on a second connection before the exclusive key is + taken. ## 6. Mark @@ -259,21 +542,32 @@ operations; a missing locator or corrupt dependency aborts with no writes. A recursive query from the retained roots over the history edges: -- Branch (not retired) → `head_commit_id`, `base_layer_id`, `forked_from_id` +- Branch → `head_commit_id`, `base_layer_id`, `forked_from_id` (followed for + every retained Branch; the **roots** are every non-retired Stack and every + Branch with no own or stack marker) - Stack (not retired) → `head_layer_id`, published Layers - Commit → `parent_commit_id`, `second_parent_id`, `base_layer_id`, - `merged_from_id`, `displaced_id` -- Layer → `parent_layer_id`, `source_commit_id` + `merged_from_id`, `displaced_id`, `branch_id` +- Layer → `parent_layer_id`, `source_commit_id`, `source_branch_id` `merged_from_id` and `displaced_id` are followed because doctor requires their targets (`diagnostic/history.rs:40,146`; #42 item 28). A retired Branch is collected as a unit: if a retained row references it (`commit.branch_id`, -`layer.source_branch_id`), its row stays and so does its head chain, because -`branch.head_commit_id` is a foreign key; the closure is the fixpoint of this -rule (#42 item 29). Every history edge is stack-local, so a live stack keeps -all its Layers and only Commits and retired Branch rows can go; a deleted -project goes whole. A fork keeps its source's history up to the fork point. -History rows outside the closure are collected by the rebuild (§7, §8); this +`layer.source_branch_id`), its row stays and so do `head_commit_id`, +`base_layer_id` and `forked_from_id`, and the head commit enters the Commit +edge set above; the closure is the fixpoint of these rules (#42 item 29). A +Layer's `source_branch_id` is a foreign key, so a published Branch and its +head chain stay retained while the Layer lives; that Branch keeps its name +(`NameTaken`) and compaction does not free it. "Free after compaction" (§4.4) +applies only to a Branch that nothing retains: no retained +`layer.source_branch_id` and no retained `commit.branch_id` (a fork's +`forked_from_id` is such a Commit). Every history edge is +stack-local, so a live stack keeps all its Layers and only Commits and +retired Branch rows can go; a deleted project goes whole. A fork keeps its source's history up to the fork point. +`store.history.branches` validates every retained `commit.branch_id`: it must +name a Branch in the same stack (SQLite cannot add the FK, so this is a doctor +edge). History rows outside the closure are collected by the rebuild (§7, §8); +this is required because the full doctor qualifies every Commit and Layer root it finds (`engine/crates/layerfs-persistence/src/diagnostic/history.rs:155-198`). @@ -285,31 +579,53 @@ finds (`engine/crates/layerfs-persistence/src/diagnostic/history.rs:155-198`). with `ChainBases::base_of` (`core/crates/layerfs-storage/src/encoding/delta/read.rs`). Do not reuse `qualify`: it skips file mappings, symlink targets and attribute trees and ignores delta bases. -- One visit level (#42 item 27): an object reached by a logical edge or a - base edge is kept with all of its references, because the full doctor - checks the references of every stored object - (`engine/crates/layerfs-persistence/src/diagnostic/scan.rs:129-224`). - Objects kept only because they are bases are reported in `keptWaste`. -- `remove_file`: when descending a root at or below a marker's frontier, the - walk does not enter the removed path. An object stays live if any other - path, project or delta chain reaches it. -- On PostgreSQL the base edges live inside pack groups, so a remote server - sends about the live payload to the client during the mark. No format - change is made for this until #52 measures it; the fallback is a nullable - `object_location.base_id` filled at publish (#42 item 36). +- Transitive closure (#42 item 27): each object is admitted once; admitting + it enqueues its logical references (`references()`) and its delta base + (`base_of`), and the queue runs to empty. An object reached only as a base + still keeps all of its references, because the full doctor checks the + references of every stored object (`diagnostic/scan.rs:129-224`). Objects + kept only because they are bases are reported in `keptWaste`. +- `remove_file` is **visibility-only in 2.0.0**: the marker hides the path in + reads, listings, mounts, merges and publishes, and compaction reclaims + nothing that only the removed path reached. A retained root's inode page + must stay (qualification refuses a missing inode, + `filesystem/qualify/walk.rs:183`), and rewriting the tree would change the + root and the Commit/Layer id (S10). A path-aware walk that frees bytes, and + per-entry page pruning, are a follow-up; `keptWaste` has no page component + in 2.0. +- On PostgreSQL the base edges live inside pack groups, so the mark (§8 + Phase M) reads that payload. A remote caller transfers it; `markBytes` is + the distinct live-pack body bytes read once, and #52 measures them. The + table copy moves no pack bytes. 2.0 keeps the format unchanged; adopting a + nullable `object_location.base_id` is a separate #52 decision with a schema + version and a defined NULL/backfilled distinction, not a runtime switch + (#42 item 36). - Window by window, sort lookups by object ID before locating them. - Whole files and chunks have no logical references; only their delta base is needed. Reading it still loads and authenticates the whole group - (`delta/read.rs:561-571`), so the payload part of the walk runs inside the - copy (§7 step 4), not as a separate pass. + (`delta/read.rs:561-571`); on SQLite the payload part of the walk runs + inside the copy (§7 step 4), and the PostgreSQL mark reads it in Phase M. ### 6.3 Visited set -A scratch table `visited(object_id PRIMARY KEY, pack_id) WITHOUT ROWID` -in a temporary file owned by the compaction connection (`temp_store=FILE`): -bounded memory, no second persistent database. Session connections keep -`temp_store=2` (`engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs:88`). -On PostgreSQL, a temporary table. +A scratch table `visited(object_id PRIMARY KEY, pack_id) WITHOUT ROWID` in a +named scratch database beside the Store (`.compact-visited`), attached +to the compaction connection: it is in the op/artifact set and in +`peakTempBytes`. Its own journal (`.compact-visited-journal`) and any +multi-database super-journal (`-mj*`) are in the artifact set too; or the +visited DB runs `journal_mode=OFF` and the copy is not atomic with it. Session +connections keep `temp_store=2` +(`engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs:88`); +both paths spill the closure to `.compact-visited` and choose the +path from the refined live set; the `none` path deletes the scratch before +replying, so it leaves no residue and never answers `Capacity` for a cap. On PostgreSQL it is a temporary table. + +`kept_node(kind, id)` is the history side of the same walk: `kind` is +`commit`, `layer`, `branch` or `stack`, and it holds the §6.1 closure. A +rebuild copies `layer_lineage.layer_id ∈ K_layer`, `commit_lineage.commit_id +∈ K_commit`, `absorbed.run ∈ R`, `reach.key ∈ K_stack ∪ R` (R := runs of kept +`commit_lineage` rows), and the history/catalog rows by `kept_node`; it +deletes the complement, and a deleted project goes whole. ### 6.4 Decision @@ -317,69 +633,153 @@ On PostgreSQL, a temporary table. - `none` or `rebuild` from §3, using the bytes of packs that hold no live object after the metadata walk (an upper bound that §7 step 4 refines). - Dead rows: `object_location`, `content_signature`, `file_digest` (keep a - row only if its root is retained, #155 R-4), history rows outside §6.1, and - retired `branch`/`layer_stack` rows (§4.4). + row only if its root is in `visited`; a digest or signature root is not a + mark root, so an uncommitted saved file's row is dead), history rows + outside §6.1, retired `branch`/`layer_stack` rows (§4.4), a `branch_retired` row whose + branch row is collected, a `layer_stack_retired` row whose stack is + collected, and `path_removed`/`project_epoch` rows of a deleted project (a + project goes whole). A live project's markers stay: `path_removed` is one + per path, and a serial allocated later never falls below a frontier. - Kept in 2.0: every `inode_value_block` row; the ordinals of pooled leaves are private to core, and the blocks are small. The report gives their bytes (#42 item 37). - Cleared: `binding_hint_block`. The hints are advisory and not per project, and they hold the file names of deleted projects and removed paths (#42 - item 38). + item 38). Clearing all of them costs only advisory lookup speed, never + correctness; a later Save repopulates hints for live paths. A per-project + clear is a follow-up, not 2.0.0. + +`live_heads_digest` (both backends) is BLAKE3 over +`"layerfs/live-heads/v1\0"`, then the non-retired stacks in id order as +`(stack_id, head_layer_id)`, then the Branches with no own or stack marker in +id order as +`(branch_id, head_commit_id or 33 zero bytes, base_layer_id)`, each +fixed-width. Commit ids are 33 bytes (tag `0x12` + 32), Layer ids 33 (tag +`0x32`), Branch and stack ids 17; a golden vector pins the bytes. A retired +row is absent, not zeroed; timestamps are not hashed, so SQLite and +PostgreSQL agree. ## 7. SQLite rebuild Runs as one operation on the Store's machine (S6). -1. Checkpoint the WAL and confirm no `-wal` remains (an immutable attach - ignores the WAL). -2. Create `.compact` beside the Store with a raw connection: +1. Take the persistence writer guard and hold it through step 7's rename. + Checkpoint the source with `PRAGMA wal_checkpoint(TRUNCATE)` and require + `busy == 0` (never unlink a WAL file); checked-close the last source + session and check for `-wal`/`-shm` while the guard is held; pass the + guard into the seal (no second acquire); use + `seal_holding() -> (size, mtime, db-header change counter, Writer)` + captured after the sidecar check and before release. The current + `Session::seal` releases the lane, so a returned Writer is required; there + is no re-acquire alternative. A busy checkpoint is + `Busy`. Record the sealed `(size, mtime, db-header change counter)`. +1.5 After the seal, `ATTACH` the old file `immutable=1` (it stays + byte-identical), create `.compact-visited`, run the complete + §6.1+§6.2 closure into it, and choose `none`/`rebuild` from the refined + live set. On `none`, delete the scratch and stop: no `.compact`, no + `.compact-visited`, no residue. On `rebuild`, keep the scratch and reuse + `visited` in step 4; step 2 creates the new file only on this path. +2. If the path is `rebuild`, create `.compact` beside the Store with a + raw connection: `page_size=4096`, the source's `auto_vacuum`, `schema::scripts`, - `application_id`, `user_version`. Give this connection a large - `cache_size`, and the attached old file a large `mmap_size`; the session - defaults are a 2 MiB cache and no mmap (`session/profile.rs:86-87`). -3. `ATTACH` the old file `immutable=1`; it stays byte-identical. -4. One rollback-journal transaction, foreign keys on, in this order: + `application_id`, `user_version`. The new file uses the source's + `(layout, acquisition)` derived from its `user_version`: `Monolithic` has + `pack.body` and no `pack_unit`; `GroupRows` has `pack_unit`; + `GroupRowsIndexed` also has its index. `schema_source`/`schema_definition` + must be those of that pair, or step 6's `validate` refuses. Give this connection a + large `cache_size`, and the attached old file a large `mmap_size`; the + session defaults are a 2 MiB cache and no mmap (`session/profile.rs:86-87`). +3. The old file is already attached `immutable=1` in step 1.5; it stays + byte-identical. +4. `PRAGMA foreign_keys=ON` outside the transaction (inside it the pragma + is a no-op), then one rollback-journal transaction, in this order: - Metadata walk (§6.1–§6.2 logical part) into `visited`. - - Payload sweep: live packs in descending `pack_id`. Copy each live pack - once, add the delta bases of its live records to `visited`. A base found - in a pack already passed goes on a late list and is copied at the end. - A base is not guaranteed to sit in a lower `pack_id` (first-wins - locators can move forward, - `core/crates/layerfs-storage/src/store/source.rs:9-13`), so the late - list is an indexed table, not an exception path. + - Payload sweep: a worklist of packs to copy, run to empty. Start with the + packs the mark marked live; copy each pack once. For every object + inserted into `visited` (a base, its references and their bases), run the + same `admit` fixpoint, insert each newly discovered object if absent and + enqueue **every** newly visited object's locating pack (passed or not); + do not follow records whose `object_id` is not in `visited`. A pack + reached only by a base is copied whole; `pack_id` is preserved and new `unit_id`s are + assigned; the dead set and `deadBytesRefined` are recomputed from the + final copied set. A base is not guaranteed to sit in a lower `pack_id` + (first-wins locators can move forward, + `core/crates/layerfs-storage/src/store/source.rs:9-13`), so the worklist + is an indexed table, not an exception path. - `pack_unit` for the copied packs, as a complete unit set (`engine/crates/layerfs-persistence/src/backend/sqlite/objects/units_read.rs:55-75`). + A `Monolithic` Store copies `pack` with its body and skips `pack_unit` + (the Monolithic decoder is used); `GroupRows` copies both; + `GroupRowsIndexed` copies both and the schema creates its unit index + (the index is not "copied"). `sqlite_sequence` is copied only when the + source has it; `store_totals` is UPDATEd as a whole row (keeps `init_*`) + and `store_policy` is `INSERT ... SELECT * FROM old.store_policy` + (allocators and salt verbatim); transient `init_*`/acquisition working + rows + are **not** copied, and a live Init op is admitted as `Busy`. - `object_location` by an ordered join with `visited`. - - `content_signature` entries that name a copied object; `file_digest`, - `inode_value_block`, history and catalog tables per §6.4, with a deleted - project's rows left out by the `removal_*` statements' selection; - `sqlite_sequence`, `store_policy` and `scope_allocator` copied; - `binding_hint_block` left empty; `history_meta` with - `catalog_incarnation = G + 1`. - - `store_totals`: `logical` is rebuilt by the history triggers and must - equal the old value minus the collected history rows' `logical`; - `canonical`, `stored` and `metadata` are summed. -5. Switch the new file to WAL and run the sidecar checks - (`engine/crates/layerfs-persistence/src/backend/sqlite/snapshot.rs:101-140`). -6. Verify: a real Session open (`validate`), `live_heads_digest` before and - after (a new function beside `heads_digest`, - `engine/crates/layerfs-persistence/src/backend/sqlite/heads.rs:83`, over - non-retired Branches and stacks only; #42 item 30), row counts against the mark, - `foreign_key_check`, the catalog allocator and totals checks. No sha256 of - either file. `quick_check` and the full doctor stay separate commands. -7. Install: re-read `G`; check `(catalog_id, incarnation, dev, ino)` and that - no sidecars exist; `sync_all` the new file; `rename` it over the Store; - fsync the directory. The old inode is freed at once. -8. A definite failure of compaction's own write (disk full, write refused) - aborts the transaction and deletes `.compact` in the same call, - returning `Capacity` or the failure (#155 D-22). + - `content_signature` entries that name a copied object; `file_digest` + rows whose root is in `visited`; all `inode_value_block` rows; history, + lineage (`layer_lineage`, `commit_lineage`) and catalog tables copied by + `kept_node`; `absorbed` by `run`; `reach` by its kind-mapped key; with a + deleted project's rows left out by the `kept_node`/retired predicate + (`removal_*` runs only on the writable new Store in Phase C); + `sqlite_sequence`, `store_policy`, `scope_allocator`, the marker tables + (`branch_retired`, `layer_stack_retired`, `path_removed`) and + `project_epoch` copied for rows the closure keeps; + `commit.history_serial`/`layer.history_serial` copied with their rows; + `history_meta.next_history_serial` copied unchanged; `binding_hint_block` + left empty; `history_meta` with `catalog_incarnation = G + 1`. + - `store_totals`: after **all** history copies, one UPDATE sets `logical` + to the triggers' value (or `old - dead_logical`), `canonical`/`stored`/ + `metadata` to the new sums, and `init_*` to the old row; assert + `old.logical == SUM(all history.logical)` before the copy. A copy before + the history insert would make the AFTER INSERT triggers add the + collected rows again (`old + collected`). +5. Switch the new file to WAL with a raw connection (`temp_store=MEMORY`, + `busy_timeout=0`, the source's `synchronous`); on macOS call + `disable_persistent_wal` before the checkpoint + (`snapshot.rs:116`, `seal.rs:31-32`); checkpoint with + `wal_checkpoint(TRUNCATE) == 0`; close it. No Session is opened. +6. Verify with an `immutable=1` open (`READ_ONLY|URI`, + `file:...?immutable=1`, percent-encoded path bytes so `?`/`#`/`%` do not + misdirect the URI, no profile apply/check, no lane) and run `validate` + there before the rename: + `schema::check`, + `foreign_key_check`, row counts against the mark, both `live_heads_digest` + values and the catalog allocator/totals checks. No sha256 of either file. + Close the reader; `quick_check` and the full doctor stay separate + commands. +7. Install: stat both paths for `-wal`/`-shm`/`-journal` and the + super-journal (`-mj*`); any sidecar is `Conflict` with no writes. Re-read + `G` through a fresh immutable handle and re-stat last; check + `(catalog_id, incarnation, dev, ino)` and the sealed `(size, mtime, + db-header change counter)`; a change is `Conflict`. `chmod 0600`; + `sync_all`; `rename`; fsync the directory; release the guard. A + transaction that began before install is caught at its next transaction + by the `(dev, ino)` fence, not at commit (documented limit). +8. A definite failure before the rename (disk full, write refused, sidecar, + checkpoint, validate, digest/count/FK, chmod/sync) aborts or rolls back and + deletes the artifact set `.compact{,-journal,-wal,-shm,-visited, + -visited-journal,-mj*}`; the old Store is unchanged apart from step 1's checkpoint. A + failure of the rename/fsync itself is installed-but-uncertain: keep the new + Store, settle the op `unknown` and report it; never delete the artifact as + if nothing happened. `SQLITE_FULL`/`BackendError::Capacity` is + `HistoryError::Capacity`, or the real `Refused { status: "Capacity" }` + path; `ENOSPC`/`EDQUOT`/`EFBIG` (`StorageFull`/`QuotaExceeded`/ + `FileTooLarge`) from chmod/sync/rename also map to `Capacity`, and + `SQLITE_FULL`/`SQLITE_TOOBIG`/`SQLITE_NOMEM` are Capacity on SQLite; no + `_` arm may turn it into another `Refused` status. `CapacityExceeded` is a + codec/canonical bound and is documented as `InvalidInput`/`ProviderFailure`, + not disk exhaustion. The existing transfer pipeline is not reused for this: its `Install` refuses an existing Store file and its `Retire` deletes one (`control/crates/layerfs-store/src/sqlite/transfer/steps.rs:262-288`). Cost: read about the live bytes plus the metadata, write about the live -bytes; peak disk = old Store + live bytes + the visited table. +bytes; peak disk = old Store + the new Store + the rollback journal/WAL + the +visited scratch file. ## 8. PostgreSQL rebuild @@ -387,66 +787,163 @@ Only `object_location.pack_id` references `pack`, so the payload lives in two tables. `content_signature` (at most 8192 rows, no foreign key: `sql/postgresql/schema.sql:45-50`) is cleaned in place. -**Phase A — build (exclusive advisory lock held, no table lock).** +**Phase M — mark (before Phase A).** Take the session-level exclusive Store +key before this phase and hold it through Phase C. The mark's **first +statement** reads `(catalog_incarnation, live_heads_digest, +history_meta.next_history_serial, next_pack_id, next_ordinal, +store_totals canonical/stored/metadata/logical)` and records that tuple; +any later publication makes Phase B `Conflict`, so a commit that lands during +the walk cannot be recorded while its objects are absent from `visited`. (Or +run the whole mark in one REPEATABLE READ transaction and record the tuple +inside it; state the vacuum cost.) One session, kept through Phase B, builds +`visited(object_id PRIMARY KEY, pack_id)` as a temporary table: start from the +retained roots, read pack groups in batches, follow +`FinalizedObject::admit(..).references()` and `ChainBases::base_of` as a +worklist to a fixpoint (each object once, bases of late objects included), and +record `live_heads_digest` and the live pack/object sets. After the walk, +write `gc_op` and `gc_dead` in batches (phase `mark`); a crash leaves both, +and the next `store.compact` adopts the op, truncates `gc_dead` and re-marks. +A `none` decision writes `gc_op` phase `mark`, truncates `gc_dead`, removes +the named artifact and clears `gc_op` before replying. +`gc_dead` carries `mark_stamp` for signature slots. The payload read is +reported as `markBytes`; the Phase A table copy itself moves no pack bytes. + +**Phase A — build (session-level exclusive Store advisory key, no table lock).** 1. Load the live object and pack IDs into temporary tables; `ANALYZE` them. -2. Create `pack` and `object_location` in a scratch schema with the Store's - column definitions and `STORAGE EXTERNAL` on `pack.body`, without the - trigger and the foreign key. -3. Copy live rows with `INSERT … SELECT` and a semi-join (`object_location` - has no unique key, `schema.sql:29-38`, so a plain join can duplicate rows), - in batches of about 1 GB by `pack_id`, each batch its own transaction. No - pack bytes cross the network. Batches may pause for standby lag. +2. Create `pack` and `object_location` in a scratch schema with + `CREATE TABLE ... (LIKE store.pack INCLUDING STORAGE, COMPRESSION, + CONSTRAINTS)` and the same for `object_location` (PK explicit; do not copy + `object_location_prefix`), plus the old `TABLESPACE` and `WITH + (reloptions)` read from `pg_class`. `pack.body` stays STORAGE EXTERNAL and + `object_location.object_id` stays STORAGE PLAIN; the trigger and foreign + key are added later. +3. Copy live rows with `INSERT … SELECT` and an `EXISTS` semi-join + (`object_location` has no unique key, `schema.sql:29-38`, so a plain join + can duplicate rows): `pack` rows with at least one visited object and + `object_location` rows whose `object_id` is visited, in batches of about + 1 GB by `pack_id`, each batch its own transaction. A retried batch first + deletes its own `pack_id` range from the scratch tables (idempotent), then + inserts; assert `count(*) = count(DISTINCT object_id) = |visited|`. No pack + bytes cross the network. Batches may pause for standby lag. 4. Build `object_location_prefix` and the foreign key, then `ANALYZE`. **Phase B — swap (one short transaction).** -5. `LOCK TABLE` the Store's tables `ACCESS EXCLUSIVE NOWAIT`. -6. Re-check `catalog_incarnation = G`, `live_heads_digest` (PostgreSQL gains - one; it has no heads digest today), and that `store_policy.next_pack_id` - and the ordinal allocator have not moved. Any change abandons the round - with no writes (`Conflict`). -7. Delete dead rows in place: `content_signature`, `file_digest`, history - rows (a deleted project's through the `removal_*` statements, which already - run on PostgreSQL) and retired name rows; clear `binding_hint_block`; - `inode_value_block` is kept (§6.4). `logical` in `store_totals` is reduced - by the collected history rows, as `removal_totals` does today - (`engine/crates/layerfs-persistence/src/history/removal.rs`); `canonical`, - `stored` and `metadata` are recomputed. -8. Copy the old tables' owner and ACL onto the new ones. -9. `DROP TABLE` the old `object_location` and `pack`; `ALTER TABLE … SET SCHEMA` +5. `LOCK TABLE` every base table of the Store schema (from `pg_class`, + including `store_lane`, `store_totals`, `history_meta`, + `content_signature`, the history tables and `gc_intent`/`gc_op`) in one + `ACCESS EXCLUSIVE NOWAIT`, with `store.*`/`scratch.*` qualified names. A + `55P03` abort is `Busy` with holder detail (not `Conflict`); `ROLLBACK` + first, then read holders and drop the scratch schema in a new + transaction. +6. Re-check `catalog_incarnation = G`, `live_heads_digest`, + `store_policy.next_pack_id`/the ordinal allocator, + `history_meta.next_history_serial` and `store_totals` + (`canonical`/`stored`/`metadata`/`logical`) against the values the mark + recorded. A + publication that reused a pre-reserved pack/ordinal may move none of the + first three but always bumps `store_totals`; any change abandons the round + with no writes (`Conflict`). No delta copy is attempted, so the new pack + stays in the old Store. +7. Copy the old tables' owner and ACL onto the new ones. +8. `DROP TABLE` the old `object_location` and `pack`; `ALTER TABLE … SET SCHEMA` the new ones into the Store's schema; create the `pack_immutable` trigger there, bound to the Store schema's `pack_immutable()` so doctor sees `EXECUTE FUNCTION pack_immutable()` (`diagnostic/postgresql_contract_expected.rs:213`). -10. Verify inside the transaction: row counts against the mark, - `live_heads_digest` unchanged, and the doctor contract query (tables, - constraints, indexes, trigger) clean. A failure rolls back. -11. `catalog_incarnation = G + 1`; `SET LOCAL synchronous_commit` to the - Store's setting or `local`, whichever is stronger; `COMMIT`. Then drop - the empty scratch schema. - -- **Crash:** a crash in phase A leaves only the scratch schema, which the op - record names and the next attempt removes. Phase B is atomic. -- **Lost `COMMIT` reply:** first prove the old backend has ended (the - exclusive advisory lock is free, or its recorded `pg_backend_pid()` is - gone), then read the incarnation. A read, not a replay. -- **Lock time:** `ACCESS EXCLUSIVE` is held for phase B only, seconds rather - than the length of the copy. No long transaction holds back vacuum for - other Stores in the same database. +9. Verify inside the transaction: row counts against the mark; + `SUM(new) = mark_totals - dead_*` with `logical` unchanged; the + `pg_attribute`/`attstorage`/`reloptions`/`tablespace` fingerprint recorded + in `gc_op` at mark equals the scratch tables' fingerprint; persist the + `dead_*` totals **before** the swap COMMIT. The settle transaction then + subtracts once and asserts the four `store_totals` sums. + `live_heads_digest` unchanged, and the bounded checks on the moved tables + (the `pack_immutable` trigger/function and `pg_attribute`/storage). Run only + the C/I contract entries for `pack`/`object_location` on the scratch tables + in Phase A (the trigger/function entries need the Store-schema function and + stay in the swap's bounded check), with `search_path=scratch`. A failure + rolls back. +10. `catalog_incarnation = G + 1`; the effective `synchronous_commit` is the + stronger of the Store's and the server's in the order + `off < local < remote_write < on < remote_apply`, set with `SET LOCAL` + around the swap and never lowered. It waits iff the level is at least + `remote_write` **and** `current_setting('synchronous_standby_names')` is + non-empty; then require the named FIRST/ANY standbys to appear in + `pg_stat_replication` with `state='streaming'` and the matching + `sync_state` (`sync`/`quorum`). Check before Phase M and again inside the + swap before COMMIT; a missing standby abandons with no writes (`Busy`, + standby list). A standby that dies after the check blocks the COMMIT; + `SET LOCAL` cannot bound that wait. + `DROP` the empty scratch schema before `COMMIT` (it is transactional). + +**Phase C — cleanup (batched, after the swap).** + +11. Delete only ids in the mark's **persisted dead set** in `gc_op` + (membership, not a predicate: exact `content_signature` slots, + `file_digest` roots, history/lineage ids, a deleted project's + `path_removed`/`project_epoch` rows, and marker/base row pairs). Per-table + watermarks (`commit`/`layer` serial, signature stamp) are only guards; a + row created after the mark is never a member, so a Commit/Layer/save + created during cleanup survives. Assert each batch's own `pack_id` range + and, after the last batch, `count(*) = count(DISTINCT object_id) = + |visited|`. Delete child-first (a project's closure + in one transaction; a marker with its base row) so the deferred FKs hold. + Clear `binding_hint_block` wholesale (it is not id-addressable, §6.4). + The swap already dropped dead packs and locations, so persist + `dead_canonical` (dropped `object_location.canonical_length`), + `dead_stored`/`dead_metadata` (dropped pack `length`; `stored` is all + domains, `metadata` only domain 0) and `dead_logical` in `gc_op` + **before** the swap COMMIT; `content_signature` deletes are by + `(slot, stamp)` with `stamp <= mark_stamp`; reduce + `store_totals` by those totals in the **same transaction that settles the + op**, or gate on `totals_applied`; never re-sum. Assert the new tables' + sums equal old totals minus dead totals. The op phase is `cleanup`; a crash resumes the + persisted set — there is no "re-derive from the current Store" path. The + Store is usable at `G + 1` throughout; the full doctor runs after cleanup. + +- **Scratch schema:** one deterministic, length-safe name per Store derived + from `catalog_id` (`lfs_gc_`, truncated to 63 bytes), recorded in the + Store-side `gc_op` row (§4.1) before `CREATE SCHEMA`; a crash names it from + the Store, not only from the caller's home. +- **Crash:** a crash in Phase M/A leaves only the scratch schema, which the + Store's op names and the next attempt removes. Phase B is atomic; a crash + after COMMIT resumes Phase C. +- **Lost `COMMIT` reply:** record `pid`, `backend_start`, `usename` and the + incarnation in `gc_op` in a transaction **before** the swap. Recovery on a + second connection (before taking the exclusive key) is one statement: + `SELECT pg_terminate_backend(a.pid, ) FROM pg_stat_activity a WHERE + a.pid = $1 AND a.backend_start = $2 AND a.usename = $3 AND + a.datname = current_database() AND a.pid <> pg_backend_pid()`, checking the + boolean and polling until the pair is gone, then reading the incarnation. + Terminating another role's backend needs `pg_signal_backend` or membership; + the precheck reports `Unsupported` when neither is available. +- **Lock time:** `ACCESS EXCLUSIVE` is held for the swap only, seconds rather + than the length of the copy or the cleanup. No long transaction holds back + vacuum for other Stores in the same database. - **Space:** old tables + new heap/TOAST + indexes + up to the live bytes of WAL. The guide states the WAL headroom requirement; a full `pg_wal` stops the whole server. -- **Privileges:** ownership of the Store's tables and `CREATE` on the schema, - otherwise `Unsupported`. +- **Privileges:** `CREATE` on the database (for the scratch schema), `TEMP` + (the visited tables), `USAGE` and `CREATE` on the Store schema, ownership + of the Store's tables, and `pg_read_all_stats` for `client_addr`/ + `backend_start` (which may be NULL); check all before Phase M, otherwise + `Unsupported`. ## 9. Failure and leftovers -- The op record names `G` and the staged artifact (`.compact` or the - scratch schema). +- The op record names `G` and the staged artifact. The SQLite artifact is + the set `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; all of it is removed for the + reserved name, and the op is settled only after removal. The PostgreSQL + artifact is the scratch schema, whose deterministic name is recorded in the + Store's op row before it is created, so a crash on another machine can still + name and remove it. - The next `store.compact` judges it: the Store at `G` means not installed, so the named artifact is removed; `G+1` means installed, so the outcome is - recorded. A machine that cannot be reached reports `unknown`, never clean. + recorded. An op left uncertain by a dead owner is adopted by that + `store.compact` (its artifact and high-water are in `gc_op`); only a live + owner blocks it. A machine that cannot be reached reports `unknown`, never + clean. - A file or schema at the staged name that no record names is refused with the existing `Exists` and is not deleted. - Install is the only irreversible step: the Store is wholly old or wholly new. @@ -458,46 +955,130 @@ tables. `content_signature` (at most 8192 rows, no foreign key: - **SQLite:** at the start of every transaction, compare the Store path's `(dev, ino)` with the one recorded at open; a mismatch is `StaleStore` before anything is read or written. This covers both a reader and a writer - still holding the old file (formerly C8 and C8b). + still holding the old file (formerly C8 and C8b). Compaction holds + `lane::writer_admission` from the seal through the rename, so an admitted + writer cannot be in flight at install; a connection that bypassed admission + is caught at its next transaction. Install also requires the sealed + `(size, mtime, db-header change counter)` unchanged; a change is `Conflict` + with no writes. - **PostgreSQL:** the generation check of §5.2. - **`lfq1` tokens** of an older generation already fail, because their authenticator is keyed by the incarnation. | Code | Status | |---|---| -| `MountsActive`, `Busy`, `NotFound`, `NoSuchStore`, `Unsupported`, `ReservedName`, `Conflict`, `NameTaken`, `Capacity`, `Exists` | existing | +| `MountsActive`, `Busy`, `Unresolved`, `NotFound`, `NoSuchBranch`, `NoSuchCommit`, `NoSuchProject`, `NoSuchStore`, `Unsupported`, `ReservedName`, `Conflict`, `NameTaken`, `Capacity`, `Exists`, `InvalidInput` | existing (`Unresolved` now used by the four calls) | | `StaleStore` | new; one `HistoryError` variant with an explicit mapping arm, because the `_ => "Failed"` wildcard (`engine/crates/layerfs-store-service/src/history/lookup.rs:54,64`) would hide it | -| `Removed` | new; reading a path removed by `remove_file` | +| `Removed` | new; reading a path removed by `remove_file`; an explicit `HistoryError::Removed` arm in `lookup.rs` before the `_ => "Failed"` wildcard; detail `{paths: [{path, frontier_serial}]}` | + -- A project create whose name is held returns `NameTaken` (today it returns - `Exists`), the same as a Branch create or fork. -- New `HistoryCatalog` methods get default `Unsupported` bodies so the eight - implementations need not change at once (C9). +- `project.init` whose name is held returns `NameTaken` (today it returns + `Exists`), the same as a Branch create or fork; both the home record and the + Store name collision map to it. +- The new `HistoryCatalog` methods are enumerated in the #42 list + (`mark_*`, `retained_roots`, `get_retained_roots`, `remove_unreachable`); + both production providers implement them; the default body is `Unsupported`, + and `lookup.rs` maps `HistoryError::Unsupported` explicitly (C9). - All four calls belong to the MCP admin profile and are refused under a - folder scope (`api/crates/layerfs-api/src/calls/scope.rs`; #42 item 39). + folder scope (`api/crates/layerfs-api/src/calls/scope.rs:33`, `out of + scope: `; #42 item 39). A folder scope guards one folder and one + Branch; none of the four is a folder operation (`branch.remove` retires the + whole Branch, `project.*` the project, `store.compact` the Store), so they + are refused rather than reinterpreted at a wider scope. +- The four calls take `branch.remove(project, branch)`, + `project.delete(project)`, `project.remove_file(project, path)` and + `store.compact(name)`; no options. +- MCP: the admin classifier checks its admin list before the work families; + the four calls join it and `project.remove` leaves both the call table and + that list. `whyOf`, the profile prose and `--profile admin` usage name + the four calls; an old MCP `project_remove` is `InvalidParams "unknown + tool"`, a CLI usage error exits 2, and the library reports `InvalidInput`; + the precedence test covers `branch.remove` under `work`. The table becomes + 76 calls (38 work/38 admin); regenerate `gen-profiles`/`gen-coverage` and + the hard-coded 73/38/35 counts, and give `whyOf` one line per call. +- Results: `branch.remove` → `{kind: "Removed", project, branch, removed_ms}`; + `project.delete` → `{kind: "Removed", project, deleted_ms, + home_entry_removed}`; `project.remove_file` → §4.3's `Removed`/`UpToDate`; + `store.compact` → the §11 report. Success results and refusal details are + snake_case; refusal `detail` is snaked in the dispatcher (or built snake at + the refusal site), and a camelCase detail is a bug; the CLI prints the + rendered map; §11 names are logical. +- `MountsActive` carries `detail.holders` (mounts/Sandboxes); `Busy` carries + `{entity, phase, op}` plus the PostgreSQL holders (`pid`, `client_addr`, + `backend_start`). `InUse` stays for non-GC calls only. +- `Unresolved` carries `{entity: {kind, name}, phase, op}` and is observed by + a non-owning call and settled by the owning one: `store.compact` adopts a + dead compaction op; `project.delete`'s settlement reads the Store by the + recorded stack id (retired/absent finishes the entry, live redoes the + marker under the project op); `sandbox.close`/`machine.remove`/ + `store.remove` clear their own ops. `logs.reset` only truncates `calls.jsonl`; it cannot clear an op. + A live owner stays `Busy`; a provably dead, certain op is ignored. +- Capacity is the real `PersistenceError::Refused { status: "Capacity" }` + path (`StorageError::is_capacity_refusal`); add `StorageError::CapacityExceeded` + to that predicate, keep PG 53100/53200/53400/54000 (53300 + `too_many_connections` is transient and maps to `Busy`; 54001 is out), and + name the SQLite codes (`SQLITE_FULL`, `SQLITE_TOOBIG`, `SQLITE_NOMEM`). + `CapacityExceeded` (a codec/canonical bound) is documented as + `InvalidInput`/`ProviderFailure`, not disk exhaustion. The `_ => Refused` + arm gets an explicit Capacity arm. ## 11. Report -While compaction runs, it writes its phase (`mark`, `build`, `verify`, -`install`) into the op, and `store.status` shows it with the op's other fields +While compaction runs, it writes its phase into the op (`mark`, `build`, +`verify`, `install` on SQLite; `mark`, `build`, `swap`, `cleanup` on +PostgreSQL), and `store.status` shows it with the op's other fields (`control/crates/layerfs-home/src/op.rs:189-201`); there is no `PROGRESS` -option (#42 item 40). `store.compact` returns: +option (#42 item 40). On PostgreSQL `store.status` reads the Store-side +`gc_op`; on SQLite it reads the home op. The `op` projection is +`{id: op_id, name, phase, started: started_ms, uncertain}` from +`gc_op`/`gc_intent`, and `state` is `busy`/`unresolved` from it; a remote +SQLite Store's op is read from its home/sidecar via `step_on`. `store.compact` +returns: | Field | Meaning | |---|---| | `path` | `none` or `rebuild` | | `generation` | `{ before, after }` | -| `logicalDiscarded` | canonical bytes of collected objects | -| `backendReusable` | SQLite: the old file's free pages; PostgreSQL: 0 for rebuilt tables | +| `deadBytesBound` | bytes in wholly dead packs after the metadata walk; the value that chose the path | +| `deadBytesRefined` | the same bytes after the payload sweep (§7 step 4) | +| `logicalDiscarded` | the collected history rows' `logical` bytes (the `store_totals.logical` delta) | +| `backendReusable` | SQLite: rebuild = the new artifact's `freelist_count * page_size` before the rename (normally 0), none = the live Store's; PostgreSQL: 0 (no in-place reclaim) | | `physicalShrink` | SQLite: old file size minus new; PostgreSQL: `pg_total_relation_size` delta | -| `keptWaste` | bytes of dead objects in retained packs, bytes kept only as delta bases (with their references), and the kept `inode_value_block` bytes | - -A zero is always explained. Estimates are marked. +| `keptWaste` | object: `{deadInRetainedPacksBytes, deltaBases: {bytes, references: [id]}, inodeBlocksBytes}` | +| `peakTempBytes` | SQLite: the new Store + rollback journal/WAL + the `visited` scratch file and its journals/super-journal; PostgreSQL: the scratch schema + WAL | +| `residue` | object `{bytes, artifacts}`; a nonzero residue is explained | + +The result is `{kind: "Compacted", ...}`. `keptWaste` is +`{deadInRetainedPacksBytes, deltaBases: {bytes, references: [id]}, +inodeBlocksBytes}`; `residue` is `{bytes, artifacts: [string], reason?}`; +top-level `estimate: bool`; a capped field is `{value, cap}`; `zeroReason` +is per zero field with the closed vocabulary `empty-store`, `none-path`, +`no-history`, `no-waste`, `postgresql-in-place`; `markBytes` is a #52 metric, +not a result field. The SDK/CLI render every key +snake_case. On the `none` path `deadBytesRefined` equals `deadBytesBound` (no +payload sweep runs below the threshold), and the report says so. `physicalShrink` +reads the old size before the rename; `backendReusable` reads the SQLite +free-page count × page size before it; `logicalDiscarded` is the summed +`logical` of the collected history rows; PostgreSQL reports 0 for rebuilt tables and does +not count in-place free space. `physicalShrink` samples `pg_total_relation_size` +before and after the swap; on PostgreSQL `deadBytesRefined` equals +`deadBytesBound` (there is no payload sweep to refine it). A zero is always +explained; a capped field carries its cap (S7). ## 12. Acceptance (#52) +Each bullet is one case; its label is its case ID and must be unique. + - **Crash injection** (`kill -9`, per backend) at mark, build, verify, before and after install: wholly old or wholly new; the leftover is judged by §9. +- **Backend matrix:** every removal, admission, report and name item runs on + SQLite and PostgreSQL; a converted 1.0 Store opens with the additive step on + both. +- **Crash matrix:** named failpoints at every phase (SQLite: mark, build, + verify, install, WAL switch, artifact removal, op settle; PostgreSQL: mark, + build, swap, cleanup, scratch drop), plus lost COMMIT, power-loss + (fsync/rename) and unresolved op/lease takeover; assert §9 state, + generation, digest and closure. - **No live object lost:** property tests over random fork / commit / merge / publish / `branch.remove` / `project.delete` / `remove_file` histories; closure sets compared before and after. @@ -512,18 +1093,178 @@ A zero is always explained. Estimates are marked. transaction open at swap time makes the swap abandon. - **Admission races:** a mount and a removal of the same Branch started together never both succeed. -- **Names:** `NameTaken` after removal (Branch and project), a free name after - compaction collects the row, `ReservedName` for `main`. +- **Names:** `NameTaken` after removal (Branch and project) and on create + collision (Branch and project; project create was `Exists`), a free name + after a rebuild collects the row, `ReservedName` for `main`. - **`remove_file`:** `Removed` for Commits at or below the frontier, before and after compaction; the next Commit of a mounted older head records the - deletion; a file created later at the path is visible; bytes reclaimed only - when no other path, project or delta chain uses them. -- **Speed and space** (one sample per arm, declared caches, generated - fixtures): downtime per GB of live data and per GB freed, bytes written, - peak disk, `ACCESS EXCLUSIVE` duration on PostgreSQL. The rebuild's build - phase is compared with plain `VACUUM INTO` of the same Store; the mark is - reported separately. A storage option that costs about 50% speed for about - 5% storage is rejected. + deletion; a file created later at the path is visible; 2.0 frees no bytes + for the removed path (visibility only). +- **`remove_file` and derived nodes:** a non-fast-forward `branch.merge` of + two heads at or below the frontier keeps the path hidden (the whiteout is + materialized), and a file created after the frontier is visible; a + fast-forward merge is the control (no new tree); a `branch.publish` of a + Commit at or below the frontier yields a hidden Layer, a mount of it + whiteouts, and its next Commit records the deletion; a merge that cannot + apply the whiteout is refused `Removed`. +- **Removed path in a merge:** a non-fast-forward merge of two heads at or + below the frontier reports no conflict for the removed path and omits it; a + merge of an at-or-below head with a post-frontier re-creation keeps the + re-created file, and only an ordinary edit conflict on it is reported. +- **Unmerged Branches:** `remove_file` hides the path in a Commit on an + unmerged Branch created before the frontier; a Commit on that Branch after + the frontier does not inherit it. +- **Layer-headed mount:** a Branch whose head is empty and whose base Layer is + at or below the frontier whiteouts the path at mount. +- **Visibility only:** `remove_file` frees no bytes; an object reachable only + through the removed path is retained; the marker hides reads/listings and a + mount/merge/publish whiteouts the path. +- **Published Branch:** a Branch published by a retained Layer, or named by + a retained `commit.branch_id`, keeps its name after `branch.remove`; only an + unreferenced Branch's name is freed by a rebuild (not by a `none` pass). +- **Late base:** a delta base in an already-passed pack and a depth-2 chain + are copied by the worklist; `pack_id` is preserved. +- **Digest roots:** an uncommitted saved file's `file_digest` row is collected + unless another retained root reaches its content. +- **Path grammar:** `/a` and `a` are accepted; empty, `/`, `//a`, `a/`, + `a//b`, `./a`, `a/./b`, `.`/`..` components, `\`, NUL, >4096 bytes and >256 + components are refused `InvalidInput`; a directory hides its subtree; a + child of a removed non-directory is `NotFound`; an intermediate symlink is + `InvalidInput` and a leaf is hidden; a hardlink survives. +- **Repeated removal:** a second `remove_file` after a re-creation hides the + re-created file; a repeat with no newer project node is `UpToDate`, while a + commit in another project neither rewrites the marker nor advances it; a + never-existing path is accepted. +- **Marker epoch:** a one-shot write racing `branch.remove` or + `project.delete` gets `Conflict` and leaves no Commit; a merge racing + `remove_file` gets `Conflict`, and the retry applies the whiteout. +- **Concurrent merge and removal (property):** a merge and a `remove_file` + started together never leave the path visible in a derived tree: one side + wins, the other either sees the marker or gets `Conflict` and retries with + the whiteout. +- **Marker cleanup:** a rebuild of a Store with a deleted project leaves no + `path_removed` row for it and keeps a live project's markers; a `none` pass + writes nothing and retains them. +- **Position state:** a converted Store backfills serials; + `next_history_serial` never falls after compaction; `project_epoch` copies + and bumps only on a marker write. +- **Marker survival:** a retained `branch_retired`/`layer_stack_retired`/ + `path_removed` survives a rebuild; a collected row takes its marker with it. +- **project.delete retry:** the marker commits, the home removal fails → + hidden, entry present, `home_entry_removed: false`; a same-home retry + finishes and returns the stored `deleted_ms`, with no second marker; the + own-Init identity is saved before the marker and a save failure removes + nothing. +- **Home listing:** a second home's stale entry is absent from `project.list` + and `project.get` answers `NoSuchProject`; Store calls agree. +- **PG cleanup:** a Commit/save during Phase C survives the high-water bound; + a crash resumes from `gc_op`; a marker and its base row are deleted in one + batch (no resurrection/orphan), with a failpoint at the boundary. +- **Admission:** a mount from another home/machine is visible in the Store-side + intent; `machine_stopped` is ignored; a second mount of one project + proceeds; an uncertain dead op is `Unresolved` and the owning call clears it. +- **NameTaken:** `detail {holder: {kind, name, state: live|retired|deleted}, + source}`; a forced two-reference fixture (Layer FK and `commit.branch_id`) + keeps the name; one reference and neither are the controls. The error + matrix covers `MountsActive`, `Busy`, `NoSuchBranch`/`NoSuchCommit`/ + `NoSuchProject`, `NameTaken`, `ReservedName`, `Removed`, `InvalidInput`. +- **Trace salt:** the call log stores `` for the path; the trace key + is the keyed hash; `Removed` echoes the path; a rebuild copies the salt. +- **Admission (liveness):** a live-but-unreachable Sandbox is `Busy`; + `machine_stopped` is ignored; a `gc_intent` from another home is visible; + `Unresolved` is observed by a non-owning call and settled by the owner. +- **Checkpoint/install:** a live reader makes `wal_checkpoint(TRUNCATE)` + return `busy>0` → `Busy`, no artifact, no op; a sidecar or `-mj*` at install + is `Conflict` with no writes. +- **Bind failure:** a committed intent whose bind fails is deleted in the + error path; a failed delete is reported; no `MountsActive` remains. +- **PG refcount:** two mounts keep the Store/project keys after one unmount; + a failed all-or-none acquire unlocks the taken subset. +- **Worker/lost reply:** an unreachable worker op is `unknown, never clean`; + a lost reply keeps the sidecar and the retry resolves retired/live/absent. +- **Trace:** the six path names are redacted; refusal details are not traced; + the salt is backfilled for an upgraded Store. +- **Artifacts (cross-backend):** SQLite leaves/removes + `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; + PostgreSQL's `lfs_gc_` is recorded before CREATE, removable from + another machine, and a stray schema is `Exists`. +- **Position state (2.0 backfill):** every upgraded Store (2.0 included) + gets one `project_epoch` row per stack; `gc_op` is cleared on settle and + not copied; `next_history_serial` never lowered. +- **Digest agreement:** SQLite and PostgreSQL report the same + `live_heads_digest` for the same fixture; a retired head changes it. +- **PostgreSQL mark:** the mark transfers only `markBytes`; the table copy + moves no bytes; a base object that only a dead pack holds is kept. +- **Reserved pack:** a Save that reserves a pack/ordinal before the mark and + publishes during Phase A (or Phase C) moves `store_totals`; the swap + `Conflict`s, the pack stays in the old Store, and the cleanup failpoint + keeps it; no dangling location. +- **PG cleanup (resume):** after the swap the Store is usable at `G + 1`; + Phase C resumes after a crash; totals/ACL/trigger/hints are asserted (no + `inode` on PG; that assertion belongs to the SQLite install). +- **PG privileges and reply:** missing `CREATE`/`TEMP`/`pg_read_all_stats` is + `Unsupported`; a lost `COMMIT` reply is resolved by proving the backend + ended, then reading the incarnation. +- **SQLite layouts:** schema 36 (`Monolithic`), 37 (`GroupRows`), 38 + (`GroupRowsIndexed`) and their 42/43/44 acquisition variants rebuild; + `Monolithic` copies `pack` (body) and no `pack_unit`; `GroupRows` copies + both; `GroupRowsIndexed` copies `pack_unit` and the schema creates its + index (not "copied"); `sqlite_sequence` is kept when present and + `init_*`/acquisition tables are empty. +- **Install fence:** a connection that bypassed the writer lane and opens a + transaction after install gets `StaleStore`; an in-flight transaction is + excluded by the lane (a documented limit); a sealed-state or sidecar change + makes install `Conflict` with no writes. +- **Artifacts (SQLite exact):** a crash leaves + `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`; the next + compact removes all of it; `none` leaves no `.compact`; a stray name with + no op is `Exists`. +- **Threshold and failures:** 9% wholly dead → `none`; 10% and 30% → + `rebuild`; an empty Store (0/0, COALESCEd sums) → `none` with a + `zeroReason`; a started rebuild completes when the refined share drops + below 10%; `Capacity` and a verify failure abort, delete the artifact set + and leave the old Store unchanged apart from step 1's checkpoint. +- **Mount/compact race:** a mount during compact is `Busy` with the compact + op; compaction returns `MountsActive` when a mount wins; a Commit during the + rebuild is not lost (PG: a write during Phase A makes the swap `Conflict`; + during Phase C it survives the high-water bound). +- **Profile and scope:** all four calls are refused under a folder scope + (`out of scope`) and allowed only in the MCP admin profile. +- **Progress:** while `store.compact` runs, `store.status` shows the op phase + (`mark`, `build`, `verify`, `install` on SQLite; `mark`, `build`, `swap`, + `cleanup` on PostgreSQL). +- **Report:** both paths; `kind:"Compacted"`, snake_case keys, every §11 + field with its unit and sampling point (`freelist_count * page_size`, old + size before rename, history `logical` delta, peak temp = new file + + journal/WAL + visited, `physicalShrink`, `keptWaste` components, + `residue {bytes,artifacts}`, `estimate`, capped `cap`, a `zeroReason`); + `markBytes` is a #52 metric (PG-only, not a result field); + `keptWaste` is `{deadInRetainedPacksBytes, deltaBases:{bytes,references}, + inodeBlocksBytes}`; bound≥refined; PG `backendReusable = 0` and PG/none + `deadBytesRefined == deadBytesBound`. +- **Read surface:** `file.read`/`stat`/`list`/`glob`/`diff`/`revision`/ + `export`/cp-from before and after compaction, with the §4.3 shapes; add + `edit`/`grep`/`blame`/`rm`/`mv`/`mkdir`/`cp`/`apply`, the + visible/hidden/absent diff table, `removed_count` (0 and >0), + `branch.diff`/`commit.diff`, and one whiteout case per one-shot tree + creator (`write`/`edit`/`apply`/`rm`/`mkdir`/`mv`/`cp`). +- **Restore and listing:** `backup.restore` before and after a removal; a + second home's stale entry; the §4.2 guard test and allow-list. +- **Error matrix:** exact code and message per case, including `StaleStore`, + `Removed`, `InvalidInput` for `project.remove`, `Exists`, `Unsupported` and + `Capacity`. +- **Digest golden vector:** a fixed-id fixture (out-of-order ids, a retired + Branch and stack, differing timestamps) hashes to one literal vector on both + backends; an absent head is 33 zero bytes, not 32, and widths are fixed at + 17/33. +- **Speed and space** (at least three runs per arm, median and range, + declared caches, generated fixtures): downtime per GB of live data and per + GB freed, bytes written, peak disk, `ACCESS EXCLUSIVE` duration on + PostgreSQL. Downtime per GB freed is absolute bytes when nothing is freed + (the `none` path and removal-only runs). The rebuild's build phase is + compared with plain `VACUUM INTO` of the same Store; the mark is reported + separately. A storage option that costs about 50% more time for about 5% + less space is rejected. ## 13. Estimates, not measurements @@ -532,3 +1273,18 @@ the review of 2026-11-14 estimated about 2–3 minutes of downtime for §7, and hours if the full doctor ran before and after. For PostgreSQL with the same Store, about 9 minutes in total with `ACCESS EXCLUSIVE` held for about two seconds. Nothing has been measured; #52 owns the numbers. + +#52 measures, per backend, these arms: `none` at 9% wholly dead; `rebuild` at +10% and 30%; a 50 GB total / 35 GB live (30% dead) Store, plus a 50 GB +live-store arm; and each removal type (branch, project, remove_file) as a +marker-only arm, plus removal-then-compact at a declared share. A +`remove_file`/`remove_file`-then-compact arm is 0-freed and reports absolute +bytes. At least three runs per arm, median and range; declare the hardware, +SQLite 3.53.2 with `page_size`, `journal_mode`, caches, `synchronous_commit` +and PostgreSQL version. Record the +§11 fields, `markBytes`, downtime per GB of live data and absolute bytes when +nothing is freed (0/0 or removal-only), bytes written, peak disk, +`ACCESS EXCLUSIVE` duration, WAL bytes and the `VACUUM INTO` baseline; use +`store.status` for the phase. A storage option that costs about 50% more time +for about 5% less space is rejected, naming the option and the measured +numbers. diff --git a/core/docs/issues/project-41/gc-42-decisions.md b/core/docs/issues/project-41/gc-42-decisions.md new file mode 100644 index 0000000000..eb45bbac9a --- /dev/null +++ b/core/docs/issues/project-41/gc-42-decisions.md @@ -0,0 +1,581 @@ +# #42 decisions: GC 2.0 contract amendments and SQLite implementation deviations + +> **Status:** Proposal for layerfs-project#42 owner sign-off; not a released +> contract. The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) (frozen for +> layerfs-project#41 on 2026-11-15, baseline `ed14ed53b`) is **byte-identical** +> on this branch: nothing here is written into it. On sign-off the accepted +> entries become spec text (or a new #42 decision entry), the spec's decision +> table indexes them, and this file is retired. + +**Why.** The #41 implementation and the #711 fix-branch integration need a +different reading than the frozen sentences in sixteen places: eleven contract +amendments (A1–A11, mostly refusal detail shapes, one additive sidecar field, +seven `gc_op` columns, the snake_case rendering rule, the one-valued +`deadBytesRefined` and the §10 in-process `Busy` mapping) and five SQLite +implementation deviations (D1–D5) that the +frozen text either forbids or does not describe. `gc-spec-gaps.md` (one of the implementation notes moved out of +the tree; its decisions are folded into these entries where they carry contract +text) already flagged two of the earlier seven as "#42 correction"; this note is the +complete list, and it changes no product behaviour. + +The seven A entries before A8 were written directly into the frozen file +while #41 was implemented and were backed out before review (commit +`122a168502`); A8–A12 are added by this note for the same sign-off. +The five D entries are present in the shipped SQLite code and are recorded here +so the contract and the implementation do not drift apart. + +G7 records one **deferred design decision** (the install-only lane). It +continues the retired G1–G6 series, changes no contract text and keeps the +frozen whole-lane exclusion; its measurements live under +`tools/bench/installlane/`. + +--- + +## A. Contract amendments (11) and one reading + +### A1. §4.1 `project.delete` sidecar gains `op_id` (additive) + +- **Frozen:** `/store//delete/.json` is + `{store, project, stack_id, deleted_ms, attempt_ms, state}`. +- **Proposed:** add `op_id`, the project op the admit/mark attempt ran under. + An older sidecar without `op_id` stays valid. The retry then settles a lost + admit/mark reply instead of leaving it `Unknown` (§5.1, §10). +- **Reason:** the retry cannot otherwise name the op whose reply was lost. + +### A2. §8 `gc_op` gains the `usename` column + +- **Frozen:** `gc_op` records `artifact, pid, backend_start, incarnation`. +- **Proposed:** also record `usename`, so the lost-COMMIT recovery statement + can terminate the owner's backend by `(pid, backend_start, usename, datname)` + and not merely by pid/backend_start. +- **Reason:** `pg_terminate_backend` recovery in §8 matches all four fields. + +### A3. §4.3 merge whiteout refusal detail is explicit + +- **Frozen:** a merge that cannot materialize the whiteout is refused + `Removed` with `detail {paths: [...]}`. +- **Proposed:** `detail {paths: [{path, frontier_serial}]}`, listing **every** + unmaterializable path as a full element, deduplicated and sorted by + normalized path bytes. +- **Reason:** a merge can find several paths at once; the client needs the + exact shape and a stable order. + +### A4. §4.3 `Removed` refusal detail is one shape on every call + +- **Frozen:** "Every refusal carries one detail shape: `{paths: [{path, + frontier_serial}]}` (one element for read/stat; the governing maximum + frontier and the request path)." +- **Proposed:** a `Removed` refusal always carries that one shape. For + `file.read`/`file.stat` it is one element: the request path with its + governing maximum frontier. A merge lists every unmaterializable path, + sorted by normalized path bytes. +- **Reason:** makes the one-element vs many-element cases explicit instead of + parenthetical. + +### A5. §4.3 hidden-path read surface: `cp`/`edit` vs `mv`/`rm` + +- **Frozen:** "`project.export` omits it; a cp-from source is `Removed`; + `file.edit` is `Removed`; `file.mv` from a hidden path is `NotFound` and to a + hidden path is a visible re-creation; `mkdir`/`cp`/`apply` follow the same + construction rule; `file.rm` of a hidden path is `NotFound`." +- **Proposed:** "`project.export` omits it; `file.cp` from a hidden source and + `file.edit` are `Removed`; `file.mv`/`file.rm` from a hidden path return the + `NotFound`/`NoSuchPath` wire code, not `Removed`; `file.mv` to a hidden path + is a visible re-creation; `mkdir`/`apply` follow the same construction rule." +- **Reason:** the wire code for a hidden `mv`/`rm` source is `NotFound` / + `NoSuchPath`; naming `cp` in the construction list contradicted its own + `Removed` sentence. + +### A6. §10 code table gains `Unknown` + +- **Frozen:** §10's table has no `Unknown` row, although §4/§4.2 use it. +- **Proposed:** `Unknown` is a service/control-plane outcome, not a + `HistoryError` code: a lost reply, or a Store listing that cannot be + determined. Detail `{entity: {kind, name}, phase, op}`, with `phase`/`op` + `null` when no durable op is known. +- **Reason:** callers must be able to distinguish a lost reply from a refusal; + the row makes the existing §4 use explicit. + +### A7. §10/§11: §11 names are logical; wire/SDK/CLI keys are snake_case + +- **Frozen:** "The SDK/CLI render every key snake_case." +- **Proposed:** the §11 names (e.g. `deadBytesBound`, + `keptWaste.deadInRetainedPacksBytes`) are logical Rust names only; the wire, + SDK and CLI keys are snake_case (`dead_bytes_bound`, + `kept_waste.dead_in_retained_packs_bytes`), per §10. +- **Reason:** the single sentence did not say which names are logical and + which are rendered; examples remove the ambiguity. + +--- + +### A8. §11 report: canonical `logicalDiscarded`, source-Store `backendReusable`, two more zero reasons, and the pre-release `gc_op` columns + +- **Frozen:** §11 defines `logicalDiscarded` as "the collected history rows' + `logical` bytes (the `store_totals.logical` delta)"; `backendReusable` as + "SQLite: rebuild = the new artifact's `freelist_count * page_size` before + the rename (normally 0), none = the live Store's; PostgreSQL: 0"; and the + zero vocabulary as `empty-store`, `none-path`, `no-history`, + `no-waste`, `postgresql-in-place`. +- **Proposed:** + - `logicalDiscarded` is the **canonical bytes of the collected objects**, + deduplicated by object id: a tree shared by several collected roots counts + once, and a collected object that no history row reaches (an orphan) + still counts. SQLite reports `tuple.canonical - built.canonical` after + the install; PostgreSQL reports `swapped.dead.canonical`. It is not the + `store_totals.logical` delta and not a sum over history rows, which + double-counts shared trees and misses an orphan. + - `backendReusable` is the **source Store's pre-mark free-page bytes**: + `freelist_count * page_size`, sampled before the mark, on a rebuild. That + is the space the run judges and reclaims; the freshly vacuumed target is + empty by construction and names nothing. The `none` path keeps the live + Store's count, and PostgreSQL stays 0 (`postgresql-in-place`). + - the §11 zero vocabulary gains two words: `no-free-pages` (a SQLite rebuild + reports `backendReusable = 0`, so the source Store had no reusable free + pages) and `no-shrink` (a rebuild collected dead bytes but + `physicalShrink = 0`, as a resumed `G + 1` run can). Neither is + explained as `no-waste`. +- **Schema (pre-release):** PostgreSQL's `gc_op` also carries seven columns. + The mark (Phase M, `persist_mark`) writes `op_id BYTEA` (the Store + compaction op the round runs under — `GcIntentKind::Compaction` with + `GcScopeKind::Store` in `.../postgresql/compact/run.rs` — so a retry that + loses its reply can name that op), + `mark_fingerprint TEXT` (the `pack`/`object_location` + `pg_attribute`/storage/tablespace fingerprint Phase B re-checks against the + scratch tables), `mark_visited BIGINT` (the mark's visited-object count, the + exact retained set Phase C asserts) and `kept_waste BYTEA` (the retained + waste the mark counts, encoded as below). The swap writes + `physical_before BIGINT NOT NULL DEFAULT 0`, + `physical_after BIGINT NOT NULL DEFAULT 0` and + `peak_temp BIGINT NOT NULL DEFAULT 0`, so a resumed rebuild reports the + swap's measured shrink (the before size sampled before the swap + transaction opens, the after size sampled inside it), peak temp and the + mark's retained waste instead of zero placeholders; a save between the swap + and Phase C cannot inflate the resumed shrink. 2.0.0 is + unreleased, so the columns go into the 2.0 schema creation in place: there + is **no additive open-time step** (`add_*`) and no compatibility arm for a + 2.0 `gc_op` created before them (the Disposable profile, §2, and item 42). + `kept_waste` is a versioned + encoding of the §11 `keptWaste`: one version byte (1), three big-endian + `u64` byte totals, the capped flag, a big-endian `u32` reference count + and the identity bytes. +- **Reason:** the frozen sentences describe history-row accounting as object + accounting (a shared tree is counted twice and an orphan is missed) and + name a target artifact that is empty by construction, so the report never + names the free space a rebuild reclaimed. The two zero words keep a zero + honest where `no-waste` conflated "nothing was collected" with "the size + did not change"; the `gc_op` columns are what let a resumed rebuild finish + the report from the swap's own measurements. +- **Implementation:** the #711 report fix (`fix711-report2`, head + `129c4b59c7`) carries the code: the canonical `logicalDiscarded` on both + backends, the source-Store `backendReusable` sample, the two zero words and + the `gc_op` columns. +- **Where:** `engine/crates/layerfs-persistence/src/report.rs` (the shared + `CompactReport`), `.../backend/sqlite/compact/{run,receipt}.rs`, + `.../backend/postgresql/compact/{run,report}.rs` and + `engine/crates/layerfs-persistence/sql/postgresql/gc.sql`. The wire keys + stay snake_case (`logical_discarded`, `backend_reusable`, `zero_reasons`) + under A7. + +### A9. §4.2: the retry's Stack read is the marker plus the filtered by-id lookup + +- **Frozen:** §4.2 allow-lists `layerstack_stack_by_id_retired` as "the retry's + live/absent read by the caller's recorded stack id", and keeps the ordinary + `layerstack_stack_by_id` filtered. +- **Proposed:** no statement of that name is shipped. The `project.delete` + same-home retry resolves the caller's recorded Stack id with two statements + that already exist: `layerstack_deleted_ms` (the marker's `deleted_ms`, or + none) and the filtered `layerstack_stack_by_id` (the live row, or absent). A + retired Stack reuses its marker's `deleted_ms`, a live Stack redoes the + marker, and an absent one finishes the entry. No unfiltered by-id statement + is needed: the filtered read is the live/absent half the frozen sentence + names, and a retired or deleted project must not resolve there. +- **Reason:** the frozen name describes one statement that the shipped code + splits into two. The split keeps every ordinary Stack lookup filtered and + returns the same retired/live/absent classification the frozen sentence + describes, so no behaviour changes and no resurrection path is added. +- **Alternative recorded, not adopted:** add a dedicated + `layerstack_stack_by_id_retired` statement. It duplicates + `layerstack_stack_by_id`'s row shape only to return the same absence the + filtered read already returns, and it would need its own allow-list + justification. +- **Where:** `engine/crates/layerfs-persistence/src/history/admission/removal.rs` + (`delete_stack`), `.../src/history/layerstack.rs` (`stack_deleted_ms`, + `layer_stack`), and `control/crates/layerfs-store/src/project/delete.rs` + (the retry path). + `engine/crates/layerfs-persistence/tests/visibility_guard.rs` pins the two + statements and the statement set. + +### A10. §6.4/§11: `deadBytesRefined` equals `deadBytesBound`; there is no second payload sweep + +- **Frozen:** §6.4 chooses the path "using the bytes of packs that hold no live + object after the metadata walk (an upper bound that §7 step 4 refines)", and + §11 samples `deadBytesRefined` as "the same bytes after the payload sweep + (§7 step 4)". §11 already states that the value equals `deadBytesBound` on + the `none` path and on PostgreSQL, because neither runs a payload sweep there. +- **Proposed:** the field and its wire key stay; the value is the bound on + **every** path and on both backends. The mark resolves every logical reference + and delta base, so `visited` is the whole retained object set and the build + copies exactly those, with no second payload sweep; §8 Phase M resolves the + same closure before the swap. No live object can be discovered after the + mark, so no refinement exists to report (`deadBytesRefined == + deadBytesBound`, including a SQLite `rebuild`). +- **Reason:** §6.4's parenthetical and §11's "after the payload sweep" describe + a second pass over the live payload that the implementation does not run (the + walk that refined the bound is the mark itself, so the two numbers cannot + differ). Keeping a separate field that promises a value the + implementation cannot produce is the drift this note exists to stop; keeping + the field at its documented value keeps the §11 shape, the snake_case wire + key (A7) and the `bound >= refined` invariant that clients read. +- **Where:** `engine/crates/layerfs-persistence/src/backend/sqlite/compact/receipt.rs` + (`dead_bytes_refined: dead_bytes`, and the `deadBytesRefined` zero reason), + `.../backend/postgresql/compact/{run,cleanup}.rs` (both `report(..)` calls pass + the bound as `dead_refined`), `.../backend/sqlite/compact/types.rs` (the + field's doc comment) and `.../backend/sqlite/compact/mod.rs` (the module doc + states the equality). `.../compact_metrics.rs` keeps recording the field, and + the capacity receipt + `engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure.json` + records `deadBytesRefined == deadBytesBound` on both arms. +- **Alternative recorded, not adopted:** restore a refinement pass, i.e. a second + payload scan after the mark that re-derives each pack's dead share from the + admitted references. It would need its own op phase and artifact-set entry, it + reads up to the live payload a second time under the Disposable profile (§2), + and it cannot change the path choice (§3 takes the bound), so it is not built + for 2.0.0. +- **Effect on #52:** AC-52's golden pins both numbers and AC-57's receipts carry + them, so the reports stay valid; no acceptance case asks for a value that + differs from the bound. The frozen spec file stays byte-identical until + sign-off: this entry, like A8 and A9, is the amendment #42 records. + +### A11. §10: `store.compact`'s in-process call refusal is `Busy`, not `InUse` + +- **Frozen:** §10 says `Busy` carries `{entity, phase, op}` plus the + PostgreSQL holders and that "`InUse` stays for non-GC calls only". It does + not name the code `store.compact` reports when the open cache refuses to + close a Store because a call still runs on it in this process. +- **Proposed:** that refusal is `Busy`, naming the Store as `entity` with + `phase` and `op` null: no operation holds the Store, a call does. The + cache's own `InUse` is reported by the non-GC calls that close a Store: + `store.push` and `store.pull` (through the shared transfer's `settle`), + `store.remove`, `store.disconnect` and `backup.restore`. The list is + illustrative, not exhaustive. +- **Reason:** `store.compact` is a GC call, so its refusals speak the GC + vocabulary; reporting the open cache's `InUse` from the compaction would put + a non-GC code on the call that §5's `Busy` and §10's code table govern. +- **Where:** `control/crates/layerfs-store/src/compact.rs` (`gc_refusal`, the + open-cache mapping), `control/crates/layerfs-store/tests/compact_in_process_call.rs` + (`a_running_call_is_busy_for_the_store_not_in_use` and + `exactly_the_open_cache_refusal_is_mapped`), the `store.compact` error row + in `docs/architecture/api/sdk/store.md`, and the frozen spec file, which + stays byte-identical until sign-off: this entry is the amendment #42 + records, like A8–A10. + +### A12. §4.1/§5.1 reading: a rebuild carries the run's own `gc_intent` row + +- **Frozen:** §4.1 says `gc_intent` "is the shared Store-side admission table + (created empty, never copied)"; §5.1 says compaction "writes its `gc_intent` + operation row before its first checkpoint; the row lives in `.compact` + during build/verify, moves to the live Store after install, and is cleared + at settle." +- **Reading:** this is a reading, not a twelfth contract amendment; it changes + no contract text and the eleven amendments A1–A11 stand. The two sentences + agree when "never copied" means no admission row of the sealed Store is + carried across a rebuild: the fresh file starts with `gc_intent` empty. The + run's own row is not a copy of an existing intent — the compaction writes + it, and §5.1 requires it to live in the build artifact. So the rebuild + stages the table empty and then inserts exactly the row whose `op_id` is + this run's. +- **Where:** `engine/crates/layerfs-persistence/src/backend/sqlite/compact/build.rs` + (`copy_rule` returns `Copy::Empty` for `gc_intent`, and the `copy_rows` + special case inserts only `op_id=x''`; the `copy_rule` doc comment + says it "keeps only this run's admission intent"). +- **No contract change:** neither frozen sentence is edited; the spec stays + byte-identical until sign-off, like A8–A11. + +--- + +## D. SQLite implementation notes: four deviations and the D2 build decision + +This section keeps five records. D1, D3, D4 and D5 are deviations from the +frozen text that the shipped code still carries. D2 is the build decision that +replaced the whole-Store copy: the fresh-copy rebuild follows S5's "fresh +Store" sentence but deviates from §7 step 4's payload sweep and rollback +journal and from §11's peak-temp line, and the earlier `VACUUM INTO` + delete + +`VACUUM` build is historical. Each paragraph states the frozen sentence, the +behavior and the decision. The code in +`engine/crates/layerfs-persistence/src/backend/sqlite/compact/` is the +measured implementation of this branch. + +### D1. §7 step 4: the fresh load runs with `foreign_keys=OFF` + +- **Frozen:** "`PRAGMA foreign_keys=ON` outside the transaction (inside it the + pragma is a no-op), then one rollback-journal transaction." +- **Implemented:** the fresh file's bulk load runs with `foreign_keys=OFF` and + copies only the retained rows, in foreign-key order. The final state is + proven by the §7 step 6 `foreign_key_check` on an `immutable=1` open before + the rename, and the load never deletes from an enforced Store because there + is no prune: the old whole-Store copy's delete order (which the deferred + counter refused even with a clean final row set) is gone. The pragma stays + off for the load so the ordering cannot turn a clean retained set into a + `SQLITE_CONSTRAINT_FOREIGNKEY` commit. +- **Decision requested:** record the deviation (the pragma is off for the load, + the post-load check is the guard) or require enforcement during the copy. + The current code is the tested, reproduced behavior. + +### D2. S5/§3/§7: the rebuild is the fresh-copy build; peak is old + the new Store + the visited scratch + +- **Frozen:** S5 says "A rebuild copies only retained rows into a fresh Store"; + §7 step 4 describes the payload sweep and one rollback-journal transaction + with `foreign_keys=ON`; §3 says a rebuild copies at least nine live bytes per + byte freed below the 10% threshold; §7's cost sentence bounds peak disk as + old Store + the new Store + the rollback journal/WAL + the visited scratch + file. +- **Implemented (superseding the earlier whole-Store copy):** the build + creates `.compact` with the sealed Store's schema, copies only the + rows the mark retained (pack-as-unit), builds the secondary indexes and + triggers after the bulk load, verifies the fresh file and renames it. It + stages no whole-Store copy, writes no prune journal and runs no `VACUUM`. + The earlier `VACUUM INTO` + delete + `VACUUM` build wrote the retained bytes + about four times over and needed a ~2.9× peak; §11's `peakTempBytes` for + SQLite is now the new Store plus the visited scratch, because the fresh path + stages no whole-Store copy and writes no rollback journal (a deviation from + the frozen §11 line, recorded here). +- **Measured:** the validated prototype (`tools/bench/freshcopy/`, branch + `val-freshcopy`) on a 10 GiB store-shaped fixture: fresh 32.5 s, 1.007× live + written, peak 1.003× `(old + live)`; the old build 120.8 s, 4.03× written, + 1.577× peak. The 50 GB `VACUUM`-build numbers in `gc-runbook.md` are + historical and are replaced by this cost model. +- **Decision (owner, this branch): the fresh-copy build is the default + rebuild.** The old build's 50 GB measurement (5,131.835 s, 143.1 GB sampled + peak, 147.609 GB bound) stays the record for that algorithm only. +- **Alternative recorded, not adopted:** "clone + delete + incremental + vacuum" — `fs::copy`/`copy_file_range` (reflink on xfs/btrfs) the sealed + file, delete dead rows in the clone with `journal_mode=OFF`, run + `incremental_vacuum` until the free list is empty, verify, rename. It follows + dead bytes instead of live bytes and can hold peak near one Store copy. It is + not adopted: the fresh copy already writes ~1.0× live and needs no reflink + size threshold. +- **Runbook consequence:** a rebuild needs old Store + about one live Store + (+ margin) free, not 2.9× the Store; re-check free space before every + rebuild. The runbook carries this (see its "Disk headroom" section). + +### D3. §3/§7 step 1.5: the `none` path writes a transient visited scratch + +- **Frozen:** §3's `none` row is "Mark only; nothing is written"; §7 step 1.5 + says "On `none`, delete the scratch and stop: no `.compact`, no + `.compact-visited`, no residue." +- **Implemented:** both paths spill the closure into `.compact-visited` + (the metadata walk needs a bounded-memory visited set). The §6.1 closure and + the §5.1 admission intent are one write transaction: on `none` it is rolled + back, so no `kept_node` row, no `gc_intent` row and no WAL frame is + committed, and the live Store keeps its pre-call `(size, mtime, change + counter)`. Only the scratch is written; it is deleted before the call + replies and counted in `peakTempBytes`, so no `.compact`, no residue and no + generation change survive the call. +- **Decision requested:** say "nothing is written **that survives the call**": + the Store file is untouched (the mark and its intent are a rolled-back + transaction), and only the transient visited scratch is written, counted in + `peakTempBytes` and deleted before the reply (the §11 table already names the + visited scratch as part of `peakTempBytes`). Alternative is an in-memory + visited set, which the measurement rules out for large Stores. + +### D4. §11: SQLite records a fifth observable phase, `cleanup`, during settle + +- **Frozen:** "`mark`, `build`, `verify`, `install` on SQLite; `mark`, `build`, + `swap`, `cleanup` on PostgreSQL." +- **Implemented:** the SQLite settle writes phase `cleanup` to the home op + while it removes the artifact set and clears the intent, so a crash in the + settle reads `cleanup` (the next `store.compact` adopts the op). The four + build phases are unchanged; `cleanup` is the settle phase, not a build step. +- **Decision requested:** add `cleanup` to §11's SQLite phase list (it already + exists on the PostgreSQL list), or restrict SQLite to four phases and lose + the settle-phase diagnosis. +- **Runbook consequence:** `store.status` shows `cleanup` on SQLite after the + install point; the runbook's phase list includes it. + +### D5. S6/§7: `store.compact` is local-only; a remote sqlite Store is `Unsupported` + +- **Frozen:** S6 "A SQLite Store compacts on the machine where it lives + (`place::plan`, `control/crates/layerfs-store/src/place.rs:40-49`)"; §7 + "Runs as one operation on the Store's machine (S6)." +- **Implemented:** the call consults `place::plan` but runs compaction only on + the `Local` branch, in the caller's process. Its `Worker` branch is refused + `Unsupported` before the Store is opened or the machine is dialled, and the + worker has no `store.compact` operation at all. A sqlite Store pushed to a + machine (`docker`, `lima`, `apple-container`) therefore cannot be compacted + from the home that owns the record; the owner pushes it back to `native`, + compacts there, and pushes it again. `store.compact` of a PostgreSQL Store + is unchanged: the call dials the Store's recorded URL itself. +- **Reason:** the worker route needs a worker-side compaction operation, the + Store record and lease read under the worker's root, and a crash/resume + answer for a call whose run can be hours long; 2.0 ships the local engine + and the transfer route (`store.push`/`store.pull`) instead. The refusal is + named, never a silent local run on the wrong file. +- **Decision requested:** accept local-only for 2.0 as a deviation from S6 + (and say so in the API and control docs, which now name #42 D5), or require + the worker `store.compact` route before the contract closes. +- **Consequence:** `store.compact` of a Store whose record names another + machine is `Unsupported` with "local-only: run it on that Store's machine + (S6 deviation #42 D5; the worker has no compaction route)"; the operation is + refused before any effect. `api/crates/layerfs-api/tests/store_family.rs` + `compact_of_a_remote_sqlite_store_is_a_documented_local_only_refusal` pins + it. + +## G. Deferred design decision (1) + +G1–G6 were the retired implementation-note entries (their contract-carrying +decisions are folded into A/D above). G7 continues the numbering as a +**deferred design decision**: it recommends no contract text change and no +code change, and it records the measurements that settle the question. + +### G7. §7/§10: the install-only lane stays deferred (#42) + +- **Frozen:** §7 step 1 takes the persistence writer guard before the + checkpoint and holds it through step 7's rename; §10 repeats that compaction + holds `lane::writer_admission` from the seal through the rename, and install + requires the sealed `(size, mtime, db-header change counter)` and the catalog + tuple unchanged, otherwise `Conflict` with no writes. §8's Phase M records a + catalog tuple and Phase B re-checks it for equality, so PostgreSQL has the + same fence. +- **Proposed and measured (not adopted):** mark on the build's read snapshot, + release the lane, let writers commit during the build, and take the lane + again only for install. `tools/bench/installlane/` implements the contract + that would need: an open high-water mark, an incremental re-mark of roots + committed after it, a delta copy of newly-live packs/objects/metadata, a + bounded WAL, and a replacement for the sealed-tuple equality fence. It is a + store-shaped model (10 GiB / 10,240 packs, 25% dead, content-addressed + dedupe references onto skipped packs), not product code; the 50 GB arm was + not run. Raw numbers and the method are in + `tools/bench/installlane/RESULTS.md`. +- **Measured (10 GiB model, fresh-copy build):** build 18.96 s for 8.04 GB. + With the build snapshot open, 6,348 commits wrote 6.35 GiB (348 MiB/s) and + the WAL reached **7.36 GB**. The re-mark scan took 0.88 s; the closure delta + was 6,380 packs / 6.69 GB (6,348 new + 32 skipped packs reached by dedupe) + and copied in 40.78 s (164 MiB/s). The checkpoint took **38.43 s for + 7.38 GB (183 MiB/s)**; verify 0.57 s. The install-only fence (checkpoint + + delta + verify) is **80.65 s — 4.25× the fastest measured build** and about + 2.5× the 32.5 s premise. Checkpoint throughput across the small probes was + 156–183 MiB/s against a 176–420 MiB/s writer rate, and + `wal_checkpoint(TRUNCATE)` returned `busy` on every call while writers + committed (0/430 and 0/3,105 succeeded); the backlog grew +366 MiB/s. + The prototype ran on the sandbox machine filesystem (Python 3.11.2, + SQLite 3.40.1); these rates are host-specific and do not replace the R730 + evidence basis below. +- **Why deferred, three independent reasons:** + 1. **No bounded WAL.** The backlog cannot drain while writers commit + (checkpoint slower than writers; `TRUNCATE` always `busy`). A cap with no + checkpointer is a hard pause: at a 256 MiB cap the writers committed + 256 MiB and then stalled 17.33 s of a 10 s build. That is the frozen + whole-build exclusion again, with a WAL on disk. + 2. **No bounded delta.** Content-addressed dedupe lets a post-mark commit + reference a pre-mark object whose pack the mark skipped. A "new packs + only" delta lost 8 reachable objects in a 64 MiB probe; the closure + re-mark recovered them but copied the skipped pack too. A post-mark + commit re-referencing the whole skipped set produced a delta of **1.00× + the dead set with zero new bytes written**. The delta is bounded by what + the rebuild exists to free, not by post-mark writes. + 3. **No shorter outage.** The fence is longer than the build it replaces and + still needs a final quiesce: a delta watermark is always behind the newest + commit (verifying at w2 missed 1,984 objects; the catch-up after writers + stopped verified clean). The frozen tuple fence also always fails once + writers continue (mark tuple `(7665, 10241, 81920, 10.74 GB)` vs current + `(14013, 16589, 81920, 17.39 GB)`). +- **Correctness is reachable, but not cheaply:** the closure re-mark verified + (112,353/112,353 objects, `missing=0`, `extra=0`), so the mechanism works. + What it needs is a new state machine: the open mark, its high-water tuple, + the delta watermark and the delta set must be durable and crash-resumable, + and writers must be quiesced for the final catch-up + checkpoint + swap. +- **Exact contract changes it would require** (none made here): + 1. §7 step 1 / §10: seal, then *release* the writer lane; keep the sealed + file uncheckpointable (the `immutable=1` attachment already guarantees + it) and admit writers for the build; re-acquire for install. + 2. §6.1 / §7 / §8 Phase M: record an **open** mark tuple + `(catalog_incarnation, live_heads_digest, next_history_serial, + next_pack_id, next_ordinal, store_totals canonical/stored/metadata/ + logical)`; a later publication no longer makes the run `Conflict`. + 3. §7 steps 4/6/7 and §8 Phase B: replace the tuple-equality fence with an + incremental re-mark from the tuple, a delta copy (a pack reached only by + a base or a dedupe reference is copied whole), a re-verify over the + post-delta frame, then a final quiesce, catch-up delta and swap. + 4. §9 / §10: add the WAL cap/backpressure contract (what a commit above the + cap does, the maximum backlog, the latency/refusal shape) and the final + catch-up fence; the SQLite seal check becomes a delta-watermark check. + 5. §4.1 / §9: persist the open mark tuple, the delta watermark and the delta + set in `gc_op`/the sidecar so a crash during the delta is adoptable, and + add that crash/recovery row to §9's matrix. + 6. §8 PostgreSQL: Phase M/B's equality re-check must accept the same delta + path (or the entry must say PostgreSQL is out of scope); PG's Phase C + already allows commits after the swap, but not during Phase M→B. + 7. §11: report `remarkSeconds`, `deltaPacks`, `deltaBytes`, + `walBytesAtInstall` and `backpressureSeconds`, and make `peakTempBytes` + path-dependent (target + WAL/delta temp + visited) instead of the + `VACUUM INTO` formula. + 8. §5 / S5: state the dedupe rule — a post-mark commit may reference a + pre-mark object whose pack the mark skipped, so "newly written packs" is + not a delta bound; the bound is the dead set. +- **Decision:** defer install-only for 2.0.0 and keep the frozen whole-lane + exclusion. The prototype and its measurements stay under + `tools/bench/installlane/`; no product code, schema or spec text changes. + +--- + +## Evidence basis used by D2 and the runbook + +Measured on the host class **Dell PowerEdge R730, PERC H730 Mini controller, +rotational (HDD) backing — not NVMe** (88 vCPU across two Xeon E5-2699C v4, +125–135 GB visible RAM, ext4, 4.0 TB free). The rotational device explains the +~33 MB/s build copy rate; do not read the 50 GB rates onto an NVMe host. + +| Step (GB = 10^9 B) | 10 GB arm [M] | 50 GB arm [M] | +|---|---|---| +| Mark (`none`) | 14.9 s/stored GB, 16.4 s/live GB | 35.9 s/live GB (32.7 s/stored GB) | +| Mark (`rebuild`) | 16.4 s/live GB | 17.2 s/live GB | +| Build (`VACUUM INTO` + prune + `VACUUM`) | 10.3 s/new GB | 93.0 s/new GB (moves ~141 GB to keep 45.8 GB, ~33 MB/s) | +| Verify | 0.034 s/new GB | 1.87 s/new GB | +| Install | ~1.9 s fixed | 7.311 s | + +- **10 GB is the medium arm** every agent can qualify: three runs per path + (9.987 GB pack bytes; `none` median 149.318 s, `rebuild` median 246.371 s; + one `Durable` rebuild 284.078 s as a control). The whole-Store build's + medians are in + `engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_whole_store.json` + (a transcribed summary of the T-G61 run: the raw per-run samples were not + retained); the fresh-copy re-measurement on the same fixture is + `.../sqlite_compact_capacity_measure.json` (`rebuild` median 118.439584 s, + `none` 88.439 s). That receipt predates #49 round 4: its build path has no + 256 MiB `src.mmap_size` source window and its verify still runs the + `quick_check` page scan, so it is the pre-head8 build/verify path, not a + head8 re-measurement. +- **50 GB is the owner-final arm** (#42 item 44): fixtures 20–50 GB, run once + at the final identity, one run per path (49.999 GB, 384,850 packs, 8.876% + dead; `none` 1,633.585 s, `rebuild` 5,131.835 s); receipt + `engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json`. + It is not part of a normal PR run. +- The 50 GB numbers do **not** scale linearly from the 10 GB ones (the 10 GB + Store and its artifacts fitted the page cache; the 50 GB one did not), so a + capacity claim states per-GB rates with its arm and host. +- **Fixture basis (2026-11-16, #711 review).** Every number above was measured + on the earlier whole-file-only fixture: one whole-file object per pack and one + genesis Layer per project. The harness now models real Layer history depth, + content shared across projects and delta chains kept only as bases, and it + records the realized fixture shape, the cold-cache method and the 3-run + variance (`tools/ci/gc-bench.md`, `examples/gc_bench/gc_fixture.rs`). These + numbers are evidence for the earlier shape: the gated 50 GB owner-final arm + has **not** been re-run on the current fixture. The 10 GB arm **was** + re-measured on the current shape on 2026-11-16 (`rebuild` median 133.076 s, + `none` 82.683 s, three runs per arm, warm overlayfs cache); its receipt is + `engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_gc_bench.json`. + It is a different generated fixture from the `capacity_measure` 9.987 GB run, + so the two 10 GB numbers are not interchangeable. That receipt too predates + #49 round 4's source mmap window and `quick_check` removal, so its build and + verify phases are the pre-head8 path, not a head8 re-measurement; the ~4 s + `quick_check` cost is derived, not measured. Treat both 10 GB receipts as + pre-head8 until a head8 re-run replaces them. + +## Request + +Please record A1–A11 as #42 items (or amend the frozen text) and note the A12 +reading (it changes no contract text), and accept or reject D1–D5 individually +when the owner signs off, and accept G7 as the record of a deferred design (it +changes no contract text). Until then the implementation follows the frozen +text except where A8–A12 and D1–D5 are named, and the spec file in this branch +is byte-identical to the frozen version. diff --git a/core/docs/issues/project-41/gc-runbook.md b/core/docs/issues/project-41/gc-runbook.md new file mode 100644 index 0000000000..a6d09e0f37 --- /dev/null +++ b/core/docs/issues/project-41/gc-runbook.md @@ -0,0 +1,285 @@ +# GC operator runbook: `store.compact` and the three removals + +> Operational page for the frozen `LayerFS-GC-spec.md` (the spec wins). Baseline +> `a6d84131b`; calls land in T3–T11. Admin-profile only, refused under a folder +> scope (§10). No confirm/force/dry-run; a lost reply stays uncertain, nothing retries itself (§1, §9). + +## 1. Triage: stuck compact or a phase in `store.status` + +`layerfs store status ` shows `holders`, `state` (`busy`/`unresolved`) and +`op = {id, name, phase, started, uncertain}` (§11). Phases — SQLite `mark`, `build`, +`verify`, `install`, `cleanup` (the settle after install); PG `mark`, `build`, +`swap`, `cleanup`. The SQLite `cleanup` phase is a recorded deviation (D4): the +settle writes it while it removes the artifact set. PostgreSQL writes `cleanup` in +its own small transaction before the first Phase C batch, so a crash during the +batched deletes reads `cleanup` and the next call resumes the persisted set. + +- **SQLite.** The op/phase is the *home op* (`store//record.json`, field `op`; + a remote home is read via `step_on`); the Store DB has the admission row: + `sqlite3 -readonly .sqlite "SELECT kind, scope_kind, state, uncertain, phase, generation, artifact FROM gc_intent WHERE state <> 'done';"` + Artifacts sit beside the Store file: `.compact{,-journal,-wal,-shm,-visited,-visited-journal,-mj*}`. + A direct reader may see `busy`/`locked` in the exclusive window; don't loop. Never unlink `-wal`/`-shm` (§7 step 1). +- **PostgreSQL.** `store.status` reads the Store-side `gc_op` (§11). On a direct or session-pooled `psql`: + + ```sql + SELECT generation, phase, scratch_schema, artifact, pid, backend_start, usename, incarnation FROM gc_op; + SELECT (SELECT count(*) FROM gc_dead), (SELECT catalog_incarnation FROM history_meta); + SELECT kind, scope_kind, scope_id, state, uncertain, phase, artifact FROM gc_intent WHERE state <> 'done'; + SELECT nspname FROM pg_namespace WHERE nspname LIKE 'lfs_gc_%'; + ``` + + `gc_op` is the one compaction row; `gc_dead` is its persisted dead set (membership, + not a predicate); `gc_intent` is admission. `lfs_gc_` is recorded in `gc_op` + **before** `CREATE SCHEMA` (§8). Never INSERT/UPDATE/DELETE `gc_intent`, `gc_dead`, + marker tables or `project_epoch` by hand. + +**Free space.** A SQLite `rebuild` holds the old Store plus about one new +Store (the retained bytes) and the visited scratch. Check the free space on the +Store's filesystem and its temp directory before every rebuild: see §8. + +## 2. `Unresolved`, live vs dead owner, lost reply + +`Unresolved` = a provably dead owner left an **uncertain** op; a non-owning call observes +it, the owning/next call settles it (§5.1, §10). A live owner is `Busy` — wait, don't kill it. + +1. Liveness comes from the recorded owner, not the clock: SQLite home op + `owner.pid`/`pid_start`/`pid_boot`; PG `gc_op.pid` + `backend_start` + `usename` against `pg_stat_activity`. +2. Dead owner, uncertain op: run `layerfs store compact ` again; it adopts the dead + op with the artifact/high-water in the record (§9). `sandbox.close`/`machine.remove`/ + `store.remove` clear their own ops; `logs.reset` cannot (§10). A removed machine record alone is no proof (§5.1). +3. **PG lost `COMMIT` reply** — the next `store.compact` resolves it automatically on a + *second* connection **before** it takes the exclusive key (§8). The mark recorded + `pid`, `backend_start`, `usename` and the incarnation in `gc_op` (`usename` is a + `gc_op` column, §4.1). A recorded owner that no longer holds the Store's exclusive + key is a lost reply: the call terminates exactly that backend; a recorded owner that + still holds the key is a live run and stays `Busy`, never terminated. The operator + can run the same resolution by hand: + + ```sql + SELECT pg_terminate_backend(a.pid, 30000) + FROM pg_stat_activity a + WHERE a.pid = AND a.backend_start = '' + AND a.usename = '' AND a.datname = current_database() + AND a.pid <> pg_backend_pid(); + ``` + + Check the boolean, poll until the `(pid, backend_start, usename)` triple is gone, then read + `history_meta.catalog_incarnation`: `G` = not committed, `G+1` = installed. Needs PG 14+ + and `pg_signal_backend`/membership, else `Unsupported` (§8); the automatic recovery + reports the same refusal. Never terminate a live key holder to clear `Busy`. + +## 3. Safe cleanup + +Default: **let the next `store.compact` judge it** (§9). The op names `G` and the artifact. +Store at `G` = not installed → next compact removes the named artifact set; `G+1` = +installed → it records the outcome. A dead owner's op is adopted; only a live owner blocks. + +- Delete by hand only when **no record names it**: no compaction `op`/`gc_op` row (PG: no + `gc_op.scratch_schema`/`artifact`), no live owner, no exclusive key. A stray at the staged + name with no record is refused `Exists` and not deleted by the code (§9); after the checks it is safe, otherwise leave it. +- Never remove `.compact-visited*` during a compact: it is the live visited set and part of `peakTempBytes` (§6.3, §11). +- **Never** delete a live Store's `gc_intent`/`gc_dead` by hand: admission and the Phase C + dead set are cleared only by the owning call on settle (§5.1, §8 Phase C). +- PG: drop `lfs_gc_` only when `gc_op` names no schema and no session holds the exclusive key; a named schema is the next compact's to remove. +- A definite pre-rename SQLite failure deletes its own artifact set; a rename/fsync failure is + **installed-but-uncertain** — keep the new Store, settle `unknown`, report it, never delete it (§7 step 8). + +## 4. Rollback and reversal (limits) + +- **Schema rollback is additive.** Marker/position/admission tables and the + `history_serial`/`trace_salt` columns are added at open; a downgrade leaves them (§4.1, + decision 6). No down migration; dropping them is not a rollback. The PG open check records + an exact table set (§4.1), so a pre-GC PG binary may refuse the added tables — forward-fix or restore instead. +- **Markers cannot be undone by dropping marker tables.** Dropping `branch_retired`/ + `layer_stack_retired`/`path_removed` resurrects retired rows without home entries, breaks + the "name never reused while rows exist" `BranchId::named` rule (§4.4), and cannot restore rows a rebuild already collected. +- **Supported reversal is re-creation after the frontier, never deleting markers** (§4.3): + a file is written/created at serial > `frontier_serial`; a Branch is created/forked once a + rebuild frees the name (new id/history); a project is `project.init`-ed once its retired stack row is collected (new empty project). +- **Limits.** Old ids/history/bytes never return; a Branch pinned by a retained Layer or + `commit.branch_id` keeps `NameTaken` (§4.4); `remove_file` reversal does not unhide + at-or-below versions; `rebuild` install is irreversible (§9). Content recovery is + `backup.create`/`backup.restore` from before compaction (SQLite; the server's backup for PG); + no undo copy is kept (§9, #155 D-24). The 2026-11-15 branch cutover keeps its + pre-change home and Store in `/home/dell/lfs-work/backup-20261115-180517` + (earlier `backup-20261115-180149`); restore that backup rather than editing + markers (see §7). +- `project.remove` is gone; an old client gets `InvalidInput` (§4, §10). + +## 5. Known gaps at this baseline + +- **Pre-GC Store upgrade (current, both backends):** a Store written before the GC + tables, the `kept_node` closure or the position columns gains them at its first + writable open — SQLite runs `add_gc`/`add_kept_node`/`add_position_state` (plus the + definition gate), PostgreSQL runs `add_gc`/`add_kept_node`/`add_position_state`/ + `add_position_columns`. Open and `store.compact` therefore accept a pre-GC Store + instead of refusing it; a schema whose recorded source/definition matches no known + earlier step is still refused and is never migrated by hand (§4.1). + *Historical:* the earlier notes here (T1.3b "PG existing-Store additive", T1.2b + "old-Store column ALTER") described the baseline `a6d84131b`, before those open + steps landed. +- **`remove_file` is visibility-only in 2.0:** it hides reads/listings/mounts/merges/publishes + and frees no bytes; objects reachable only via the path are retained (§6.2, decisions 16/35). +- **No path-level physical reclaim:** path-aware walk and per-entry page pruning are follow-ups; `keptWaste` has no page component (§6.2). +- 1.0 Stores convert offline (§4.1); no online migration. +- **Four SQLite deviations from the frozen text remain** (D1, D3, D4, D5), + plus the settled D2 build decision; all are recorded in + [`gc-42-decisions.md`](gc-42-decisions.md) and the frozen spec text is + unchanged: the fresh bulk load runs with `foreign_keys=OFF`; the rebuild is + the fresh-copy build (no whole-Store copy, no `VACUUM`, peak old + new + + visited, §8 — it follows S5's fresh-Store sentence but deviates from §7 step + 4's payload sweep and rollback journal, D2); `none` writes only a transient + visited scratch and rolls its mark transaction back; the settle records a + `cleanup` phase; `store.compact` is local-only, and a Store pushed to another + machine is compacted there or pushed back. + +## 6. First 10 minutes + +0. Check free space first (§8): the old Store plus about one new Store and + its visited scratch — roughly the Store's size again (with margin). A full + disk is `Capacity`. +1. `layerfs store status `: read `state`, `holders`, `op.phase`, `op.uncertain`. Don't kill a live owner; `Busy` is expected. +2. `Unresolved`: prove the owner dead (home pid/start, or PG `pid`+`backend_start`), then + re-run `layerfs store compact ` to adopt. For a lost PG swap reply, use §2 on a second connection. +3. Before manual steps, back up (`backup.create` on SQLite; server backup on PG) and record generation `G`, op id, artifact name. +4. Compare artifact/schema with the op; leave anything a record names to the next compact (`G` removes, `G+1` records). +5. Never hand-edit `gc_intent`, `gc_dead`, markers or `project_epoch`; never unlink `-wal`/`-shm`; never drop a schema `gc_op` names. +6. To re-use a name after a removal, re-create after the frontier (or after a rebuild frees it); don't delete markers. Escalate if old content must be restored. + +## 7. Branch retirement (`branch.remove`, live since 2026-11-15) + +`branch.remove` is live on the 2.0.0 build `4e65ebbae` (host CLI `1dc15838…`, +bundle `layerfs` `c4737445…`, daemon `b8dd0766…`, helper `3d033ec0…`); +`layerfs help branch` lists it, and a probe on a missing name returns +`NoSuchBranch` (exit 125). The call is a marker write, not a delete: it hides +the Branch, keeps its rows and its name until a later `store.compact` rebuild +collects the marker, and never touches `main` (`ReservedName`). + +**Preflight.** + +1. `layerfs sandbox list ` is empty, or every listed Sandbox belongs + to a task you have agreed to exclude; a mounted Branch is `MountsActive`. +2. `layerfs doctor` is `{"ok":true,"findings":[]}`; `machine status docker` is + `running`. +3. Take a backup; the cutover used + `/home/dell/lfs-work/backup-20261115-180517` (bundle, host, daemon, helper, + `bundle.json`, `worker.json` and the Store volume; the earlier one is + `backup-20261115-180149`). Keep it until the retired names are no longer + needed. +4. Page the inventory: `layerfs branch list --page-size 100 + --page-pos `; the last page has `has_more: false`. Record the + names and the count before and after. +5. Build the retire list: an empty Branch (`head: null`), or an explicitly + reviewed superseded Branch (typically a `-merged`/`-integ`/`-final`/ + `-publish` variant) whose content is already on `main` or no longer needed; + `layerfs branch diff main --paths-only` shows the paths. + Never include `main`, a running task's Branch, or a Branch with a live + mount. + +**Procedure.** + +- For each name: `layerfs branch remove `. One call per + Branch; there is no batch, force or dry-run form. `Busy` is another + operation holding the Store scope (wait, call once more); `MountsActive` is + a live mount; `NoSuchBranch` means already gone; `ReservedName` is `main`. + Record every refusal. +- Verify: the retired name reads `NoSuchBranch`, a re-fork is `NameTaken` with + the holder's `state: "retired"`, `branch list` no longer shows it, and the + `main` head is unchanged. +- **Soft name retention.** The name is not reusable until a rebuild collects + the `branch_retired` rows; plan around that and never hand-delete markers + (§4). `store.compact` below is the only supported reclaim. + +**Reclaim (optional, later).** `layerfs store compact ` only when the +Store is idle (no mounts, daemon or other op). A rebuild copies retained rows +into a fresh Store and installs `G + 1`; only then are retired names and their +rows collected. A `none` outcome (<10 % wholly dead) keeps everything, and the +retired names stay taken. Follow the compaction procedure in §6 and the +rollback limits in §4. + +**Rollback.** Restore the whole home and Store from the pre-cutover backup +above (or `backup.restore` from a `backup.create` taken during the window); +markers cannot be undone by dropping tables (§4). The 2026-11-15 cutover was +not rolled back: no call failed, and `main` and `Cargo.toml` were byte-identical +before and after. + +**2026-11-15 result.** 145 Branches retired (20 empty at cutover + 125 +superseded), 0 refusals; 162 remained by choice (unknown or non-empty, +deliberately untouched). `main` stayed `32977da5…` (root `56dbe645…`), +`layerfs doctor` stayed clean, and no Sandbox/Workspace of the cleanup was +left running. The new build's one-way GC schema migration grew the sqlite +Store 1,111,203,840 -> 1,111,310,336 B on first open. + + +## 8. Disk headroom and peak temp for a SQLite rebuild (S5, D2) + +The rebuild writes a fresh Store with only the retained rows +(S5; decision D2 in [`gc-42-decisions.md`](gc-42-decisions.md)). Peak temp is +the new file plus the visited scratch; there is no whole-Store staged copy, no +rollback journal and no `VACUUM` rewrite (the frozen §7 cost sentence includes +a rollback journal/WAL; D2 records why the fresh path has none). Peak disk is +therefore + + old Store + (retained bytes + visited scratch) + margin + +The retained bytes are at most the live bytes, so the run needs roughly the +Store's size again, never a multiple of it. The validated prototype measured +1.007× live written and a 1.003× `(old + live)` peak on a 10 GiB fixture (the +earlier `VACUUM INTO` build wrote 4.03× and peaked at 1.577×). Reserve free +space before a rebuild on both the Store's filesystem and its temp directory: + +| Store / dead share | Reserve | +|---|---| +| any `rebuild` | old Store + about one live Store (the retained bytes) + a margin | +| any `none` run | a few GB (the visited scratch only) | +| 50 GB Store, ~10% dead | about the Store's size again [E] | + +Per-GB rates on the measured host class (**Dell PowerEdge R730, PERC H730 Mini +controller, rotational HDD — not NVMe**; 88 vCPU, 125–135 GB visible RAM, +ext4): the historical `VACUUM INTO` arm measured mark 17.2 s/live GB, build +93.0 s/new GB at 50 GB and 10.3 s/new GB at 10 GB, verify 1.87 s/new GB, +install 7.311 s. The fresh-copy prototype's 10 GiB run measured 32.5 s against +120.8 s for that build; the 10 GB arm is the agent budget, the 50 GB arm is the +owner-final measurement (#42 item 44). Its receipt is +`engine/crates/layerfs-persistence/tests/fixtures/sqlite_compact_capacity_measure_50gb.json` +(`none` 1,633.585 s, `rebuild` 5,131.835 s, one run per path on the superseded +build), and the whole-Store 10 GB medians are kept in +`.../sqlite_compact_capacity_measure_whole_store.json` (a transcribed summary: +the raw per-run samples were not retained). The alternative build (clone + +delete + `incremental_vacuum`, below a size threshold on reflink filesystems) is +recorded in D2 and is not adopted. + +The per-GB rates above were measured on the earlier whole-file-only fixture +(one whole-file object per pack, one genesis Layer per project). The current +harness (`tools/ci/gc-bench.md`) models Layer history depth, cross-project +shared content and delta chains and records the cold-cache method and 3-run +variance; re-measure the arm before quoting a rate against the current +fixture. The gated 50 GB owner-final arm has not been re-run on it. + +## 9. Implementation notes: deviations and the D2 build decision (D1–D5) + +Four frozen sentences still differ (D1, D3, D4, D5); D2 is the settled build +decision, whose fresh-copy rebuild follows S5's fresh-Store sentence but +deviates from §7 step 4's payload sweep and rollback journal and from §11's +peak-temp line. All five are recorded in +[`gc-42-decisions.md`](gc-42-decisions.md), which is the only place they are +written: the frozen spec file is unchanged. None changes the artifact set, the +op record or the next-call adoption. + +| Entry | Frozen sentence | Implemented | +|---|---|---| +| D1 (§7 step 4) | `PRAGMA foreign_keys=ON` outside the transaction (frozen text, unchanged) | the fresh bulk load runs `foreign_keys=OFF`; the step 6 `foreign_key_check` on the `immutable=1` open is the guard | +| D2 (S5/§3/§7/§11) | "copies only retained rows into a fresh Store"; §7 step 4's payload sweep and rollback-journal transaction; peak old + new + journal/WAL + visited | fresh-copy build (pack-as-unit, no payload sweep, no rollback journal; indexes and triggers after the load); peak old + new + visited (§8) — a recorded deviation from §7/§11, not an edit to the frozen text | +| D3 (§3/§7 step 1.5) | `none` writes nothing; delete the scratch and stop | `none` spills the closure to `.compact-visited`, counts it in `peakTempBytes` and deletes it before replying; the mark and its intent are one transaction that is rolled back, so no `kept_node`/`gc_intent` row and no WAL frame survives; no residue survives | +| D4 (§11) | SQLite phases `mark`, `build`, `verify`, `install` | the settle also writes `cleanup` (§1) | +| D5 (S6/§7) | "A SQLite Store compacts on the machine where it lives"; §7 "Runs as one operation on the Store's machine (S6)" | `store.compact` is local-only: its `Worker` branch is refused `Unsupported` before the Store is opened; a pushed sqlite Store is compacted after `store.push` back to `native` | + +A10 is an amendment rather than a deviation: it records that +`deadBytesRefined` equals `deadBytesBound` on every path, because the mark +leaves no second payload sweep to refine it. A report never names a different +refined figure on either backend. + +Operators do not need to act on D1–D5 during an incident: the artifact set, the +op record and the next-call adoption are unchanged. They matter when reading +the phase or the reported peak, and to a client that compares the frozen text +with the shipped report. diff --git a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md new file mode 100644 index 0000000000..34f634fa6a --- /dev/null +++ b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md @@ -0,0 +1,292 @@ +# T12.3a: SQLite acceptance-matrix runner report + +> Generated by `cargo test -p layerfs-persistence --test sqlite_compact_matrix`. +> The frozen [`LayerFS-GC-spec.md`](LayerFS-GC-spec.md) section 12 and +> AC-01..AC-60 dispositions (layerfs-project#52) win; this page records +> evidence and adds no contract. Regenerate with +> `UPDATE_REPORT=1 cargo test -p layerfs-persistence --test sqlite_compact_matrix`. + +## Scope and counts + +- Rows: 60 (AC-01..AC-60); backend split: both 47, SQLite-only 6, PG-only 7. +- Executed here: 8; covered by a named existing test/fixture: 30; partial: 16; deferred to T12.3b: 6; SQLite gaps: 0. +- **T-G28/T-G60 follow-up.** T-G28 later added the real-process SIGKILL arms + and T-G60 extended them to mark, copy, verify and rename, so + `sqlite_compact_crash` now has 23 crash tests (9 real-process SIGKILL arms and the + child placeholder), not the 13-test T12.3c snapshot; the binary also runs one + lifecycle guard (`unfinished_init_or_merge_rows_refuse_compaction`), which + is not a crash arm. Those arms still kill a + re-executed test child, so T-G65 added + `api/crates/layerfs-cli/tests/sqlite_compact_process_kill.rs`: 8 SIGKILL + arms on the built `layerfs store compact` process at mark, build, verify and + install, so AC-01 is `covered`. +- A `covered`/`partial` row names a file and test function that this runner + asserts exists; an `executed` row names an arm this binary really runs. +- A cross-backend row records its SQLite half only; its PostgreSQL half is + deferred to T12.3b by name. Of the 7 PostgreSQL-only rows, 6 + are deferred too and 1 (`AC-44`) is covered here by the `postgresql_compact` + arms listed under Run environment. +- **T-G61 capacity_measure run (fixture A: 9.987 GB pack).** The opt-in + `sqlite_compact_capacity.rs#capacity_measure` (ignored by default) measured a + real **9.987 GB** Store. On the superseded whole-Store `VACUUM INTO` build + (its medians are kept in + `tests/fixtures/sqlite_compact_capacity_measure_whole_store.json`, a + transcribed summary: that run's raw per-run samples were not retained): + `none` at 8.88% dead (three runs, median 149.318 s) and `rebuild` at 10.00% + dead (three runs, median 246.371 s; mark 149.089 s, build 94.317 s, verify + 0.312 s, install 1.916 s; dead 1.011 GB, shrink 1.017 GB; the T-G72 packed + peak bound 29.481 GB, the old `new + visited` formula 9.144 GB), plus a + WAL/FULL control (284.078 s). + Measured on the pre-#49-round-4 fresh-copy build on the same fixture A: the + committed receipt predates both the 256 MiB `src.mmap_size` source window and + the `quick_check` removal, so it is the pre-head8 build/verify path, not a + head8 re-measurement: + `rebuild` median 118.439584 s (mark 74.462-75.704 s, build 18.190-19.209 s, + verify 4.330-4.377 s (including the `quick_check` page scan #49 round 4 removed), + install 19.197-23.673 s; dead 1.011 GB, shrink 1.016-1.018 GB, peak + **9.144 GB** = the fresh Store + visited) and + `none` median 88.439 s (82.0-149.9 s; this Sandbox shares the host with other + runs). The fresh build phase is about 5x shorter (18-19 s against the + 94.317 s superseded-build median, whose per-run samples were not retained, + so it has no recorded range); + the removed `quick_check` page scan cost a derived ~4 s (it is not in §7 + step 6's validate list; #49 round 4 removed it; no scan-free same-fixture + verify is recorded, so the ~4 s is an estimate), and the install pays a + ~19 s `sync_all` for the freshly written 9.1 GB, both inside the 118 s total. + The same-fixture comparison is **246.371 s -> 118.439584 s = 2.08x**; the + fresh-copy medians are in + `tests/fixtures/sqlite_compact_capacity_measure.json` and the superseded + build's in `tests/fixtures/sqlite_compact_capacity_measure_whole_store.json`. Both + 10 GB receipts predate #49 round 4, so every §13 number quoted here is the + pre-head8 build and verify path, not this branch's. +- **gc_bench medium run (fixture B: 9.789 GB live / 10.731 GB pack).** The + separate `examples/gc_bench.rs` medium scale (16 projects x 52 Layers, + 75,524 live objects, 8.8% dead) ran three times per arm on 2026-11-16, before + #49 round 4 changed the build (256 MiB source window) and verify (no + `quick_check`) paths, and + its receipt is committed as + `tests/fixtures/sqlite_compact_capacity_measure_gc_bench.json`: `rebuild` + median **133.076 s** (runs 132.274-134.601 s, 1.75% relative range; mark + 82.050 s, build 23.469 s, verify 6.060 s, install 21.011 s; peak temp + 9.836 GB, shrink 1.093 GB) and `none` median 82.683 s. Fixture B is a + different generated Store from fixture A, so 133.076 s is **not** the other + side of the 246.371 s -> 118.439584 s comparison; the earlier 1.85x mixed the + two fixtures and is withdrawn. Section 13 fields: the creation profile + declares `SqlitePersistenceProfile::Disposable` (journal_mode WAL, + synchronous OFF, page_size 4096) and the caches (warm; `drop_caches` refused + 6/6, overlayfs TMPDIR); bytes written is the per-run `processIo.writeBytes` + median (9,839,857,664 B rebuild, 7,086,080 B none). The SQLite version and + WAL bytes are **not recorded** by the gc-bench/1 receipt (recorded + deviation), and the page_size/synchronous_commit values are declared from + the creation profile rather than sampled with PRAGMA at run time. Its build + (23.469 s) and verify (6.060 s) are the pre-#49-round-4 path, like fixture A's. +- T-G66 measured the 50 GB arm once per path on the old build (#42 item 44): + `none` 1,633.585 s and `rebuild` 5,131.835 s, receipt + `tests/fixtures/sqlite_compact_capacity_measure_50gb.json`; see the runbook + disk-headroom section. The 30% arm remains [E]. + +## Matrix (SQLite half) + +| Case | Backend | Matrix tier | Disposition | Evidence | Note | +|---|---|---|---|---|---| +| AC-01 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_after_verify; sqlite_compact_crash.rs#crash_after_install; sqlite_compact_crash.rs#crash_during_settle; sqlite_compact_crash.rs#sigkill_before_seal; sqlite_compact_crash.rs#sigkill_after_seal; sqlite_compact_crash.rs#sigkill_during_copy; sqlite_compact_crash.rs#sigkill_during_verify; sqlite_compact_crash.rs#sigkill_during_rename; sqlite_compact_crash.rs#sigkill_after_build; sqlite_compact_crash.rs#sigkill_before_install; sqlite_compact_crash.rs#sigkill_after_install; sqlite_compact_crash.rs#sigkill_during_settle; sqlite_compact_process_kill.rs#sigkill_before_seal; sqlite_compact_process_kill.rs#sigkill_after_seal; sqlite_compact_process_kill.rs#sigkill_before_load; sqlite_compact_process_kill.rs#sigkill_after_build; sqlite_compact_process_kill.rs#sigkill_during_verify; sqlite_compact_process_kill.rs#sigkill_before_install; sqlite_compact_process_kill.rs#sigkill_after_install; sqlite_compact_process_kill.rs#sigkill_during_settle | The SQLite section 9 crash matrix through named failpoints; the 9 sigkill_at! arms in sqlite_compact_crash.rs and the CLI test's 8 SIGKILL arms on the built layerfs store compact process at mark, build, verify and install; no daemon serves Store calls. | +| AC-02 | both | E2 | partial | sqlite_compact.rs#a_wholly_dead_pack_is_rebuilt_away; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | The SQLite arm runs through the suite; the converted 1.0 Store arm and the PostgreSQL backend matrix are deferred to T12.3b. | +| AC-03 | both | E2 | covered | sqlite_compact_crash.rs#crash_before_seal; sqlite_compact_crash.rs#crash_after_wal_switch; sqlite_compact_crash.rs#crash_before_install; sqlite_compact_crash.rs#crash_after_install | SQLite mark/build/verify/install and WAL-switch stop points; lost COMMIT and power-loss arms belong to the PostgreSQL/fsync halves. | +| AC-04 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Deterministic closure fixture plus a seeded six-history reachability property arm; the full random fork/commit/merge/publish/removal generator is not built here. | +| AC-05 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; this runner #arm_random_store_shapes | Full doctor (including the diagnostic scan) on the compacted Store, also in the property arm. | +| AC-06 | both | E2 | executed | this runner #arm_digest_row_after_collection | A Save's `file_digest` row named a root the rebuild collected; the row goes with its root and recording the same content again succeeds. The PostgreSQL half is T12.3b. | +| AC-07 | both | E2 | executed | this runner #arm_mount_and_compaction_exclude_each_other; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | SQLite half executed: bypass gives `StaleStore`, a mount blocks compaction (`MountsActive`, naming the holder) and a live compaction refuses a mount and a second compaction (`BusyOp`). The open one-shot SDK transaction arm is PostgreSQL, T12.3b. | +| AC-08 | both | E2 | covered | admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | One winner; the loser sees MountsActive, and a removed Branch is Missing. | +| AC-09 | both | E2 | covered | branch_removal.rs#a_removed_branch_is_hidden_and_the_epoch_bumps_once; branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#branch_remove_refuses_reserved_and_absent_branches | Reserved main, absent Branch and holder details are asserted; project-side name collisions are in AC-30. | +| AC-10 | both | E2 | covered | path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Removed/UpToDate codes and the frontier; 2.0 frees no bytes for the removed path. | +| AC-11 | both | E2 | partial | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_fast_forward_merge_creates_no_tree_and_the_path_stays_hidden; sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount | Non-FF whiteouts, the fast-forward control, a post-frontier re-creation and publish-at-or-below with a layer-headed mount whiteout execute. A merge that cannot materialize the whiteout has no implementation branch to refuse (`removal::derive_root` always derives), so that clause is deferred by name, not claimed. | +| AC-12 | both | E2 | covered | sdk_path_whiteouts.rs#a_merge_of_two_hidden_sides_keeps_the_path_absent; sdk_path_whiteouts.rs#a_merge_keeps_a_post_frontier_re_creation; sdk_path_merge_removal.rs#a_recreated_removed_path_still_conflicts_as_an_ordinary_edit | A removed path in a merge is omitted with no conflict; a post-frontier re-creation is kept, and only an ordinary edit conflict on it is reported. | +| AC-13 | both | E2 | covered | sdk_path_merge_removal.rs#an_unmerged_branch_created_before_the_frontier_hides_but_does_not_inherit | The earlier Commit on an unmerged Branch is hidden by the marker; a Commit after the frontier does not inherit the path. | +| AC-14 | both | E2 | covered | sdk_path_merge_removal.rs#publishing_an_at_or_below_commit_yields_a_hidden_layer_and_a_layer_headed_mount; sdk_path_whiteouts.rs#a_marker_under_a_live_mount_is_refused_and_materializes_after_it_closes | A Branch whose head is empty and whose base Layer is at or below the frontier whiteouts the path at mount; its next Commit records the deletion. | +| AC-15 | both | E2 | partial | kept_objects.rs#a_retained_tree_and_inode_are_kept_with_their_packs; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retention of the only-reaching-path object is proven; marker hiding in reads/listings and derived whiteouts are not in one named test. | +| AC-16 | both | E2 | covered | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | T12.1 pins a retained retired Branch name and a collected unreferenced one; the branch.remove-then-rebuild arm is exercised through those markers. | +| AC-17 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#a_delta_base_is_enqueued_and_reported | Depth-2 chain copied, pack ids preserved. | +| AC-18 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Collected digest row gone, retained one kept. | +| AC-19 | both | E2 | partial | gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers; path_removal.rs#marker_frontier_and_up_to_date_follow_the_store_position | Path-grammar refusals and the marker frontier are named; the full form table (symlink/hardlink, >4096 bytes, >256 components) is not one test. | +| AC-20 | both | E2 | partial | path_removal.rs#a_raw_marker_frontier_also_yields_up_to_date; path_removal.rs#an_unknown_project_is_missing | UpToDate and never-existing paths are named; the re-creation and other-project arms are partial. | +| AC-21 | both | E2 | partial | epoch_fence.rs#an_epoch_change_during_a_commit_is_detected_and_rolled_back; epoch_fence.rs#a_concurrent_commit_and_removal_have_one_winner | One-shot writes racing removals roll back as Conflict; the merge-racing-remove_file arm is not named. | +| AC-22 | both | E3 | covered | sdk_path_merge_removal.rs#a_merge_and_a_removal_started_together_never_leave_the_path_visible; epoch_fence.rs#an_epoch_change_during_a_merge_is_detected_and_rolled_back | A deterministic barrier starts a merge and a `remove_file` together for three rounds: the loser sees the marker or gets `Conflict` and retries with the whiteout, and a tree derived afterwards never shows the path. The PostgreSQL half is T12.3b. | +| AC-23 | both | E2 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Deleted-project rows and markers are collected by the rebuild; a dedicated path_removed cleanup arm is not named. | +| AC-24 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order; gc_backfill.rs#existing_serials_are_kept_and_new_ones_start_above_the_maximum | Serials backfilled, next_history_serial never falls. | +| AC-25 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | Retained branch_retired/layer_stack_retired markers survive; collected rows take their markers. | +| AC-26 | both | E2 | covered | project_delete.rs#retry_classifies_retired_live_and_absent; project_delete.rs#deleting_a_project_hides_it_and_marks_once | Hidden, entry present, retry returns the stored deleted_ms. | +| AC-27 | both | E2 | covered | project_delete.rs#paging_skips_a_hidden_project; visibility_guard.rs#every_statement_reading_branch_or_layer_stack_is_filtered_or_allow_listed | Stale home entries stay out of list/lookup. | +| AC-28 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup and its failpoints: T12.3b. | +| AC-29 | both | E2 | covered | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved; admission.rs#a_mount_blocks_all_three_removals_until_released | Owner liveness and Unresolved settlement; machine_stopped and the other-home intent are partial. | +| AC-30 | both | E2 | partial | branch_removal.rs#a_live_holder_is_reported_for_branch_and_project_names; gc_calls.rs#wrong_kinds_and_unknown_options_are_refused_before_any_effect | Holder detail and refusal codes are named; the full exact-message matrix (AC-55) is partial. | +| AC-31 | both | E2 | covered | gc_trace.rs#path_arguments_are_byte_counts_in_the_call_log; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Byte-count path arguments and the keyed trace hash are asserted. | +| AC-32 | both | E2 | partial | admission.rs#a_live_operation_is_busy_and_an_uncertain_owner_is_unresolved | Live/uncertain ownership is named; live-but-unreachable, machine_stopped and another home's intent are partial. | +| AC-33 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_sidecar_at_install | A live reader makes wal_checkpoint(TRUNCATE) Busy with no artifact and no op; a -mj* super-journal at install is Conflict with no writes. | +| AC-34 | both | E2 | partial | sdk_mount_bind_failure.rs#a_bind_failure_cleans_its_intent_and_leaves_no_mounts_active | A committed mount intent whose bind fails at the root check is deleted in the error path and no `MountsActive` remains. The failed-delete sub-arm is not inducible on SQLite: the storage schema guard rejects a trigger on `gc_intent`, and a lock held across the bind refuses the register too; the explicit `Cleanup` report is covered by the seam's code path, not executed here. The PostgreSQL half is T12.3b. | +| AC-35 | PG | E2 | deferred (T12.3b) | - | PostgreSQL advisory-key refcount: T12.3b. | +| AC-36 | both | E2 | covered | project_delete_worker.rs#a_lost_worker_reply_is_unknown_and_keeps_the_sidecar_for_the_retry; project_delete.rs#a_retired_retry_reuses_the_intent_deleted_ms_and_finishes_the_entry; project_delete.rs#an_absent_stack_finishes_a_stale_entry | A lost marker reply is `Unknown` with the sidecar kept; the retry resolves retired, live or absent. The PostgreSQL half is T12.3b. | +| AC-37 | both | E2 | partial | gc_trace.rs#the_redacted_set_covers_every_path_bearing_call; gc_trace.rs#the_trace_key_is_salted_by_the_store_policy_salt | Redaction and the keyed hash are named; salt backfill on an upgraded Store is partial. | +| AC-38 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_during_settle; sqlite_compact.rs#the_whole_artifact_set_is_removed_by_the_next_run | The full SQLite artifact set is removed exactly: the staged file and its -wal/-shm/-journal, the visited scratch and its journal, the incomplete op record and the -mj* super-journals; the PostgreSQL schema/name arms are T12.3b. | +| AC-39 | both | E2 | covered | gc_backfill.rs#commits_layers_and_epochs_are_backfilled_in_deterministic_order | One project_epoch row per stack, gc_op not copied, next_history_serial never lowered. | +| AC-40 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; kept_objects.rs#the_digest_is_stable_and_moves_with_a_head | SQLite live_heads_digest is identical before/after and moves with a head; PostgreSQL agreement is deferred. | +| AC-41 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase M mark transfer: T12.3b. The §11 `markBytes` transfer is measured in the walking mark and recorded as the `compact.mark_bytes` metric (asserted by `postgresql_compact_fixtures.rs`; see AC-52). | +| AC-42 | PG | E2 | deferred (T12.3b) | - | PostgreSQL reserved pack/Phase B: T12.3b. | +| AC-43 | PG | E2 | deferred (T12.3b) | - | PostgreSQL Phase C cleanup resume: T12.3b. | +| AC-44 | PG | E2 | covered | postgresql_compact.rs#a_fully_privileged_role_compacts; postgresql_compact.rs#a_role_without_create_on_the_database_is_refused_as_unsupported; postgresql_compact.rs#a_role_without_create_on_the_store_schema_is_refused_as_unsupported; postgresql_compact.rs#a_role_without_temp_is_refused_as_unsupported; postgresql_compact.rs#a_role_without_pg_read_all_stats_is_refused_as_unsupported; postgresql_compact.rs#a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1; postgresql_compact.rs#a_recorded_live_owner_without_the_store_key_is_terminated; postgresql_compact.rs#a_live_key_holding_owner_is_busy_and_not_terminated; postgresql_compact.rs#a_role_without_pg_signal_backend_is_refused_as_unsupported | PostgreSQL privilege preflight (database/schema CREATE, TEMP, pg_read_all_stats, the fully privileged control, the signal backend), the lost-COMMIT reply resumed by the next compact, an ended owner without the Store key terminated, a live key holder `Busy` and kept, and a role without the signal backend `Unsupported`; T-G38/T-G54. | +| AC-45 | SQLite | E1 | covered | sqlite_compact.rs#group_rows_layout_opens_compacts_and_takes_none; sqlite_compact.rs#group_rows_indexed_layout_opens_compacts_and_takes_none; sqlite_compact.rs#acquisition_layout_opens_compacts_and_clears_working_rows; sqlite_group_rows.rs#explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order | Monolithic/GroupRows/GroupRowsIndexed and the acquisition variants compact; working rows are cleared. | +| AC-46 | SQLite | E2 | executed | this runner #arm_sidecar_at_install; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_conflict; sqlite_compact.rs#a_store_replaced_under_an_open_handle_is_stale_at_the_next_transaction | Sealed-state/sidecar change at install gives Conflict with no writes; a Store replaced between seal and install is also Conflict, because step 7 re-applies the sealed (dev, ino) identity. Only a session that opens after the install, or transacts on the old file, gets StaleStore. The in-flight-transaction exclusion is a documented lane limit, not probed. | +| AC-47 | SQLite | E2 | covered | sqlite_compact_crash.rs#crash_during_copy; sqlite_compact_crash.rs#crash_during_settle; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted; sqlite_compact.rs#the_whole_artifact_set_is_removed_by_the_next_run; sqlite_compact.rs#an_installed_leftover_reports_the_same_rebuild_as_a_completed_run; sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript | A crash leaves the full set (the fixture now creates .compact-wal, .compact-shm, .compact-visited-journal and a .compact-mj* super-journal too), the next compact removes every one and recovers, none leaves no artifact, a stray name is Exists; an installed leftover answers its recorded Rebuild G->G+1 report with the collected totals instead of re-deciding to none (F41-1). | +| AC-48 | both | E2 | covered | sqlite_compact.rs#nine_percent_wholly_dead_bytes_take_the_none_path; sqlite_compact.rs#ten_percent_wholly_dead_bytes_rebuild_at_the_threshold; sqlite_compact.rs#thirty_percent_wholly_dead_bytes_rebuild_above_the_threshold; sqlite_compact.rs#an_empty_store_takes_the_none_path; sqlite_compact.rs#a_capacity_failure_leaves_the_store_unchanged | 9/10/30 percent, empty-store zeroReason, capacity abort; verify-failure and re-refine arms are partial. | +| AC-49 | both | E2 | executed | this runner #arm_mount_during_compaction_is_busy; this runner #arm_commit_during_rebuild; admission.rs#a_concurrent_mount_and_branch_remove_have_one_winner | SQLite half executed: a mount during compact is `BusyOp` with the compaction; compaction returns `MountsActive` when the mount wins; a Commit racing the rebuild gets `StaleStore` or lands, and its retry is never lost. The PG Phase A/Phase C arms are T12.3b. | +| AC-50 | both | E2 | covered | calls_scope.rs#a_scope_refuses_every_call_it_does_not_cover; gc_calls.rs#the_four_gc_calls_take_ids_only | Folder scope refuses the four calls; the admin profile is the only allowance. | +| AC-51 | both | E2 | executed | this runner #arm_status_phases; compact_report_status.rs#status_projects_the_sidecar_phase_and_state; sqlite_compact_crash.rs#crash_before_seal | The op sidecar records mark/build/verify/install (runner); T10 projects it through store.status, and the named control test asserts the phase, state and phases list. | +| AC-52 | both | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json; compact_report_status.rs#the_report_wire_is_the_snake_case_contract_shape | Every section 11 field pinned with units and sampling point (golden) and rendered by the T10 wire-shape test; markBytes is a metric, not a result field, and PostgreSQL records it as `compact.mark_bytes`; `deadBytesRefined` is the bound on every path, because the mark leaves no payload sweep to refine it (#42 A10). | +| AC-53 | both | E2 | partial | project_export.rs#export_omits_a_path_hidden_by_remove_file; gc_calls.rs#project_remove_file_checks_its_path_grammar_then_reports_markers | One hidden-path export case is named; the full read-surface table (read/stat/list/glob/diff/revision/edit/grep/blame/cp/apply) is not executed. | +| AC-54 | both | E2 | partial | restore_install.rs#a_folder_made_unprivate_during_the_download_is_refused_before_the_install; restored_records.rs#a_restored_project_reads_as_a_sandbox_reads_it | Restore install guards and restored records are named; restore before/after a removal and the stale-entry listing arm are partial. The restore-window arm needs a lock-capable filesystem (ext4/xfs/btrfs), so on this Sandbox it reports Unsupported while the other two cases pass. | +| AC-55 | both | E2 | executed | this runner #arm_error_codes; this runner #arm_checkpoint_busy; sqlite_compact.rs#a_stray_staged_name_is_refused_and_not_deleted | Exact SQLite texts asserted: Busy, Conflict("Store sidecar at install"), Exists("unrecorded compaction artifact"), Capacity("disk space"); the full engine/API message matrix stays partial. | +| AC-56 | both | E3 | partial | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_golden.json | The literal SQLite vector is pinned (absent head 33 zero bytes); the shared PostgreSQL literal and the three-run E3 shape are deferred. | +| AC-57 | both | E3 | partial | this runner #arm_measurement; sqlite_compact_measure.json; T-G61 sqlite_compact_capacity.rs#capacity_measure; sqlite_compact_capacity_measure.json; sqlite_compact_capacity_measure_whole_store.json; sqlite_compact_capacity_measure_50gb.json; sqlite_compact_capacity_measure_gc_bench.json | The tiny three-run arm is median 32.1 ms. T-G61 measured a real 9.987 GB Store on the whole-Store `VACUUM INTO` build (3 runs per arm; medians kept in `tests/fixtures/sqlite_compact_capacity_measure_whole_store.json`): `none` at 8.88% dead median 149.318 s and `rebuild` at 10.00% dead median 246.371 s (mark 149.089 s / build 94.317 s / verify 0.312 s / install 1.916 s; the build phase's per-run samples were not retained, so it records a 94.317 s median only and no range -- a recorded missing-range deviation, see `tools/ci/gc-bench.md` section 13), plus a WAL/FULL control at 284.078 s. Measured on the pre-#49-round-4 fresh-copy build on the same 9.987 GB fixture: the committed receipt predates both the 256 MiB `src.mmap_size` source window and the `quick_check` removal, so it is the pre-head8 build/verify path, not a head8 re-measurement. `rebuild` median 118.439584 s (mark 74.462-75.704 s / build 18.190-19.209 s / verify 4.330-4.377 s (including the `quick_check` page scan #49 round 4 removed) / install 19.197-23.673 s), peak 9.144 GB; `none` median 88.439 s (82.0-149.9 s; this Sandbox shares the host); profile, host and toolchain are declared in the fixture, and the warm page cache is declared below. The §11 `markBytes` transfer is recorded per PostgreSQL run as `compact.mark_bytes`. The separate gc_bench medium fixture (9.789 GB live / 10.731 GB pack) measured `rebuild` median 133.076 s in three runs (receipt: `tests/fixtures/sqlite_compact_capacity_measure_gc_bench.json`), a different generated Store from the 9.987 GB capacity_measure fixture; the same-fixture comparison is 246.371 s -> 118.439584 s = 2.08x, and the earlier 1.85x (133.076 s against 246.371 s) mixed the two fixtures and is withdrawn. The gc-bench/1 receipt declares the Disposable profile (WAL/OFF, page_size 4096) and the cache, records bytes written as `processIo.writeBytes`, and does not record the SQLite version or WAL bytes (recorded deviation). T-G66 later measured the 50 GB arm once per path (#42 item 44; receipt `tests/fixtures/sqlite_compact_capacity_measure_50gb.json`, `none` 1,633.585 s and `rebuild` 5,131.835 s); the 30% arm, each removal type, the plain `VACUUM INTO` comparison and PostgreSQL timings remain deferred/[E]. | +| AC-58 | SQLite | E2 | covered | sqlite_compact_fixtures.rs#the_sqlite_compaction_receipts_match_the_golden_transcript; sqlite_compact_crash.rs#crash_before_copy; sqlite_compact_crash.rs#crash_during_copy | none rolls its mark transaction back (no kept_node row, no gc_intent row, no committed WAL frame), deletes the scratch and leaves no .compact; rebuild commits the mark and keeps the visited scratch through the build; step 2 writes only on rebuild. | +| AC-59 | SQLite | E2 | executed | this runner #arm_checkpoint_busy; this runner #arm_seal; sqlite_compact.rs#a_store_replaced_between_seal_and_install_is_conflict | busy != 0 refuses Busy and the WAL is never unlinked; a successful seal records (size, mtime, counter) as fenced by the Conflict arm. | +| AC-60 | PG | E2 | deferred (T12.3b) | - | PostgreSQL synchronous standby handling: T12.3b. | + +## Executed in this runner + +- AC-33 / AC-59: `arm_checkpoint_busy` - a live reader makes + `wal_checkpoint(TRUNCATE)` report busy, the compaction refuses `Busy`, + leaves no `.compact`, `.compact-visited` or `.compact-op`, and the WAL is + never unlinked; the next run rebuilds at `G -> G+1`. +- AC-33 / AC-46: `arm_sidecar_at_install` - a `-mj*` super-journal placed + at install is `Conflict("Store sidecar at install")` with no artifact, + no op and an unchanged generation. +- AC-51: `arm_status_phases` - stop points at mark, build, verify and install + record the same phase in `.compact-op`; every phase is observed. +- AC-55: `arm_error_codes` - exact `Exists("unrecorded compaction artifact")` and + `Capacity("disk space")` texts, with no artifact after the capacity abort. +- AC-59: `arm_seal` - a successful seal leaves no sidecar, reports the main-file + bytes and a SQLite version, and the file reopens. +- AC-04 / AC-05: `arm_random_store_shapes` - a seeded property arm over six + reachability shapes: every live object stays readable, every wholly dead one + is collected, the retained-closure digest is unchanged and the doctor is clean. +- AC-06: `arm_digest_row_after_collection` - a `file_digest` row named a + root the rebuild collected, so the row goes; recording the same content + again succeeds and points at the new root. +- AC-07 / AC-49: `arm_mount_and_compaction_exclude_each_other` - a live mount + makes compaction `MountsActive` naming it, and a live compaction makes a + mount and a second compaction `BusyOp`; the refused runs leave no op. +- AC-49: `arm_mount_during_compaction_is_busy` - the stopped run's live + compaction intent refuses a mount with `BusyOp`, and the next run adopts + the leftover and rebuilds. +- AC-49: `arm_commit_during_rebuild` - a Commit started inside the install + fence is `StaleStore` or lands on `G + 1`; a retry is never lost. +- AC-57: `arm_measurement` - three runs of the same rebuild with median and + range in `engine/crates/layerfs-persistence/tests/fixtures/ + sqlite_compact_measure.json` (refresh with `UPDATE_MEASURE=1`). The separate + T-G61 `sqlite_compact_capacity.rs#capacity_measure` measures a real 9.987 GB + Store (three `none` + three `rebuild` runs; `CAPACITY_DURABLE=1` adds a + WAL/FULL control) into + `sqlite_compact_capacity_measure.json`; it is opt-in because the fixture is + ~10 GB and the ignored test is not part of a normal suite. The gc_bench + medium fixture B arm (three runs) is the committed + `sqlite_compact_capacity_measure_gc_bench.json` receipt, not fixture A. + +## Deferred to T12.3b (PostgreSQL) + +- AC-28 - PostgreSQL Phase C cleanup and its failpoints: T12.3b. +- AC-35 - PostgreSQL advisory-key refcount: T12.3b. +- AC-41 - PostgreSQL Phase M mark transfer: T12.3b. The §11 `markBytes` transfer is measured in the walking mark and recorded as the `compact.mark_bytes` metric (asserted by `postgresql_compact_fixtures.rs`; see AC-52). +- AC-42 - PostgreSQL reserved pack/Phase B: T12.3b. +- AC-43 - PostgreSQL Phase C cleanup resume: T12.3b. +- AC-60 - PostgreSQL synchronous standby handling: T12.3b. +- The PostgreSQL half of every `both` row (digest agreement, artifact + names, Phase M/B/C, standby, lost reply, 50 GB arms). + +## SQLite gaps (not claimed by this runner) + +- none; every SQLite gap this runner can execute is executed or covered. + +## E3 arms + +- AC-04: the T12.1 closure fixture (deterministic) plus the seeded + reachability property arm; the full random history generator is deferred. +- AC-22: a deterministic barrier starts a merge and a `remove_file` together + for three rounds; the loser retries and a derived tree never shows the path. +- AC-56: the SQLite literal vector is pinned by the T12.1 golden; the shared + PostgreSQL literal is deferred. +- AC-57: three-run measurement arm executed on this Sandbox (small fixture), + the real 9.987 GB capacity_measure fixture A with three runs per arm + (`none` 88.439 s median, fresh `rebuild` 118.439584 s median; the superseded + whole-Store build recorded 149.318 s / 246.371 s median on fixture A, + receipt `tests/fixtures/sqlite_compact_capacity_measure_whole_store.json`, so + the same-fixture ratio is 2.08x), and the separate gc_bench medium fixture B + (`none` 82.683 s, `rebuild` 133.076 s median; receipt + `tests/fixtures/sqlite_compact_capacity_measure_gc_bench.json`). Both 10 GB + receipts predate #49 round 4's source mmap window and `quick_check` removal, + so these are the pre-head8 build/verify path, not a head8 re-measurement. + Its 50 GB projection, the 30% arm, PG `ACCESS EXCLUSIVE` duration and the + plain `VACUUM INTO` comparison are deferred/[E]. + +## Status and report projection + +T10 (merged from main after this branch forked) projects `.compact-op` +through `store.status`: `op {id, name, phase, started, uncertain}`, the +busy/unresolved `state` and the backend `phases`. The named control tests +in AC-51 and AC-52 assert that projection and the snake_case section 11 +render; the runner still asserts the recorded SQLite phases directly, so +AC-51 does not depend on the projection alone. + +## Run environment + +- T-G61 capacity run (separate opt-in test): Docker Sandbox `g61sbx` + (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 overlay, 4.0 TB free), rustc + 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF) plus one + `Durable` control, `CARGO_TARGET_DIR=/var/tmp/target-g61`, fixture under + `/var/tmp/g61big`; the test ran 1678 s. Caches: the ~10 GB Store fits in the + 135 GB page cache and no cache drop ran between the three runs of an arm; + the mark is CPU-bound at ~62 MB/s, not cache-bound. + +- gc_bench medium run (fixture B, this branch): 88 vCPU Intel Xeon + E5-2699C v4, Linux 7.0.0-43-generic, `TMPDIR` on overlayfs, rustc 1.85.1 + release, the `Disposable` (WAL/OFF) profile, declared host class + R730/PERC/rotational (detected unknown/ssd-or-unknown), warm page cache + (`drop_caches` refused 6/6), three runs per arm, fixture build 154.509 s. + Receipt: `tests/fixtures/sqlite_compact_capacity_measure_gc_bench.json`. + +- Fresh-copy re-measurement (this branch): Docker Sandbox `fxfresh` + (Linux x86_64, 88 vCPU, 135 GB RAM, ext4 `/dev/sda2` run directory), + rustc 1.85.1, release profile, `SqlitePersistenceProfile::Disposable` + (WAL/OFF), `CAPACITY_MEASURE=1 UPDATE_CAPACITY=1`, TMPDIR on the ext4 run + directory; the test ran 870 s and reused the same 9.987 GB fixture shape. + The Sandbox shares its host with other runs: the three `none` marks span + 82.0-149.9 s while the three `rebuild` marks stay 74.5-75.7 s. +- Docker Sandbox `t123csbx` (Linux x86_64, tmpfs `/tmp`, overlayfs + `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable` + (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`. +- Verified here: `sqlite_compact_matrix` (1 test); the persistence suites + `sqlite_compact` (34), `sqlite_compact_crash` (24), + `sqlite_compact_fixtures` (1), `admission` (10), `path_removal` (4), + `branch_removal` (4), `project_delete` (4), `epoch_fence` (7), + `gc_backfill` (5), `kept_objects` (7); the Store-service path suites + `sdk_path_merge_removal` (6), `sdk_mount_bind_failure` (1), + `sdk_path_whiteouts` (7), `sdk_path_hidden_semantics` (3), + `sdk_path_removal` (5); and the `layerfs-control + project_delete_worker` (8), the `layerfs-store + project_delete` (18), `compact_report_status` (6) and the + PostgreSQL AC-44 suite `postgresql_compact` (30 tests; AC-44's nine are + `a_fully_privileged_role_compacts`, + `a_role_without_create_on_the_database_is_refused_as_unsupported`, + `a_role_without_create_on_the_store_schema_is_refused_as_unsupported`, + `a_role_without_temp_is_refused_as_unsupported`, + `a_role_without_pg_read_all_stats_is_refused_as_unsupported`, + `a_lost_swap_reply_from_an_ended_owner_resumes_at_g_plus_1`, + `a_recorded_live_owner_without_the_store_key_is_terminated`, + `a_live_key_holding_owner_is_busy_and_not_terminated` and + `a_role_without_pg_signal_backend_is_refused_as_unsupported`). + All of those pass on the merged branch; the PostgreSQL + suites need `LAYERFS_TEST_POSTGRES_URL` and run in the store-ext4 + --full lane. +- The control `restore_install` arm that opens a Store needs ext4/xfs/btrfs; + this Sandbox has no such direct filesystem, so that one case reports + `Unsupported` while its other two cases pass. It is an environment limit, + not a matrix result; the cited API tests are validated by source name only + in this run. diff --git a/docs/architecture/api/README.md b/docs/architecture/api/README.md index db21e6b29f..6ba0a7f4bd 100644 --- a/docs/architecture/api/README.md +++ b/docs/architecture/api/README.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The public surface: the `layerfs-api` call table with its one dispatcher, and the front ends (the `layerfs` command, Node, Python and a local MCP server) that expose its **80 calls in 11 families** with the same names and arguments (the MCP server lists 73: six monitoring calls are for a UI only, and `doctor.fix` is no tool). +The public surface: the `layerfs-api` call table with its one dispatcher, and the front ends (the `layerfs` command, Node, Python and a local MCP server) that expose its **85 calls in 12 families** with the same names and arguments (the MCP server lists 78: six monitoring calls are for a UI only, and `doctor.fix` is no tool). | | | | --- | --- | @@ -34,7 +34,7 @@ The public surface: the `layerfs-api` call table with its one dispatcher, and th | Part | What it holds | Source | | --- | --- | --- | -| `families()` | the eleven families in help order: `project`, `branch`, `commit`, `file`, `sandbox`, `workspace`, `machine`, `store`, `backup`, `logs`, `doctor` | [`lib.rs`](../../../api/crates/layerfs-api/src/lib.rs) | +| `families()` | the twelve families in help order: `home`, `project`, `branch`, `commit`, `file`, `sandbox`, `workspace`, `machine`, `store`, `backup`, `logs`, `doctor` | [`lib.rs`](../../../api/crates/layerfs-api/src/lib.rs) | | `calls/` | the table types, argument checks, folder scope, dispatcher, argv rendering, key casing, text form, platform policy and call context | [`calls/`](../../../api/crates/layerfs-api/src/calls), [Calls](calls/README.md) | | `/` | one file per call, `/.rs`; the family's `mod.rs` lists them in help order. Each `run` builds one typed request and calls one control-crate function | [`src/`](../../../api/crates/layerfs-api/src), [SDK](sdk/README.md) | | `doctor/` | explicit diagnostics outside the call table (`layerfs doctor`) | [`doctor/mod.rs`](../../../api/crates/layerfs-api/src/doctor/mod.rs) | @@ -85,7 +85,7 @@ give each call's own arguments, answer and errors. | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | -| [`calls_table.rs`](../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 80 calls in eleven families; one source file per call; refusals before any effect | +| [`calls_table.rs`](../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 85 calls in twelve families; one source file per call; refusals before any effect | | [`coverage.mjs`](../../../api/node/test/coverage.mjs) | `node --test api/node/test/coverage.mjs` | macOS or Linux, with the built addon, Python module and CLI | every front end matches the table | | [`file-history.mjs`](../../../api/node/examples/file-history.mjs) | `node api/node/examples/file-history.mjs` | macOS or Linux, with the built addon | a Store, Init, file writes and history reads | diff --git a/docs/architecture/api/calls/README.md b/docs/architecture/api/calls/README.md index dbaa40cd8e..c0efba82ae 100644 --- a/docs/architecture/api/calls/README.md +++ b/docs/architecture/api/calls/README.md @@ -43,7 +43,7 @@ The call table and the one dispatcher every front end goes through: what a call | Behaviour | Detail | Source | | --- | --- | --- | -| Table | twelve `Family { name, list }`; a `Call` is `{ family, call, args, options, help, run, quiet, opens }`, an `Arg` `{ name, kind, optional }`; 82 calls, help order | [`table.rs`](../../../../api/crates/layerfs-api/src/calls/table.rs), [`lib.rs`](../../../../api/crates/layerfs-api/src/lib.rs) `families` | +| Table | twelve `Family { name, list }`; a `Call` is `{ family, call, args, options, help, run, quiet, opens }`, an `Arg` `{ name, kind, optional }`; 85 calls, help order | [`table.rs`](../../../../api/crates/layerfs-api/src/calls/table.rs), [`lib.rs`](../../../../api/crates/layerfs-api/src/lib.rs) `families` | | JSON form | `table_json()`: `[{ family, call, args: [{ name, kind, optional }], options, help }]`; Node's `table`, Python's `TABLE`; the CLI parser and help read the same `Call` values | `table.rs` `table_json` | | Shared options | `PAGE` (`page_size`, `page_pos`), `DIFF` (`paths_only`, `page_size`, `page_pos`, `cursor`), `MESSAGE` (`message`), `PROGRESS` (`on_progress`), `WINDOW` (`cursor`, `before`, `limit`), `FILES` (`depth`, `cursor`, `limit`, `offset`, `length`) | `table.rs` | | Kinds | below | `table.rs` `Kind` | @@ -121,8 +121,14 @@ poll would bury the calls that changed something) and with `LAYERFS_LOGS=0` | `tag` | a top-level `log_tag` argument (taken out before the check; no entry declares it), else `LAYERFS_LOGS_TAG`; at most 128 bytes | Values are cut to 200 characters. Content, JSON, and the `Text` arguments -`command`, `message`, `old`, `new`, `password` and `url` are logged only as -`` (`REDACTED`). The line is written after the outcome and never +`command`, `message`, `old`, `new`, `password`, `url` and every path argument +(`path`, `left_path`, `right_path`, `from_path`, `to_path`, `path_prefix`) +are logged only as `` (`REDACTED`), so a `project.remove_file` or +`file.*` path never enters the log (#41 §4.3). A path counts its normalized +form, one leading `/` stripped (`/a` and `a` are both `<1 bytes>`); a glob or +grep `pattern` is not a path and counts its own bytes. A refusal contributes +only its code: its message and `detail` (which may echo the path, as `Removed` +does) are never logged. The line is written after the outcome and never changes it. Reading and clearing it are the [`logs`](../sdk/logs.md) calls. ## Invariants @@ -157,7 +163,7 @@ changes it. Reading and clearing it are the [`logs`](../sdk/logs.md) calls. | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | -| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host, temporary folders | twelve families and 82 calls; unknown calls and arguments refused before any effect; each family reached; JSON round trips; sizes and keys; a relative home refused; one source file per call | +| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host, temporary folders | twelve families and 85 calls; unknown calls and arguments refused before any effect; each family reached; JSON round trips; sizes and keys; a relative home refused; one source file per call | | [`calls_scope.rs`](../../../../api/crates/layerfs-api/tests/calls_scope.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_scope` | any Unix host | a scope names its project, refuses every call it does not cover, needs a Sandbox of its project, and a scoped `branch.list` reads every page | | [`scope_references.rs`](../../../../api/crates/layerfs-api/tests/scope_references.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test scope_references` | any Unix host | fork and publish references outside the namespace refused before any effect; the JSON form refuses the same | | [`logs.rs`](../../../../api/crates/layerfs-api/tests/logs.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test logs` | any Unix host | one line per call, rendering and redaction, `logs.list` and `logs.reset` | diff --git a/docs/architecture/api/calls/coverage.md b/docs/architecture/api/calls/coverage.md index f6bccf11e3..84445a48cf 100644 --- a/docs/architecture/api/calls/coverage.md +++ b/docs/architecture/api/calls/coverage.md @@ -42,7 +42,7 @@ check and proves a missing, renamed or extra call is reported. -61 calls in 9 families, counted from the Node table. Calls present: Node 61, Python 61, Python `aio` 61, CLI 61, MCP 61 (34 in `work`, 27 admin only). +85 calls in 12 families, counted from the Node table. Calls present: Node 85, Python 85, Python `aio` 85, CLI 85, MCP 78 (38 in `work`, 40 admin only; 6 UI only, in no profile). Argument names that are Python keywords: none. @@ -50,27 +50,35 @@ Front-end problems: none. | Call | Positionals (`?` optional) | Options (`()` a function) | Node | Python | Python `aio` | CLI | MCP | | --- | --- | --- | --- | --- | --- | --- | --- | +| `home.init` | - | - | yes | yes | yes | yes | admin | +| `home.get` | - | - | yes | yes | yes | yes | admin | | `project.init` | project, folder | timeout, on_progress() | yes | yes | yes | yes (--progress) | admin | | `project.list` | ?store | page_size, page_pos | yes | yes | yes | yes | yes | | `project.get` | project | - | yes | yes | yes | yes | yes | +| `project.stats` | project, ?branch, ?commit | - | yes | yes | yes | yes | yes | | `project.export` | project, destination | branch, on_progress() | yes | yes | yes | yes (--progress) | yes | -| `project.remove` | project | - | yes | yes | yes | yes | admin | -| `branch.list` | project | page_size, page_pos | yes | yes | yes | yes | yes | +| `project.delete` | project | - | yes | yes | yes | yes | admin | +| `project.remove_file` | project, path | - | yes | yes | yes | yes | admin | +| `branch.list` | project | page_size, page_pos, page | yes | yes | yes | yes | yes | | `branch.get` | project, branch | - | yes | yes | yes | yes | yes | +| `branch.commits` | project, branch | page, page_size, at_commit, include_storage | yes | yes | yes | yes | yes | +| `branch.get_commit` | project, branch, commit | - | yes | yes | yes | yes | yes | | `branch.fork` | project, name, from_branch, ?from_commit | - | yes | yes | yes | yes | yes | | `branch.merge` | project, name, a, b | message, units | yes | yes | yes | yes | yes | -| `branch.publish` | project, branch, ?commit | - | yes | yes | yes | yes | yes | -| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | +| `branch.publish` | project, branch, ?commit | expected_base, expected_head, force | yes | yes | yes | yes | yes | +| `branch.diff` | project, from_branch, to_branch | paths_only, page_size, page_pos, cursor, page, from_commit, to_commit, path_prefix, include_attribution, continuation | yes | yes | yes | yes | yes | +| `branch.remove` | project, branch | - | yes | yes | yes | yes | admin | | `commit.list` | project, branch | page_size, page_pos | yes | yes | yes | yes | yes | | `commit.get` | project, branch, commit | - | yes | yes | yes | yes | yes | | `commit.diff` | project, from_commit, to_commit | paths_only, page_size, page_pos, cursor | yes | yes | yes | yes | yes | | `file.read` | project, branch, path, ?commit | start_line, end_line | yes | yes | yes | yes | yes | | `file.stat` | project, branch, path, ?commit | - | yes | yes | yes | yes | yes | -| `file.list` | project, branch, path, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.glob` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.grep` | project, branch, path, pattern, ?commit | page_size, page_pos | yes | yes | yes | yes | yes | -| `file.blame` | project, branch, path, ?commit | from_line | yes | yes | yes | yes | yes | -| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit | yes | yes | yes | yes | yes | +| `file.list` | project, branch, path, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.glob` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.grep` | project, branch, path, pattern, ?commit | page_size, page, page_pos | yes | yes | yes | yes | yes | +| `file.revision` | project, branch, path | at_commit, traversal, page, page_size, continuation | yes | yes | yes | yes | yes | +| `file.blame` | project, branch, path, ?commit | from_line, line_count | yes | yes | yes | yes | yes | +| `file.diff` | project, left_branch, left_path, right_branch, right_path | left_commit, right_commit, include_attribution, allow_missing, page, page_size | yes | yes | yes | yes | yes | | `file.write` | project, branch, path, content | message, mode | yes | yes | yes | yes (stdin/--from) | yes | | `file.edit` | project, branch, path | old, new, message | yes | yes | yes | yes | yes | | `file.rm` | project, branch, path | message | yes | yes | yes | yes | yes | @@ -78,10 +86,11 @@ Front-end problems: none. | `file.mv` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.cp` | project, branch, from_path, to_path | message | yes | yes | yes | yes | yes | | `file.apply` | project, branch, changes | message | yes | yes | yes | yes (stdin/--from) | yes | -| `sandbox.create` | project, name, ?machine | image | yes | yes | yes | yes | admin | -| `sandbox.list` | ?project | page_size, page_pos | yes | yes | yes | yes | yes | +| `sandbox.create` | project, name, ?machine | image, on_progress() | yes | yes | yes | yes (--progress) | admin | +| `sandbox.list` | ?project | page_size, page_pos, live | yes | yes | yes | yes | yes | | `sandbox.get` | name | page_size, page_pos | yes | yes | yes | yes | yes | | `sandbox.close` | name | force | yes | yes | yes | yes | admin | +| `sandbox.files` | sandbox, ?path | depth, cursor, limit, offset, length | yes | yes | yes | yes | no (UI only) | | `workspace.mount` | sandbox, branch | name | yes | yes | yes | yes | yes | | `workspace.exec` | ws, command | message, stream, stdout(), stderr(), exec_id, on_progress() | yes | yes | yes | yes (--progress, --stream) | yes | | `workspace.exec_cancel` | ws, exec_id | grace_ms | yes | yes | yes | yes | yes | @@ -91,15 +100,22 @@ Front-end problems: none. | `machine.list` | - | - | yes | yes | yes | yes | admin | | `machine.platforms` | - | - | yes | yes | yes | yes | admin | | `machine.get` | name | - | yes | yes | yes | yes | admin | +| `machine.images` | name | - | yes | yes | yes | yes | admin | +| `machine.load` | name, image | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.prune` | name, ?image | unused | yes | yes | yes | yes | admin | | `machine.status` | name | - | yes | yes | yes | yes | admin | -| `machine.start` | name | - | yes | yes | yes | yes | admin | -| `machine.stop` | name | - | yes | yes | yes | yes | admin | -| `machine.create` | name, platform | cpus, memory, disk, image | yes | yes | yes | yes | admin | +| `machine.stats` | machine | - | yes | yes | yes | yes | no (UI only) | +| `machine.space` | machine | - | yes | yes | yes | yes | no (UI only) | +| `machine.files` | machine, ?path | depth, cursor, limit, offset, length | yes | yes | yes | yes | no (UI only) | +| `machine.start` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `machine.stop` | name | force | yes | yes | yes | yes | admin | +| `machine.create` | name, platform | cpus, memory, disk, image, os_image, on_progress() | yes | yes | yes | yes (--progress) | admin | | `machine.remove` | name | - | yes | yes | yes | yes | admin | | `machine.update` | name | cpus, memory, image, docker, wsl, daemon | yes | yes | yes | yes | admin | | `store.list` | - | - | yes | yes | yes | yes | admin | | `store.get` | name | - | yes | yes | yes | yes | admin | | `store.status` | name | - | yes | yes | yes | yes | admin | +| `store.stats` | name | - | yes | yes | yes | yes | admin | | `store.create` | name | - | yes | yes | yes | yes | admin | | `store.connect` | name, url | password, synchronous_commit | yes | yes | yes | yes | admin | | `store.attach` | name, machine | url, password | yes | yes | yes | yes | admin | @@ -107,12 +123,18 @@ Front-end problems: none. | `store.push` | name, machine | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.pull` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.remove` | name | - | yes | yes | yes | yes | admin | +| `store.compact` | name | - | yes | yes | yes | yes | admin | | `store.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.connect` | name, url | key, region, part_size | yes | yes | yes | yes | admin | | `backup.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.list` | ?bucket | - | yes | yes | yes | yes | admin | | `backup.create` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | | `backup.restore` | store, bucket, ?file | on_progress() | yes | yes | yes | yes (--progress) | admin | +| `logs.list` | - | op, project, client, tag, outcome, since, cursor, limit | yes | yes | yes | yes | admin | +| `logs.reset` | - | - | yes | yes | yes | yes | admin | +| `logs.machine` | machine | cursor, before, limit | yes | yes | yes | yes | no (UI only) | +| `logs.sandbox` | sandbox | cursor, before, limit | yes | yes | yes | yes | no (UI only) | +| `doctor.fix` | id | - | yes | yes | yes | yes | no (no tool) | diff --git a/docs/architecture/api/calls/errors-and-events.md b/docs/architecture/api/calls/errors-and-events.md index e2b8cd3f27..20db4eb37a 100644 --- a/docs/architecture/api/calls/errors-and-events.md +++ b/docs/architecture/api/calls/errors-and-events.md @@ -33,6 +33,8 @@ front ends carry it unchanged. | `NoSuchBranch`, `NoSuchCommit` | no such Branch, or no such Commit on it | [`calls/branch.rs`](../../../../control/crates/layerfs-store/src/calls/branch.rs), [`history/lookup.rs`](../../../../engine/crates/layerfs-store-service/src/history/lookup.rs) | | `NoSuchSandbox`, `NotMounted` | no Sandbox of that name; no Workspace of that name or path | [`family.rs`](../../../../control/crates/layerfs-sandbox/src/family.rs), [`workspace/serve.rs`](../../../../control/crates/layerfs-sandbox/src/workspace/serve.rs) | | `NoSuchPath`, `IsDirectory`, `NotDirectory`, `NotFile`, `NotEmpty`, `Binary` | path refusals of the `file` calls | [`write/mutation.rs`](../../../../engine/crates/layerfs-store-service/src/write/mutation.rs), [`read/`](../../../../engine/crates/layerfs-store-service/src/read) | +| `Removed` | a path is hidden by `project.remove_file`; `detail.paths[]` names each hidden path and the frontier serial that governs it | [`history/lookup.rs`](../../../../engine/crates/layerfs-store-service/src/history/lookup.rs), [`history/error.rs`](../../../../core/crates/layerfs-history/src/contract/error.rs) | +| `StaleStore` | the Store was replaced or moved under this handle; nothing on it is read or written | [`history/lookup.rs`](../../../../engine/crates/layerfs-store-service/src/history/lookup.rs), [`sqlite/lifecycle/identity.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/identity.rs) | | `Exists`, `NameTaken`, `ReservedName` | a name is taken or reserved; a database already connected | [`postgresql/connect.rs`](../../../../control/crates/layerfs-store/src/postgresql/connect.rs), [`history/branch.rs`](../../../../engine/crates/layerfs-store-service/src/history/branch.rs) | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs), [`inventory.rs`](../../../../control/crates/layerfs-control/src/inventory.rs) | | `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), no daemon connection came free, or a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing, so the caller may repeat the call); nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | @@ -45,7 +47,7 @@ front ends carry it unchanged. | `Unavailable`, `Denied` | a server, bucket, worker or host cannot be reached, or refused the login or secret | [`place.rs`](../../../../control/crates/layerfs-store/src/place.rs), [`worker_server.rs`](../../../../control/crates/layerfs-control/src/worker_server.rs) | | `StaleSandbox`, `NoStore`, `Unauthorized`, `UnknownMethod` | a saved daemon not ready; a Store file missing where it should be; a Sandbox record not owned by this user or not private; an operation no one serves | [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`service.rs`](../../../../control/crates/layerfs-store/src/service.rs), [`files.rs`](../../../../control/crates/layerfs-sandbox/src/files.rs) | | `Failed` | tried and did not happen, or no code of its own (`outcome::failed`); whatever exists is kept | [`outcome.rs`](../../../../control/crates/layerfs-home/src/outcome.rs) | -| `Unknown` | the outcome is not known | [Uncertain outcomes](#uncertain-outcomes) | +| `Unknown` | the outcome is not known; a GC call names its durable operation (`detail {entity, phase, op}`, with `phase`/`op` null when none is known) | [Uncertain outcomes](#uncertain-outcomes) | A daemon refusal of a Workspace call carries the daemon's `ControlCode` as its code (`Busy`, `HeadMoved`, `Missing`, `Capacity`, `Invalid`, `Failed`, @@ -96,7 +98,7 @@ The CLI refuses a wrong command line itself, with exit 2, before any call | Code | Outcome | Source | | --- | --- | --- | -| `Unknown` | may have happened. Every call is one attempt; nothing is resent. Read state with `list`, `get` or `status` before acting again | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) `lost`, `answer`; [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) `retain`; [`backup/record.rs`](../../../../control/crates/layerfs-store/src/backup/record.rs) | +| `Unknown` | may have happened. Every call is one attempt; nothing is resent. A GC call's detail names its entity and, when one is known, its durable op. Read state with `list`, `get` or `status` before acting again | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) `lost`, `answer`; [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) `retain`; [`backup/record.rs`](../../../../control/crates/layerfs-store/src/backup/record.rs) | | `Unresolved` | an earlier `Unknown` on this entity stands; this call did nothing | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) `blocks` | | daemon `Unknown` | a Workspace operation's outcome is uncertain at the daemon | [`control_types.rs`](../../../../engine/crates/layerfs-bridge/src/control_types.rs) `ControlCode::Unknown` | | `Failed` with `attempted: true` after a daemon exchange | uncertain, though the code is `Failed`: a lost daemon reply is not mapped to `Unknown` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation` | diff --git a/docs/architecture/api/front-ends/README.md b/docs/architecture/api/front-ends/README.md index 840f4121af..b04e06aa60 100644 --- a/docs/architecture/api/front-ends/README.md +++ b/docs/architecture/api/front-ends/README.md @@ -105,8 +105,9 @@ usage lines built from the entry ([`help.rs`](../../../../api/crates/layerfs-cli for a `Bool`, `[--progress]` for `on_progress`). The banner ([`banner.rs`](../../../../api/crates/layerfs-cli/src/banner.rs)) prints only when stdout is a terminal; `NO_COLOR` turns colour off. `--version --json` -gives `{ version, commit, dirty, features }` (`features`: the Cargo features -`machine`, `postgresql`, `tls` built in, and `sandbox-runtime` when namespaces +gives `{ version, commit, dirty, features }` (`features`: `postgresql`, always +on because the engine is built into every artifact; the Cargo features +`machine` and `tls` when built in; and `sandbox-runtime` when namespaces Sandboxes run on the bundle's runtime layer, R5) and `help --json` gives `{ calls, commands, switches }` with `calls` exactly `table_json` ([`capabilities.rs`](../../../../api/crates/layerfs-cli/src/capabilities.rs)). diff --git a/docs/architecture/api/front-ends/bindings.md b/docs/architecture/api/front-ends/bindings.md index 231d4720a8..b0be3400cd 100644 --- a/docs/architecture/api/front-ends/bindings.md +++ b/docs/architecture/api/front-ends/bindings.md @@ -70,7 +70,7 @@ links it and holds the build and type scripts. | Test | Checks | | --- | --- | -| [`calls.mjs`](../../../../api/node/test/calls.mjs) | eleven families, every function; required-argument refusals; a Store, Init with progress, file and Branch calls, bytes, a folder scope; the call log's `client` and `log_tag` | +| [`calls.mjs`](../../../../api/node/test/calls.mjs) | twelve families, every function; required-argument refusals; a Store, Init with progress, file and Branch calls, bytes, a folder scope; the call log's `client` and `log_tag` | | [`shell.mjs`](../../../../api/node/test/shell.mjs) | the command and the package refuse every call alike; a live Store read both ways (needs `target/debug/layerfs`) | | [`events.mjs`](../../../../api/node/test/events.mjs), [`progress.mjs`](../../../../api/node/test/progress.mjs) | handler failures, delivery before settling, exec output through functions and `stream: true` (exec half needs `LAYERFS_DAEMON`) | | [`values.mjs`](../../../../api/node/test/values.mjs) | exact `bigint`, refused values, bytes and the one-key map | diff --git a/docs/architecture/api/front-ends/mcp.md b/docs/architecture/api/front-ends/mcp.md index 3797fcd6de..7fc9f79a20 100644 --- a/docs/architecture/api/front-ends/mcp.md +++ b/docs/architecture/api/front-ends/mcp.md @@ -100,14 +100,14 @@ either) | Bucket | Calls | | --- | --- | | `work` | the `branch`, `commit`, `file` and `workspace` families; `project.list`, `project.get`, `project.export`, `sandbox.list`, `sandbox.get` | -| `admin` | `project.init`, `project.remove`, `sandbox.create`, `sandbox.close`; the `machine` (but `machine.stats`, `machine.space` and `machine.files`), `store` and `backup` families; `logs.list`, `logs.reset` | +| `admin` | `project.init`, `project.delete`, `project.remove_file`, `branch.remove`, `store.compact`, `sandbox.create`, `sandbox.close`; the `machine` (but `machine.stats`, `machine.space` and `machine.files`), `store` and `backup` families; `logs.list`, `logs.reset` | | `ui` (no profile) | `sandbox.files`, `machine.stats`, `machine.space`, `machine.files`, `logs.machine`, `logs.sandbox` | | `none` | the `doctor` family (`doctor.fix`): like `layerfs doctor` itself, which is outside the table and no tool, removing a leftover is a person's repair | `work` lets an agent read a project, work on Branches and files (each write is one explicit Commit), run in a mounted Workspace and export a snapshot. `admin` adds the calls that start workers, move whole Stores, hold -credentials, delete records or change the shape of the home. A new call with +credentials, retire or compact records, or change the shape of the home. A new call with no bucket fails `tools()`, generation and the test rather than appearing silently. Regenerate the table below with `node api/mcp/scripts/gen-profiles.mjs` (`--check` compares; both need the @@ -115,18 +115,19 @@ built addon). -82 calls: 38 in `work`, 37 in `admin` only, 6 in no profile (`ui`), 1 no tool; `--profile admin` lists 75. +85 calls: 38 in `work`, 40 in `admin` only, 6 in no profile (`ui`), 1 no tool; `--profile admin` lists 78. | Call | Bucket | Profiles that list it | Why | | --- | --- | --- | --- | | `home.init` | admin | admin | set the home up and read what this build would use with it; administration | | `home.get` | admin | admin | set the home up and read what this build would use with it; administration | | `project.init` | admin | admin | import a host folder into a Store; administration | -| `project.list` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | -| `project.get` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | -| `project.stats` | work | work, admin | read a project's record or its entry counts, or export one (work); importing and removing a project are administration | +| `project.list` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | +| `project.get` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | +| `project.stats` | work | work, admin | read a project's record or its entry counts, or export one (work); importing, deleting or hiding a path in a project is administration | | `project.export` | work | work, admin | export a Branch snapshot to a local folder under a launch-time --export-root; a folder scope refuses it | -| `project.remove` | admin | admin | remove a project from its Store; administration | +| `project.delete` | admin | admin | retire a project in its Store and remove its home entry; administration; a folder scope refuses it | +| `project.remove_file` | admin | admin | hide one absolute path from every node at or below its marker frontier; administration; a folder scope refuses it | | `branch.list` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.get` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.commits` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | @@ -135,6 +136,7 @@ built addon). | `branch.merge` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.publish` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | | `branch.diff` | work | work, admin | work on Branches: list, read, fork, merge, publish and diff | +| `branch.remove` | admin | admin | retire one Branch in its Store; administration; a folder scope refuses it | | `commit.list` | work | work, admin | read a Branch's Commit history | | `commit.get` | work | work, admin | read a Branch's Commit history | | `commit.diff` | work | work, admin | read a Branch's Commit history | @@ -190,6 +192,7 @@ built addon). | `store.push` | admin | admin | create, connect, transfer and remove Stores; administration | | `store.pull` | admin | admin | create, connect, transfer and remove Stores; administration | | `store.remove` | admin | admin | create, connect, transfer and remove Stores; administration | +| `store.compact` | admin | admin | reclaim one Store's dead bytes; administration; a folder scope refuses it | | `store.disconnect` | admin | admin | create, connect, transfer and remove Stores; administration | | `backup.connect` | admin | admin | upload and restore a Store on a bucket; administration | | `backup.disconnect` | admin | admin | upload and restore a Store on a bucket; administration | diff --git a/docs/architecture/api/sdk/README.md b/docs/architecture/api/sdk/README.md index f9df716804..8d21e23e92 100644 --- a/docs/architecture/api/sdk/README.md +++ b/docs/architecture/api/sdk/README.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -Every call a program can make: **82 calls in 12 families**, the same names and arguments in JavaScript, Python and the `layerfs` command, one page per family. +Every call a program can make: **85 calls in 12 families**, the same names and arguments in JavaScript, Python and the `layerfs` command, one page per family. | | | | --- | --- | @@ -51,14 +51,14 @@ built table. | Family | Calls, in help order | Runs by default | Source | | --- | --- | --- | --- | | [`home`](home.md) | [`init`](home.md#homeinit), [`get`](home.md#homeget) | the caller's process; `get` reads files only and never opens the home | [`home/`](../../../../api/crates/layerfs-api/src/home) | -| [`project`](project.md) | [`init`](project.md#projectinit), [`list`](project.md#projectlist), [`get`](project.md#projectget), [`stats`](project.md#projectstats), [`export`](project.md#projectexport), [`remove`](project.md#projectremove) | `list`, `get`: home records; the rest where the Store's calls run | [`project/`](../../../../api/crates/layerfs-api/src/project) | -| [`branch`](branch.md) | [`list`](branch.md#branchlist), [`get`](branch.md#branchget), [`commits`](branch.md#branchcommits), [`get_commit`](branch.md#branchget_commit), [`fork`](branch.md#branchfork), [`merge`](branch.md#branchmerge), [`publish`](branch.md#branchpublish), [`diff`](branch.md#branchdiff) | where the Store's calls run | [`branch/`](../../../../api/crates/layerfs-api/src/branch) | +| [`project`](project.md) | [`init`](project.md#projectinit), [`list`](project.md#projectlist), [`get`](project.md#projectget), [`stats`](project.md#projectstats), [`export`](project.md#projectexport), [`delete`](project.md#projectdelete), [`remove_file`](project.md#projectremove_file) | `list`, `get`: home records; the rest where the Store's calls run | [`project/`](../../../../api/crates/layerfs-api/src/project) | +| [`branch`](branch.md) | [`list`](branch.md#branchlist), [`get`](branch.md#branchget), [`commits`](branch.md#branchcommits), [`get_commit`](branch.md#branchget_commit), [`fork`](branch.md#branchfork), [`merge`](branch.md#branchmerge), [`publish`](branch.md#branchpublish), [`diff`](branch.md#branchdiff), [`remove`](branch.md#branchremove) | where the Store's calls run | [`branch/`](../../../../api/crates/layerfs-api/src/branch) | | [`commit`](commit.md) | [`list`](commit.md#commitlist), [`get`](commit.md#commitget), [`diff`](commit.md#commitdiff) | where the Store's calls run | [`commit/`](../../../../api/crates/layerfs-api/src/commit) | | [`file`](file.md) | [`read`](file.md#fileread), [`stat`](file.md#filestat), [`list`](file.md#filelist), [`glob`](file.md#fileglob), [`grep`](file.md#filegrep), [`revision`](file.md#filerevision), [`blame`](file.md#fileblame), [`diff`](file.md#filediff), [`write`](file.md#filewrite), [`edit`](file.md#fileedit), [`rm`](file.md#filerm), [`mkdir`](file.md#filemkdir), [`mv`](file.md#filemv), [`cp`](file.md#filecp), [`apply`](file.md#fileapply) | where the Store's calls run | [`file/`](../../../../api/crates/layerfs-api/src/file) | | [`sandbox`](sandbox.md) | [`create`](sandbox.md#sandboxcreate), [`list`](sandbox.md#sandboxlist), [`get`](sandbox.md#sandboxget), [`close`](sandbox.md#sandboxclose), [`files`](sandbox.md#sandboxfiles) | records here; start and stop on the Sandbox's machine; `files`: where the Sandbox's folder is | [`sandbox/`](../../../../api/crates/layerfs-api/src/sandbox) | | [`workspace`](workspace.md) | [`mount`](workspace.md#workspacemount), [`exec`](workspace.md#workspaceexec), [`exec_cancel`](workspace.md#workspaceexec_cancel), [`commit`](workspace.md#workspacecommit), [`unmount`](workspace.md#workspaceunmount), [`status`](workspace.md#workspacestatus) | the Sandbox's daemon; `exec` and `exec_cancel` in the Sandbox's runtime | [`workspace/`](../../../../api/crates/layerfs-api/src/workspace) | -| [`machine`](machine.md) | [`list`](machine.md#machinelist), [`platforms`](machine.md#machineplatforms), [`get`](machine.md#machineget), [`images`](machine.md#machineimages), [`load`](machine.md#machineload), [`prune`](machine.md#machineprune), [`status`](machine.md#machinestatus), [`stats`](machine.md#machinestats), [`files`](machine.md#machinefiles), [`start`](machine.md#machinestart), [`stop`](machine.md#machinestop), [`create`](machine.md#machinecreate), [`remove`](machine.md#machineremove), [`update`](machine.md#machineupdate) | the caller's process, through the platform adapter; `stats`, `files`: where the machine's data is | [`machine/`](../../../../api/crates/layerfs-api/src/machine) | -| [`store`](store.md) | [`list`](store.md#storelist), [`get`](store.md#storeget), [`status`](store.md#storestatus), [`stats`](store.md#storestats), [`create`](store.md#storecreate), [`connect`](store.md#storeconnect), [`attach`](store.md#storeattach), [`detach`](store.md#storedetach), [`push`](store.md#storepush), [`pull`](store.md#storepull), [`remove`](store.md#storeremove), [`disconnect`](store.md#storedisconnect) | the caller's process; sqlite file steps in the Store machine's worker | [`store/`](../../../../api/crates/layerfs-api/src/store) | +| [`machine`](machine.md) | [`list`](machine.md#machinelist), [`platforms`](machine.md#machineplatforms), [`get`](machine.md#machineget), [`images`](machine.md#machineimages), [`load`](machine.md#machineload), [`prune`](machine.md#machineprune), [`status`](machine.md#machinestatus), [`stats`](machine.md#machinestats), [`space`](machine.md#machinespace), [`files`](machine.md#machinefiles), [`start`](machine.md#machinestart), [`stop`](machine.md#machinestop), [`create`](machine.md#machinecreate), [`remove`](machine.md#machineremove), [`update`](machine.md#machineupdate) | the caller's process, through the platform adapter; `stats`, `space`, `files`: where the machine's data is | [`machine/`](../../../../api/crates/layerfs-api/src/machine) | +| [`store`](store.md) | [`list`](store.md#storelist), [`get`](store.md#storeget), [`status`](store.md#storestatus), [`stats`](store.md#storestats), [`create`](store.md#storecreate), [`connect`](store.md#storeconnect), [`attach`](store.md#storeattach), [`detach`](store.md#storedetach), [`push`](store.md#storepush), [`pull`](store.md#storepull), [`remove`](store.md#storeremove), [`compact`](store.md#storecompact), [`disconnect`](store.md#storedisconnect) | the caller's process; sqlite file steps in the Store machine's worker | [`store/`](../../../../api/crates/layerfs-api/src/store) | | [`backup`](backup.md) | [`connect`](backup.md#backupconnect), [`disconnect`](backup.md#backupdisconnect), [`list`](backup.md#backuplist), [`create`](backup.md#backupcreate), [`restore`](backup.md#backuprestore) | the caller's process; `create` stages on the Store's machine | [`backup/`](../../../../api/crates/layerfs-api/src/backup) | | [`logs`](logs.md) | [`list`](logs.md#logslist), [`reset`](logs.md#logsreset), [`machine`](logs.md#logsmachine), [`sandbox`](logs.md#logssandbox) | `list`, `reset`: the caller's process, home files only; `machine`, `sandbox`: where the log file is | [`logs/`](../../../../api/crates/layerfs-api/src/logs) | | [`doctor`](doctor.md) | [`fix`](doctor.md#doctorfix) | the machine its id names: the caller's process on `native`, else that machine's worker | [`doctor/`](../../../../api/crates/layerfs-api/src/doctor) | @@ -106,21 +106,21 @@ in the [API layer page](../README.md#calling-conventions). | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | | [`inventory.mjs`](../../../../api/node/test/branch-file/inventory.mjs) | `perl -e 'alarm 120; exec @ARGV' node --test api/node/test/branch-file/inventory.mjs` | macOS or Linux, with the built addon (`npm --prefix api run build:native`) and `target/debug/layerfs` | the count line and one heading per call match the built table; branch and file CLI help exposes every option | -| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 80 calls in eleven families; one source file per call; unknown calls and arguments refused before any effect | +| [`calls_table.rs`](../../../../api/crates/layerfs-api/tests/calls_table.rs) | `cargo +1.85.1 test --locked -p layerfs-api --test calls_table` | any Unix host | 85 calls in twelve families; one source file per call; unknown calls and arguments refused before any effect | | [`file-history.mjs`](../../../../api/node/examples/file-history.mjs) | `node api/node/examples/file-history.mjs` | macOS or Linux, with the built addon | a Store, Init, file writes and history reads | ## Pages | Page | What it covers | | --- | --- | -| [project](project.md) | Init, project records, stats, export, remove | -| [branch](branch.md) | Branches, saved history, fork, merge, publish, diff; the saved-query rules | +| [project](project.md) | Init, project records, stats, export, delete, path removal | +| [branch](branch.md) | Branches, saved history, fork, merge, publish, diff, retire; the saved-query rules | | [commit](commit.md) | a Branch's Commits and saved comparisons | | [file](file.md) | reads at a saved version; one-Commit writes | | [sandbox](sandbox.md) | Sandboxes: create, list, get, close, files | | [workspace](workspace.md) | mount, exec, cancel, commit, status, unmount | | [machine](machine.md) | built-in and VM machines | -| [store](store.md) | sqlite and PostgreSQL Stores | +| [store](store.md) | sqlite and PostgreSQL Stores; compaction | | [backup](backup.md) | buckets and Store backups | | [logs](logs.md) | the home's call log; a machine's worker log and a Sandbox's daemon log | | [doctor](doctor.md) | `doctor.fix`: remove one leftover the report listed | diff --git a/docs/architecture/api/sdk/branch.md b/docs/architecture/api/sdk/branch.md index e658f92160..dcc9eebd86 100644 --- a/docs/architecture/api/sdk/branch.md +++ b/docs/architecture/api/sdk/branch.md @@ -2,7 +2,7 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The `branch` calls: list and read a project's Branches and their saved history, fork, merge, publish to `main`, and compare two Branches, within the [SDK](README.md). +The `branch` calls: list and read a project's Branches and their saved history, fork, merge, publish to `main`, compare two Branches and retire one, within the [SDK](README.md). | | | | --- | --- | @@ -35,6 +35,7 @@ The `branch` calls: list and read a project's Branches and their saved history, | [`branch.merge`](#branchmerge) | `branch.merge` | write | | [`branch.publish`](#branchpublish) | `branch.publish` | write | | [`branch.diff`](#branchdiff) | `branch.diff_branches` | saved read | +| [`branch.remove`](#branchremove) | `branch.remove` | write | Each entry forwards a typed request to `layerfs_store::calls::branch`, which renames the table's snake_case fields to the Store method's @@ -304,6 +305,31 @@ comparison and attribution are each bounded to 64 MiB per request ([`read/branch_compare.rs`](../../../../engine/crates/layerfs-store-service/src/read/branch_compare.rs), [`read/compare.rs`](../../../../engine/crates/layerfs-store-service/src/read/compare.rs)). +### branch.remove + +Retires one Branch in its Store (spec §4). Its row is hidden from listings and +name resolution; fork, merge, publish and mount of it are refused, its Commits +and objects stay until a later [`store.compact`](store.md#storecompact), and +`main` is `ReservedName`. + +```text +JS await layerfs.branch.remove(project, branch) +Python layerfs.branch.remove(project, branch) +CLI layerfs branch remove +``` + +Returns `{ kind: "Removed", project, branch, removed_ms }`. A repeat is +`NoSuchBranch` (only `project.delete` is retry-idempotent); a concurrent +remove is `Busy` with phase `"retire"`. + +| Error | When | Source | +| --- | --- | --- | +| `ReservedName` | the Branch is `main` | [`methods/branch/remove.rs`](../../../../engine/crates/layerfs-store-service/src/methods/branch/remove.rs) | +| `NoSuchStore`, `NoSuchBranch` | the Store or Branch is missing (or already retired) | same | +| `MountsActive` | a mount of the Branch is live | [`history/admission.rs`](../../../../engine/crates/layerfs-store-service/src/history/admission.rs) | +| `Busy`, `Unresolved` | another user or operation holds the Branch, or a dead owner left an uncertain one | spec §5.1 | +| `Unknown` | worker route: the answer was lost | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | + ## Saved queries Shared by `branch.commits`, `branch.get_commit`, `branch.diff` and the diff --git a/docs/architecture/api/sdk/project.md b/docs/architecture/api/sdk/project.md index 114e6945be..debad1b73a 100644 --- a/docs/architecture/api/sdk/project.md +++ b/docs/architecture/api/sdk/project.md @@ -2,12 +2,12 @@ > **Status:** Current architecture guide for LayerFS 2.0.0; maintained with the source. -The `project` calls: import a host folder into a Store as `/`, read the home's project records, count or export a saved tree, and remove a project, within the [SDK](README.md). +The `project` calls: import a host folder into a Store as `/`, read the home's project records, count or export a saved tree, retire a project, and hide one path from every node at or below a frontier, within the [SDK](README.md). | | | | --- | --- | | Source | [`api/crates/layerfs-api/src/project`](../../../../api/crates/layerfs-api/src/project), [`layerfs-store/src/project`](../../../../control/crates/layerfs-store/src/project), [`layerfs-control/src/project.rs`](../../../../control/crates/layerfs-control/src/project.rs) | -| Runs by default | `list`, `get`: the caller's process, home records only. `init`, `stats`, `export`, `remove`: where the Store's calls run (the caller's process for PostgreSQL and a sqlite Store on a locally executing machine, else that machine's worker) | +| Runs by default | `list`, `get`: the caller's process, home records only. `init`, `stats`, `export`, `delete`, `remove_file`: where the Store's calls run (the caller's process for PostgreSQL and a sqlite Store on a locally executing machine, else that machine's worker) | | Design | [Stores and projects](../../control/stores-and-projects/README.md), [projects](../../control/stores-and-projects/projects.md), [Project Init](../../engine/project-init/README.md) | ## Calls @@ -22,7 +22,7 @@ The `project` calls: import a host folder into a Store as `/`, r +--> stats export (one pinned saved tree) +--> branch.*, commit.*, file.*, sandbox.create ... v - remove / (no Sandbox of it may be recorded) + delete / remove_file / ``` | Call | Returns | @@ -32,7 +32,8 @@ The `project` calls: import a host folder into a Store as `/`, r | [`project.get`](#projectget) | the project summary | | [`project.stats`](#projectstats) | entry counts of one pinned tree | | [`project.export`](#projectexport) | pinned identity and local output counts | -| [`project.remove`](#projectremove) | `{ stack, packs_kept: true, project }` | +| [`project.delete`](#projectdelete) | `{ kind: "Removed", project, deleted_ms, home_entry_removed }` | +| [`project.remove_file`](#projectremove_file) | `{ kind: "Removed" \| "UpToDate", project, path, frontier_serial, removed_ms }` | The **project summary** is `ProjectRecord::to_sdk` ([`project/record.rs`](../../../../control/crates/layerfs-store/src/project/record.rs)): @@ -84,7 +85,7 @@ Branch's base) and `root` (that Branch's effective root) | `Exists` | the project's record, entry or LayerStack name exists | `project/family.rs`, [`identity.rs`](../../../../control/crates/layerfs-store/src/identity.rs) | | `Busy` | also when another add's working rows remain in the Store | `identity.rs` `add_failure` | | `Capacity` | worker route: the machine's free disk is below the copy plus headroom | `project/send.rs` `room` | -| `Unsupported` | the kernel guard refuses the Store's directory (non-private mode/owner/type or unsupported filesystem); PostgreSQL in a build without `postgresql` | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | +| `Unsupported` | the kernel guard refuses the Store's directory (non-private mode/owner/type or unsupported filesystem) | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | | `NoStore` | no Store file at the Store's path | `identity.rs` | | `Failed` | deadline passed, read error or any other engine failure, **including an Init whose outcome is unknown**; the entry this call made is kept | `identity.rs` `add_failure` | | `Unknown` | worker route: a stage or the `project.init` answer was lost; the home gains no record, and the worker removes its staged copy when its Init ends | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) | @@ -106,7 +107,7 @@ CLI layerfs project list [] [--page-size ] [--page-pos