From 5cb0e2806fc7f7d0827732d50e767fe467e75603 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 18:57:13 +1300 Subject: [PATCH 01/10] chore: bump the Rust and Python packages to 0.6.16 --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 12 ++++++------ awa-cli/Cargo.toml | 4 ++-- awa-cli/pyproject.toml | 2 +- awa-python/Cargo.lock | 10 +++++----- awa-python/Cargo.toml | 6 +++--- awa-python/pyproject.toml | 4 ++-- awa-python/uv.lock | 4 ++-- awa-seaorm/Cargo.toml | 2 +- awa/Cargo.toml | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9aa90294f..f21e8fc5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,7 +383,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "awa" -version = "0.6.15" +version = "0.6.16" dependencies = [ "async-trait", "awa-macros", @@ -409,7 +409,7 @@ dependencies = [ [[package]] name = "awa-cli" -version = "0.6.15" +version = "0.6.16" dependencies = [ "assert_cmd", "awa-model", @@ -431,7 +431,7 @@ dependencies = [ [[package]] name = "awa-macros" -version = "0.6.15" +version = "0.6.16" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -441,7 +441,7 @@ dependencies = [ [[package]] name = "awa-metrics" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa-model", "opentelemetry", @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "awa-model" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa-macros", "blake3", @@ -471,7 +471,7 @@ dependencies = [ [[package]] name = "awa-seaorm" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa", "awa-testing", @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "awa-testing" -version = "0.6.15" +version = "0.6.16" dependencies = [ "async-trait", "awa-model", @@ -499,7 +499,7 @@ dependencies = [ [[package]] name = "awa-ui" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa-metrics", "awa-model", @@ -522,7 +522,7 @@ dependencies = [ [[package]] name = "awa-worker" -version = "0.6.15" +version = "0.6.16" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 12453de53..e0423e97c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ exclude = ["awa-python", "spike"] [workspace.package] -version = "0.6.15" +version = "0.6.16" edition = "2021" license = "MIT OR Apache-2.0" description = "Postgres-native background job queue — transactional enqueue, heartbeat crash recovery, SKIP LOCKED dispatch" @@ -25,11 +25,11 @@ categories = ["database", "asynchronous", "web-programming"] [workspace.dependencies] # Internal crates -awa-model = { path = "awa-model", version = "0.6.15" } -awa-macros = { path = "awa-macros", version = "0.6.15" } -awa-metrics = { path = "awa-metrics", version = "0.6.15" } -awa-worker = { path = "awa-worker", version = "0.6.15" } -awa = { path = "awa", version = "0.6.15" } +awa-model = { path = "awa-model", version = "0.6.16" } +awa-macros = { path = "awa-macros", version = "0.6.16" } +awa-metrics = { path = "awa-metrics", version = "0.6.16" } +awa-worker = { path = "awa-worker", version = "0.6.16" } +awa = { path = "awa", version = "0.6.16" } # Database sea-orm = { version = "=2.0.0-rc.38", default-features = false, features = ["sqlx-postgres", "runtime-tokio-rustls", "with-chrono", "with-json"] } diff --git a/awa-cli/Cargo.toml b/awa-cli/Cargo.toml index 96bffb3e5..75ca22fec 100644 --- a/awa-cli/Cargo.toml +++ b/awa-cli/Cargo.toml @@ -14,7 +14,7 @@ path = "src/main.rs" [dependencies] awa-model.workspace = true awa-worker.workspace = true -awa-ui = { path = "../awa-ui", version = "0.6.15" } +awa-ui = { path = "../awa-ui", version = "0.6.16" } axum.workspace = true sqlx.workspace = true tokio.workspace = true @@ -28,5 +28,5 @@ hex.workspace = true uuid.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.15" } +awa-testing = { path = "../awa-testing", version = "0.6.16" } assert_cmd = "2" diff --git a/awa-cli/pyproject.toml b/awa-cli/pyproject.toml index 3609b14c2..8bc2d2be4 100644 --- a/awa-cli/pyproject.toml +++ b/awa-cli/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "awa-cli" -version = "0.6.15" +version = "0.6.16" description = "CLI for the Awa Postgres-native job queue (migrations, admin, serve)" readme = "README.md" requires-python = ">=3.10" diff --git a/awa-python/Cargo.lock b/awa-python/Cargo.lock index f29fa6999..4946bf44a 100644 --- a/awa-python/Cargo.lock +++ b/awa-python/Cargo.lock @@ -95,7 +95,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "awa-macros" -version = "0.6.15" +version = "0.6.16" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "awa-metrics" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa-model", "opentelemetry", @@ -113,7 +113,7 @@ dependencies = [ [[package]] name = "awa-model" -version = "0.6.15" +version = "0.6.16" dependencies = [ "awa-macros", "blake3", @@ -133,7 +133,7 @@ dependencies = [ [[package]] name = "awa-python" -version = "0.6.15" +version = "0.6.16" dependencies = [ "async-trait", "awa-model", @@ -156,7 +156,7 @@ dependencies = [ [[package]] name = "awa-worker" -version = "0.6.15" +version = "0.6.16" dependencies = [ "async-trait", "awa-macros", diff --git a/awa-python/Cargo.toml b/awa-python/Cargo.toml index be022c1a4..363843d5f 100644 --- a/awa-python/Cargo.toml +++ b/awa-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "awa-python" -version = "0.6.15" +version = "0.6.16" edition = "2021" [lib] @@ -12,8 +12,8 @@ default = ["cel"] cel = ["awa-model/cel"] [dependencies] -awa-model = { path = "../awa-model", version = "0.6.15" } -awa-worker = { path = "../awa-worker", version = "0.6.15", features = ["__python-bridge"] } +awa-model = { path = "../awa-model", version = "0.6.16" } +awa-worker = { path = "../awa-worker", version = "0.6.16", features = ["__python-bridge"] } pyo3 = { version = "0.29", features = ["macros", "abi3-py310", "chrono"] } pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono", "json"] } diff --git a/awa-python/pyproject.toml b/awa-python/pyproject.toml index 76d77df44..d6a7b0dc6 100644 --- a/awa-python/pyproject.toml +++ b/awa-python/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "awa-pg" requires-python = ">=3.10" -version = "0.6.15" +version = "0.6.16" description = "Postgres-native background job queue — Python SDK with async/sync workers, transactional enqueue, and progress tracking. Install awa-pg[ui] to add the bundled web dashboard." readme = "README.md" license = {text = "MIT OR Apache-2.0"} @@ -34,7 +34,7 @@ classifiers = [ # # Kept opt-in so the default `awa-pg` install stays small — workers and # producers don't need the ~10 MB UI bundle. -ui = ["awa-cli==0.6.15"] +ui = ["awa-cli==0.6.16"] [project.urls] Homepage = "https://github.com/hardbyte/awa" diff --git a/awa-python/uv.lock b/awa-python/uv.lock index 063ed5887..2470233a8 100644 --- a/awa-python/uv.lock +++ b/awa-python/uv.lock @@ -88,12 +88,12 @@ wheels = [ [[package]] name = "awa-cli" -version = "0.6.15" +version = "0.6.16" source = { directory = "../awa-cli" } [[package]] name = "awa-pg" -version = "0.6.15" +version = "0.6.16" source = { editable = "." } [package.optional-dependencies] diff --git a/awa-seaorm/Cargo.toml b/awa-seaorm/Cargo.toml index 50a1c6688..7ed3fe6c1 100644 --- a/awa-seaorm/Cargo.toml +++ b/awa-seaorm/Cargo.toml @@ -14,6 +14,6 @@ serde_json.workspace = true sqlx.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.15" } +awa-testing = { path = "../awa-testing", version = "0.6.16" } serde = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/awa/Cargo.toml b/awa/Cargo.toml index 56a1d98f8..1c2a3c1e3 100644 --- a/awa/Cargo.toml +++ b/awa/Cargo.toml @@ -20,7 +20,7 @@ awa-macros.workspace = true awa-worker.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.15" } +awa-testing = { path = "../awa-testing", version = "0.6.16" } sqlx.workspace = true serde.workspace = true serde_json.workspace = true From 0a275101bdc43846a60c690a460279cce6e23550 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:02:36 +1300 Subject: [PATCH 02/10] docs: write the 0.6.16 release notes Turn the unreleased entries into the 0.6.16 section for operators upgrading from 0.6.15: a lead paragraph, the upgrade steps for the three schema patches and the prune control grants, the rolling-upgrade caveats for new queue lanes and 0.6.15 leaders, and the batch insert limits under Changed. --- CHANGELOG.md | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41880eab4..b7cea2415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,23 +4,42 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [Unreleased] -### Upgrading +## [0.6.16] — 2026-11-09 -- Run `awa migrate` once as the schema owner to apply `prune_controls` after the existing `claim_batch_job_index` and `insert_jobs_compat_batch` patches. It adds two tables and functions without altering existing tables; schema stays v40. Upgrade or fence every possible maintenance leader before relying on halt. See the [0.6.15 to 0.6.16 upgrade guide](docs/upgrade-0.6.15-to-0.6.16.md). -- The same run re-applies `claim_batch_job_index` on a 0.6.15 database, replacing each schema's `ensure_claim_batch_job_index` function. Like the first application it builds no index and takes no table locks. External migration runners: replace `R__claim_batch_job_index.sql` with the new version as well. -- Run `awa migrate` once as the schema owner, before or after rolling binaries. It replaces the body of `awa.insert_jobs_compat_batch`, which a 0.6.15 database now reports as the pending `insert_jobs_compat_batch` patch, and takes no table locks. The fix is in the function, so 0.6.15 producers get it as soon as the patch commits. External migration runners: replace `R__insert_jobs_compat_batch.sql` along with the rest of the `R__` set from `awa migrate --extract-to --from 40`. See the [upgrade guide](docs/upgrade-0.6.15-to-0.6.16.md). +Recommended for everyone on 0.6.15 using queue storage. Batch inserts (`insert_many_copy`, `insert_many_skip_unique_conflicts`) give way to per-row producers that share their keys or lanes instead of deadlocking with them, and creating a new queue lane no longer deadlocks between `QueueStorage` and SQL producers. Operators can durably halt and resume `maintenance.prune` across the fleet. `admin::get_job` no longer misses a failed job while prune moves it, and `awa migrate` re-applies the claim batch index and batch insert patches with fixes. -### Added +### Upgrading -- Durable operator halt/resume for `maintenance.prune`, in drain mode. Compatible leaders observe it on their existing session; dispatch, cron, promotion, rescue and enqueue continue. Status lists live runtimes without supported, recent reports and cannot report the fleet halted while any remain. Older binaries ignore the control. See [runtime controls](docs/runtime-controls.md). +- The schema stays at v40. Run `awa migrate` once as the schema owner while traffic keeps running. On a 0.6.15 database it applies three repeatable patches, each in its own transaction and none taking a table lock: `claim_batch_job_index` again, replacing each queue-storage schema's `ensure_claim_batch_job_index`; `insert_jobs_compat_batch`, replacing the body of `awa.insert_jobs_compat_batch`, so 0.6.15 producers get the deadlock fix as soon as it commits; and the new `prune_controls`, which adds the `awa.runtime_controls` and `awa.runtime_control_observations` tables and the halt and resume functions without altering existing tables. See the [upgrade guide](docs/upgrade-0.6.15-to-0.6.16.md). +- The patch does not grant runtime or operator access. Grant runtime roles `SELECT` on `awa.runtime_controls` and `SELECT, INSERT, UPDATE, DELETE` on `awa.runtime_control_observations`, and an operator role `SELECT` on both and on `awa.runtime_instances` plus `EXECUTE` on `awa.halt_runtime_scope` and `awa.resume_runtime_scope` ([runtime controls](docs/runtime-controls.md#install)). A 0.6.16 runtime that cannot read the control stops admitting prune within five seconds of the patch committing, while other maintenance continues, so apply the grants before rolling binaries, or expect prune to pause until they are in place. +- External migration runners: apply the new `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql`, then `R__prune_controls.sql`, after `V40`. Replace all `R__` files together with the full set from `awa migrate --extract-to --from 40`; `--pending` lists only the patches a database lacks. +- During the roll, a 0.6.15 `QueueStorage` process that is the first to use a new `(queue, priority)` lane can still deadlock with another producer of that lane. Create a new queue's lanes ahead of traffic with the 0.6.16 `awa storage prepare-queue`, or retry the failed transaction. +- 0.6.15 binaries ignore a prune halt and can still become maintenance leader. Upgrade or fence every process that can lead before relying on a halt, and check `awa control status`. +- Binaries can roll back to 0.6.15 with the patches left in place. A 0.6.15 leader ignores a stored halt. ### Fixed -- `insert_many_copy` and `insert_many_skip_unique_conflicts` no longer deadlock with per-row producers (generic `insert_many`, single-job inserts, 0.6.14 and earlier binaries) that share their unique keys or lanes. Since 0.6.15, when a per-row transaction held a lane and then inserted a key a waiting batch had claimed, PostgreSQL aborted one of the two with `deadlock detected` (`40P01`). The batch function now waits at most 500 ms (or half of its session's `deadlock_timeout`, if lower) while it holds claims or lanes: on a longer wait it releases them and tries again, so the per-row transaction proceeds and the batch then treats the keys it committed as already claimed. The same applies to a transaction that enqueues lanes in separate statements in the opposite order to a batch. This covers per-row sessions whose `deadlock_timeout` is above 500 ms, as PostgreSQL's default of 1 s is, and batches whose unique claims insert within that budget (10,000 to 20,000 keys on a fast server, fewer on a slow or busy one); larger batches behave as in 0.6.15. The protection lasts for 2 s of retries; then the batch makes one last attempt with ordinary lock waits, as in 0.6.15. A per-row transaction that holds a lane for longer than that before it inserts one of the batch's keys can therefore still deadlock with it, as in 0.6.15. A cycle through a lock the batch's own transaction took in an earlier statement also ends in `deadlock detected`, after about 3 to 4 s instead of 1 to 2 s, and usually aborts the batch's transaction. The caller's `lock_timeout` applies to each lock wait, as before. Results and the lane lock window are unchanged. +**Enqueue deadlocks** + +- `insert_many_copy` and `insert_many_skip_unique_conflicts` no longer deadlock with per-row producers (generic `insert_many`, single-job inserts, 0.6.14 and earlier binaries) that share their unique keys or lanes. Since 0.6.15 the batch function took all of a batch's unique claims before its lanes, so a per-row transaction that held a lane and then inserted a key the batch had claimed waited on the batch while the batch waited on it, and PostgreSQL aborted one of them with `deadlock detected` (`40P01`). For its first 2 s the function now limits each lock wait to 500 ms, or half of its own session's `deadlock_timeout` if lower. A longer wait rolls back the claims and lanes it holds and retries, so the per-row transaction proceeds and the batch then skips, or fails on, the keys that transaction committed. The same covers a transaction that enqueues lanes in separate statements in the opposite order to a batch. This holds when the batch call is the first thing its transaction locks, the per-row session's `deadlock_timeout` is above 500 ms (PostgreSQL's default is 1 s), and the batch inserts its unique claims within 500 ms: roughly 10,000 to 20,000 keys on a fast server, fewer on a slow or busy one. Results, the caller's `lock_timeout` on each wait, and the lane lock window are unchanged. +- Creating a new queue lane through `QueueStorage` (its batch enqueues, and workers promoting scheduled or retried jobs) no longer deadlocks with a concurrent `insert`, `insert_many` or `insert_many_copy`, or a 0.6.14 or 0.6.15 per-row producer, that is also the first to use the same `(queue, priority)` (#692). `QueueStorage` took the lane's provisioning lock before writing its `queue_lanes` row, while SQL inserts write the row first and take the lock in the head-table trigger that creates the lane's sequences, so each waited on the other and one failed with `40P01`. `QueueStorage` now writes the row first too. Lanes that already existed were never affected. No schema change. + +**Claim batch index** + - Claim prune no longer treats a partial GIN index on `lease_claim_batches.job_ids`, or a session with `quote_all_identifiers` on, as the claim batch index. It recognises only a valid, non-partial GIN index on `job_ids` alone. When another index or relation already has the index's name, `ensure_claim_batch_job_index` leaves it in place, returns false and raises a warning naming it, instead of reporting the slot indexed; the slot keeps scanning until an operator drops or renames it. - Refreshing `ensure_claim_batch_job_index` as a migration role that owns it but may only `SET ROLE` to the table owner (PostgreSQL 16+ membership `WITH INHERIT FALSE`) no longer leaves `EXECUTE` with `PUBLIC`. The refresh used to hand the function to the table owner first, after which that role could no longer revoke or grant on it, and PostgreSQL only warned. It now resets `EXECUTE` after the transfer, acting as the new owner when it cannot as itself, so exactly the roles holding `TRUNCATE` on `lease_claim_batches` can execute it, the former owner included. Migrations run with the owner as the effective role, as documented, were not affected. + +**Reads during prune** + - `admin::get_job` and the other queue-storage point reads of one job no longer return `JobNotFound` for a failed job while queue prune moves its row out of the slot being pruned. Since 0.6.14 the read used one snapshot, and a prune that committed after that snapshot was taken left the old slot empty and the moved row invisible to it. Prune moves each retained failed row once per ring cycle, and on a new schema prunes slot 0 right after the first rotation, so a read polling for `failed` could hit it within a second of a worker starting. A read that finds nothing now reads once more in a new snapshot. -- Creating a new queue lane through `QueueStorage` (its batch enqueues, and workers promoting scheduled or retried jobs) no longer deadlocks with a concurrent `insert`, `insert_many` or `insert_many_copy`, or a 0.6.14 or 0.6.15 per-row producer, that is also the first to use the same `(queue, priority)` (#692). `QueueStorage` took the lane's provisioning lock before writing its `queue_lanes` row, while SQL inserts write the row first and take the lock in the head-table trigger that creates the lane's sequences, so each transaction waited on the other and one failed with `40P01`. `QueueStorage` now writes the row first too. Lanes that already existed were never affected. No schema change; 0.6.15 `QueueStorage` processes keep the old order until they are upgraded. + +### Changed + +- When the batch function's bounded retries run out, it makes one last attempt with ordinary lock waits, as 0.6.15 did. A per-row transaction that holds a lane for longer than about 2.5 s before it inserts one of the batch's keys can therefore still deadlock with it, as can cycles of three or more transactions. A cycle through a lock the batch's own transaction took in an earlier statement, which rolling back the batch cannot release, now ends in `40P01` after about 3 to 4 s instead of 1 to 2 s, and usually aborts the batch's transaction rather than the per-row one. A batch whose unique claims take longer than 500 ms to insert behaves as in 0.6.15; split such batches if they share keys or lanes with per-row producers. + +### Added + +- Durable operator halt and resume for `maintenance.prune`, in drain mode: `awa control halt`, `status` and `resume`, or the `awa.halt_runtime_scope` and `awa.resume_runtime_scope` functions from an existing SQL session. Compatible leaders observe it on their existing session about once a second and stop admitting rotation, prune, rollups, cleanup, admin metadata refresh and descriptor publication; enqueue, dispatch, handlers, cron, promotion, rescue and admin batches continue. Resume names the current halt's UUID. Status lists live runtimes without supported, recent reports and cannot report the fleet halted while any remain. Older binaries ignore the control. See [runtime controls](docs/runtime-controls.md). ## [0.6.15] — 2026-11-09 From 505112fcc30606f0fe8b4b1389d94765284cac41 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:02:36 +1300 Subject: [PATCH 03/10] docs: make the 0.6.15 to 0.6.16 upgrade guide one sequence The guide was assembled from several changes' sections. It now has one list of upgrade steps covering the three schema patches and the prune control grants in order, a section per change, and a single roll back section. The batch insert section says the lane lock is held until the transaction ends, notes the subtransaction cache limit, and the 0.6.15 guide points to the fix. The README row lists all of 0.6.16. --- README.md | 2 +- docs/upgrade-0.6.14-to-0.6.15.md | 2 +- docs/upgrade-0.6.15-to-0.6.16.md | 101 ++++++++++++++++++------------- 3 files changed, 61 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 2995d4c7a..5b3c71ba3 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ All coordination through Postgres. The Rust runtime owns dispatch, leases, heart | [0.5 → 0.6 upgrade](docs/upgrade-0.5-to-0.6.md) | Step-by-step operator checklist for the staged storage transition | | [0.6.9 → 0.6.10 upgrade](docs/upgrade-0.6.9-to-0.6.10.md) | Rolling patch upgrade, rollback, and optional runtime privilege tightening | | [0.6.14 → 0.6.15 upgrade](docs/upgrade-0.6.14-to-0.6.15.md) | Rolling patch upgrade: bounded worker finalization, the claim batch index and the batch compatibility insert | -| [0.6.15 → 0.6.16 upgrade](docs/upgrade-0.6.15-to-0.6.16.md) | Rolling patch upgrade: operator prune halt, and batch compatibility inserts that give way to per-row producers instead of deadlocking | +| [0.6.15 → 0.6.16 upgrade](docs/upgrade-0.6.15-to-0.6.16.md) | Rolling patch upgrade: batch compatibility inserts that give way to per-row producers, new-lane provisioning order, the operator prune halt and its grants, and claim batch index fixes | | [Configuration reference](docs/configuration.md) | `QueueConfig`, `ClientBuilder`, Python `start()`, env vars | | [HTTP workers and callbacks](docs/http-callbacks.md) | Serverless dispatch, callback receiver endpoints, signature verification | | [Lifecycle hooks](docs/lifecycle-hooks.md) | React to job start/finish/retry/callback events for metrics, alerting, cleanup | diff --git a/docs/upgrade-0.6.14-to-0.6.15.md b/docs/upgrade-0.6.14-to-0.6.15.md index 820eee69c..19a0a7e48 100644 --- a/docs/upgrade-0.6.14-to-0.6.15.md +++ b/docs/upgrade-0.6.14-to-0.6.15.md @@ -47,7 +47,7 @@ What callers see: #### Unique keys shared with per-row producers -The batch function takes all of a batch's unique claims before any lane lock. Per-row inserts (generic `insert_many`, single-job inserts, and 0.6.14 and earlier binaries) take each job's claim and its lane together, lane by lane. When a per-row transaction already holds a lane the batch needs and then inserts a job whose unique key the batch has claimed but not yet committed, each waits on the other. PostgreSQL aborts one of them with `deadlock detected` (SQLSTATE `40P01`), where before the per-row insert would have waited and then skipped the job or failed with a unique violation (`23505`). Retry the aborted transaction: by then the key is either committed, giving the old outcome, or free. Two batches through the function take claims in key order and lanes in lane order, so they do not deadlock with each other this way. +The batch function takes all of a batch's unique claims before any lane lock. Per-row inserts (generic `insert_many`, single-job inserts, and 0.6.14 and earlier binaries) take each job's claim and its lane together, lane by lane. When a per-row transaction already holds a lane the batch needs and then inserts a job whose unique key the batch has claimed but not yet committed, each waits on the other. PostgreSQL aborts one of them with `deadlock detected` (SQLSTATE `40P01`), where before the per-row insert would have waited and then skipped the job or failed with a unique violation (`23505`). Retry the aborted transaction: by then the key is either committed, giving the old outcome, or free. Two batches through the function take claims in key order and lanes in lane order, so they do not deadlock with each other this way. 0.6.16 replaces the function's body so that a batch gives way to the per-row transaction instead; see [batch inserts and per-row producers](upgrade-0.6.15-to-0.6.16.md#batch-inserts-and-per-row-producers). #### Privileges diff --git a/docs/upgrade-0.6.15-to-0.6.16.md b/docs/upgrade-0.6.15-to-0.6.16.md index b0a60c7b4..967c6caff 100644 --- a/docs/upgrade-0.6.15-to-0.6.16.md +++ b/docs/upgrade-0.6.15-to-0.6.16.md @@ -1,6 +1,8 @@ # Upgrading from 0.6.15 to 0.6.16 -0.6.16 adds a durable, database-wide halt for `maintenance.prune`, and batch inserts no longer deadlock with per-row producers. It is a rolling patch upgrade, but older binaries ignore the control and can still become maintenance leader and prune. Upgrade or fence every potential leader before relying on halt. +0.6.16 is a rolling patch upgrade: the schema stays at v40 and traffic keeps running throughout. Batch inserts give way to per-row producers instead of deadlocking with them, `QueueStorage` creates new queue lanes in the same lock order as every other producer, operators can halt and resume `maintenance.prune` across the fleet, and point reads of a job no longer miss it while prune moves it. + +Two parts of the upgrade need ordering. The new prune control needs grants before 0.6.16 runtimes can read it, and a halt only holds once no 0.6.15 process can become maintenance leader. ## Upgrade @@ -12,64 +14,60 @@ awa --database-url "$AWA_MIGRATOR_DATABASE_URL" migrate ``` - The schema stays at v40. The output reports `Schema patch claim_batch_job_index: applied`, `Schema patch insert_jobs_compat_batch: applied` and `Schema patch prune_controls: applied`. The `insert_jobs_compat_batch` patch replaces the batch insert function's body; 0.6.15 producers get that fix as soon as the migration commits. - -3. Apply the [runtime and operator grants](runtime-controls.md#install). Runtime roles need `SELECT` on `awa.runtime_controls` and `SELECT, INSERT, UPDATE, DELETE` on `awa.runtime_control_observations`. Operators need `SELECT` on both tables and `awa.runtime_instances`, plus explicit `EXECUTE` on `awa.halt_runtime_scope(text, text, text)` and `awa.resume_runtime_scope(text, uuid)`. Neither role should have direct control-table writes. Reapply these exceptions after blanket grants and remove inherited mutation privileges from the granting role. -4. Roll workers, producers and admin processes to 0.6.16 after applying the grants. Compatible runtimes discover the patch without a restart or leadership change. They can also run before installation, retaining the existing reclamation policy while catalog observations remain fresh. Once they detect installation, missing read grants close prune admission when any existing permission expires, within five seconds; newly started runtimes have no permission to prune. Other scopes continue. If compatible runtimes are already deployed, expect this pause between migration and grants, and complete the grants before expecting pruning to resume. -5. Upgrade or fence every remaining process that can become maintenance leader. Inspect `awa control status` using the operator connection before relying on halt. Every live runtime, including standbys, must have a supported, recent protocol report. - -The 0.6.15 lifecycle-hook and finalizer bounds remain in place. Ring defaults, startup capacity checks and reclamation rate limits are unchanged. - -## New queue lanes during the roll - -When `QueueStorage` is the first to use a `(queue, priority)` lane, through a batch enqueue or a worker promoting scheduled or retried jobs, 0.6.16 writes the lane's `queue_lanes` row before taking its provisioning lock, the order SQL inserts already use. 0.6.15 takes them the other way round. Until every `QueueStorage` process runs 0.6.16, a 0.6.15 one and any other producer that are both first to use the same lane can still deadlock, and one transaction fails with `40P01`. Lanes that already exist are unaffected. To avoid it during the roll, create the lanes of a queue or priority before traffic first reaches it, with the 0.6.16 `awa storage prepare-queue`, or retry the failed transaction. - -## What the migration changes + On a 0.6.15 database the output reports `Schema patch claim_batch_job_index: applied`, `Schema patch insert_jobs_compat_batch: applied` and `Schema patch prune_controls: applied`. Each patch commits in its own transaction and none takes a lock on an `awa` or queue-storage table; an interrupted run can be run again to apply the rest. The batch insert fix is in the function, so 0.6.15 producers get it as soon as its patch commits. -### `claim_batch_job_index` +3. Apply the [runtime and operator grants](runtime-controls.md#install). The patch does not grant runtime or operator access. Runtime roles need `SELECT` on `awa.runtime_controls` and `SELECT, INSERT, UPDATE, DELETE` on `awa.runtime_control_observations`. Operators need `SELECT` on both tables and on `awa.runtime_instances`, plus explicit `EXECUTE` on `awa.halt_runtime_scope(text, text, text)` and `awa.resume_runtime_scope(text, uuid)`. Neither role should be able to write the control table directly: apply these exceptions after any blanket grants, and remove inherited mutation privileges from the granting role. +4. Roll workers, producers and admin processes to 0.6.16, in any order. Until every `QueueStorage` process runs 0.6.16, see [new queue lanes](#new-queue-lanes). +5. Upgrade or fence every remaining process that can become maintenance leader, standbys included. Then run `awa control status` on the operator connection: before relying on a halt, every live runtime must have a supported, recent report. -0.6.16 replaces the 0.6.15 `.ensure_claim_batch_job_index(slot)`, so `awa migrate` applies this patch again. The 0.6.15 function counted a partial GIN index on `job_ids` as the claim batch index, and also any index already holding the index's name, so claim prune could skip building the real one. The new function accepts only a valid, non-partial GIN index on `job_ids`. When something else already has the name `lease_claim_batches__job_ids_idx`, it raises a warning naming it and leaves it in place; drop or rename it and claim prune builds the index the next time it empties that slot. +Binaries can also go before the migration. A 0.6.16 runtime on a database without `prune_controls` keeps the existing reclamation policy and discovers the patch without a restart. Once the patch is installed, a runtime that cannot read the control stops admitting prune within five seconds, and a newly started one does not admit it at all, until the grants are in place; other maintenance continues. If 0.6.16 runtimes are already running, expect prune to pause between steps 2 and 3, and apply the grants before expecting it to resume. -As in 0.6.15 the patch replaces functions only: it builds no index and takes no lock on any `awa` or queue-storage table. Binaries from 0.6.15 call the new function unchanged. +### External migration runners -### `prune_controls` - -The patch creates `awa.runtime_controls` and `awa.runtime_control_observations` as ordinary tables, plus functions. The observation table has `instance_id uuid PRIMARY KEY` and `observation jsonb NOT NULL`, with no foreign key. It does not alter `awa.runtime_instances` or any other existing table, refresh the `v023` substrate installer or claim functions, or change storage routing. - -The observation JSON includes protocol version `1`, report sequence and server `received_at`; publishing keeps the latest sequence for each instance. Re-running the patch preserves the current halt and explicit operator grants. An interrupted migration can be run again. - -For an external migration runner, apply the new `R__claim_batch_job_index.sql` again, then `R__prune_controls.sql`, after `V40` and the existing `R__insert_jobs_compat_batch.sql` patch. Extract the full repeatable set with: +`awa migrate --sql --pending` prints `R__claim_batch_job_index`, `R__insert_jobs_compat_batch` and `R__prune_controls` on a 0.6.15 database. Apply them as repeatable scripts after `V40`, in that order, as the schema owner. Write the full repeatable set with: ```bash awa --database-url "$AWA_MIGRATOR_DATABASE_URL" migrate --extract-to ./sql/awa --from 40 ``` -Replace all `R__` files together and keep previously applied `V` files unchanged. `--pending` lists only patches the connected database lacks. +Replace all `R__` files together with that set and keep previously applied `V` files unchanged. `--pending` lists only the patches the connected database lacks, so its output is not the full set. -## Unique keys shared with per-row producers +## Batch inserts and per-row producers -The schema stays at v40. `awa migrate` replaces the body of `awa.insert_jobs_compat_batch`, which `insert_many_copy` and `insert_many_skip_unique_conflicts` call. A 0.6.15 database reports the `insert_jobs_compat_batch` patch as pending until it does. The patch is still one `CREATE OR REPLACE FUNCTION` with no table DDL and no table locks, and the function's signature, privileges and results are unchanged. +`insert_many_copy` and `insert_many_skip_unique_conflicts` insert through `awa.insert_jobs_compat_batch`. `awa migrate` replaces that function's body, which a 0.6.15 database reports as the pending `insert_jobs_compat_batch` patch. The patch is one `CREATE OR REPLACE FUNCTION` with no table DDL, and the function's signature, privileges and results are unchanged. The function takes a batch's unique claims before its lane locks, while per-row inserts (generic `insert_many`, single-job inserts, and 0.6.14 and earlier binaries) take each job's claim and its lane together. In 0.6.15, a per-row transaction that held a lane the batch needed and then inserted a key the batch had claimed but not committed waited on the batch while the batch waited on it, and PostgreSQL aborted one of them with `deadlock detected` (SQLSTATE `40P01`). -In 0.6.16 the function takes its claims and lanes in a subtransaction and, while it holds any of them, waits at most 500 ms per lock, or half of its own session's `deadlock_timeout` if that is lower. When a wait would run longer, the batch rolls back what it took and tries again after a short randomised pause. The per-row transaction gets the key or the lane, proceeds and commits, and on its next attempt the batch skips the keys it committed, or fails with a unique violation when it does not skip conflicts, as it would for any key committed before it. The same applies to a transaction that enqueues lanes in separate statements in the opposite order to a batch. Two batches through the function still take claims in key order and lanes in lane order, so they cannot deadlock with each other. +In 0.6.16 the function takes its claims and lanes in a subtransaction. For its first 2 s, each lock wait is limited to what remains of a 500 ms budget from the start of the attempt, or half of its own session's `deadlock_timeout` if that is lower. When a wait would run longer, the batch rolls back every claim and lane it took and tries again after a short randomised pause. The per-row transaction gets the key or the lane, proceeds and commits, and on its next attempt the batch skips the keys that transaction committed, or fails with a unique violation when it does not skip conflicts, as it would for any key committed before it. The same applies to a transaction that enqueues lanes in separate statements in the opposite order to a batch. Two batches through the function still take claims in key order and lanes in lane order, so they cannot deadlock with each other. -After 2 s of retries, the batch makes one last attempt that waits as the 0.6.15 function did. Contention that outlasts the retries is then served in lock order, and a cycle that rolling back cannot break ends in `deadlock detected` rather than endless retries. The protection therefore lasts only for the bounded phase: a per-row transaction that holds a lane for longer than about 2.5 s and only then inserts one of the batch's keys meets the ordinary attempt, and one of the two is aborted with `40P01` as in 0.6.15. +After 2 s of retries the batch makes one last attempt that waits as the 0.6.15 function did. Contention that outlasts the retries is then served in lock order, and a cycle that rolling back cannot break ends in `deadlock detected` rather than endless retries. What callers see: -- A batch insert and a per-row producer that share keys or lanes no longer abort each other with `40P01` when the batch call is the first thing its transaction locks and the per-row producer's `deadlock_timeout` is above 500 ms (PostgreSQL's default is 1 s). The batch cannot see another session's setting, so the cap is fixed; a per-row session configured at or below 500 ms can still be aborted by its own, earlier deadlock check. -- A per-row transaction that holds a lane longer than the bounded phase (about 2.5 s) before it inserts one of the batch's keys can still deadlock with the batch, as in 0.6.15. -- A batch whose transaction already holds locks from earlier statements can still be part of a deadlock: rolling back the batch cannot release them. That cycle now ends later than in 0.6.15, after about 3 to 4 s instead of 1 to 2 s, and the transaction usually aborted is the batch's caller rather than the per-row producer. -- A batch can only give way while inserting its unique claims takes less than the 500 ms budget. Each claim costs roughly 20 µs on a fast database server and more on a slow or busy one, so the limit is in the range of 10,000 to 20,000 unique keys. A larger batch, or one whose claims are slowed by load, uses up each bounded attempt before it reaches its lanes and ends in the ordinary attempt, so it behaves as in 0.6.15. Split such batches if they share keys or lanes with per-row producers. -- Under contention that outlasts the retries, a batch waits for its lanes in lock order, as in 0.6.15. +- A batch insert and a per-row producer that share keys or lanes no longer abort each other with `40P01`, provided the batch call is the first thing its transaction locks and the per-row session's `deadlock_timeout` is above 500 ms (PostgreSQL's default is 1 s). The batch cannot see another session's setting, so its cap is fixed; a per-row session configured at or below 500 ms can still be aborted by its own, earlier deadlock check. +- A batch can only give way while inserting its unique claims fits the 500 ms budget. Each claim costs roughly 20 µs on a fast database server and more on a slow or busy one, so the limit is in the range of 10,000 to 20,000 unique keys. A larger batch, or one whose claims are slowed by load, uses up each bounded attempt before it reaches its lanes and ends in the ordinary attempt, so it behaves as in 0.6.15. Split such batches if they share keys or lanes with per-row producers. +- A per-row transaction that holds a lane for longer than the bounded phase (about 2.5 s) before it inserts one of the batch's keys meets the ordinary attempt, and one of the two is aborted with `40P01` as in 0.6.15. Cycles of three or more transactions can also behave as in 0.6.15. +- A batch whose transaction already holds locks from earlier statements can still be part of a deadlock, because rolling back the batch cannot release them. That cycle now ends after about 3 to 4 s instead of 1 to 2 s, and the transaction usually aborted is the batch's caller rather than the per-row producer. - A caller's `lock_timeout` applies to each of the batch's lock waits, as it does to any statement's, and a wait it ends fails the call with `lock_not_available` (SQLSTATE `55P03`). Time the batch spends on its own work or on retries does not count against it. - The 500 ms budget applies to each wait, not to a statement: a claim statement that meets several keys still in flight, or a lane reservation that waits for the lane's provisioning lock and then its lock, can wait up to the remaining budget each time. -- The lane lock is still held only for the lane's reservation, the ready-row writes and the commit. +- Each lane's enqueue lock is still taken when the batch reaches that lane and held until the calling transaction ends. +- Each call leaves one committed subtransaction in its caller's transaction, so a transaction that makes more than 64 batch calls overflows PostgreSQL's per-backend subtransaction cache. Commit between batches. + +## New queue lanes + +When `QueueStorage` is the first to use a `(queue, priority)` lane, through a batch enqueue or a worker promoting scheduled or retried jobs, 0.6.16 writes the lane's `queue_lanes` row before taking its provisioning lock, the order SQL inserts already use. 0.6.15 took them the other way round, so a 0.6.15 `QueueStorage` process and any other producer that were both first to use the same lane could deadlock, and one transaction failed with `40P01`. Lanes that already exist are unaffected. This part needs no migration. + +Until every `QueueStorage` process runs 0.6.16, the deadlock can still happen between a 0.6.15 one and another producer. To avoid it during the roll, create the lanes of a new queue or priority before traffic first reaches it with the 0.6.16 `awa storage prepare-queue`, or retry the failed transaction. -## Halt, inspect and resume +## Prune halt -Only `maintenance.prune` with mode `drain` is supported. It stops compatible runtimes from admitting reclamation and housekeeping, including descriptor publication; work admitted before observation may finish. Enqueue, dispatch, handlers, cron, promotion, rescue, admin batches and queue health gauges continue. Fresh storage initialization, auto-finalization and storage selection proceed even during halt or an initial control-read failure. +The `prune_controls` patch creates `awa.runtime_controls` and `awa.runtime_control_observations` as ordinary tables, plus the halt and resume functions. The observation table has `instance_id uuid PRIMARY KEY` and `observation jsonb NOT NULL`, with no foreign key. The patch does not alter `awa.runtime_instances` or any other existing table, refresh the `v023` substrate installer or claim functions, or change storage routing. Re-running it preserves the current halt and explicit operator grants. + +0.6.15 binaries ignore the control, even while a halt is stored, and can still become maintenance leader and prune. Upgrade or fence every process that can lead before relying on a halt. + +### Halt, inspect and resume + +Only `maintenance.prune` with mode `drain` is supported. It stops compatible runtimes from admitting rotation, prune, terminal count rollup, cleanup, admin metadata refresh and descriptor publication; work admitted before the halt is observed may finish. Enqueue, dispatch, handlers, cron, promotion, rescue, admin batches and queue health gauges continue. Fresh storage initialization, auto-finalization and storage selection proceed even during a halt or an initial control-read failure. ```bash awa --database-url "$AWA_OPERATOR_DATABASE_URL" control halt \ @@ -86,14 +84,33 @@ SELECT * FROM awa.halt_runtime_scope('maintenance.prune', 'Investigating lock pr SELECT * FROM awa.resume_runtime_scope('maintenance.prune', ''::uuid); ``` -Status is asynchronous, best-effort evidence from runtime registrations and reports. It uses the database clock, the standard runtime liveness cutoff of `max(3 × snapshot_interval, 30 seconds)`, and a report recency window of `max(2 × snapshot_interval, 5 seconds)` (twenty seconds with default ten-second snapshots). Control permission still expires after five seconds. `unobserving_runtimes` identifies live runtimes with missing, unsupported, wrong-protocol or stale reports, or permission already stale at report creation, including their identity, version and leader flag. Without a halt request, any such runtime makes status `degraded`, including runtimes that keep prune admission closed because control reads fail. An active halt remains `unconfirmed` while any such runtime is live, or if none are live. A report for an earlier halt remains `pending`, and unknown settlement cannot produce `halted`. +The maintenance leader reads the control on its existing advisory-lock session every 0.8 to 1.2 seconds, and its permission to prune expires five seconds after its last successful read. + +### Status + +Status is asynchronous, best-effort evidence from runtime registrations and reports. It uses the database clock, the standard runtime liveness cutoff of `max(3 × snapshot_interval, 30 seconds)`, and a report recency window of `max(2 × snapshot_interval, 5 seconds)` (twenty seconds with the default ten-second snapshots). `unobserving_runtimes` lists live runtimes with missing, unsupported, wrong-protocol or stale reports, or permission already stale when the report was created, with their identity, version and leader flag. Without a halt request, any such runtime makes status `degraded`, including runtimes that keep prune admission closed because control reads fail. An active halt remains `unconfirmed` while any such runtime is live, or if none are live. A report for an earlier halt leaves the request `pending`, and unknown settlement cannot produce `halted`. + +Potentially unsettled reclamation or admin metadata refresh sets `settlement_unknown` for thirty seconds after the latest uncertainty event, and `settlement_uncertainty_remaining_ms` reports the remaining window; the cumulative `abandoned_callbacks` counter can stay nonzero afterwards. Status can return `halted` once that window expires without any backend reconciliation. Standby control reads and report writes use the shared pool with separate one-second budgets, so pool pressure can also make status `degraded` or `unconfirmed`. + +A `halted` status is not proof that PostgreSQL released every lock or that stale hosts are fenced. Runtime credentials can forge observations or delete runtime registrations, so confirmation also depends on trusting those credentials; the operator boundary protects the requested halt, not the reported proof. Inspect backend activity and cancel or fence lingering sessions if lock relief is required. See [runtime controls](runtime-controls.md#read-status) for the status fields and states and [runtime control privileges](security.md#runtime-control-privileges) for the trust boundary. + +## Claim batch index + +0.6.16 replaces the 0.6.15 `.ensure_claim_batch_job_index(slot)` in each queue-storage schema, so `awa migrate` applies the `claim_batch_job_index` patch again. + +- The 0.6.15 function counted a partial GIN index on `job_ids`, or any index already holding the index's name, as the claim batch index, so claim prune could skip building the real one. The new function accepts only a valid, non-partial GIN index on `job_ids` alone. When something else already has the name `lease_claim_batches__job_ids_idx`, it raises a warning naming it and leaves it in place, and that slot keeps scanning until you drop or rename it; claim prune builds the index the next time it empties the slot. +- A 0.6.15 refresh run by a migration role that owns the function but may only `SET ROLE` to the table owner (PostgreSQL 16+ membership `WITH INHERIT FALSE`) left `EXECUTE` with `PUBLIC`. Re-applying the patch resets `EXECUTE` to exactly the roles holding `TRUNCATE` on `lease_claim_batches`; see [Claim batch index](security.md#claim-batch-index). Migrations run with the schema owner as the effective role, as above, were not affected. + +As in 0.6.15 the patch replaces functions only: it builds no index and takes no lock on any `awa` or queue-storage table. 0.6.15 binaries call the new function unchanged. -Potentially unsettled reclamation or admin metadata refresh sets `settlement_unknown` for thirty seconds after the latest uncertainty event. `settlement_uncertainty_remaining_ms` reports the remaining window; the cumulative `abandoned_callbacks` counter can stay nonzero after expiry. Status can return `halted` after expiry without backend reconciliation; expiry does not establish PostgreSQL backend settlement or lock release. Standby control reads and report writes use the shared pool with separate one-second budgets, so pool pressure can also make status `degraded` or `unconfirmed`. +## Job reads during prune -A `halted` status is not proof that PostgreSQL released every lock or that stale hosts are fenced. Runtime credentials can forge observations or delete runtime registrations, so confirmation also depends on trusting those credentials. The operator boundary protects the requested halt, not the reported proof. Inspect backend activity and cancel or fence lingering sessions if lock relief is required. See [runtime controls](runtime-controls.md#read-status) for the status fields and states and [runtime control privileges](security.md#runtime-control-privileges) for the trust boundary. +`admin::get_job` and the other queue-storage point reads of one job read again in a new snapshot when the first read finds nothing. In 0.6.14 and 0.6.15 a read whose snapshot predated a queue prune could miss a failed job while prune moved its row out of the slot being pruned, and return `JobNotFound`. This part needs no migration and applies to each process once it runs 0.6.16. ## Roll back -Before rolling back to 0.6.15, fence processes that could prune or explicitly accept losing the halt guarantee: 0.6.15 ignores the control, even when a halt remains stored. Leave the patches installed; the `prune_controls` tables and functions are safe for older binaries, and replay does not resume the fleet. 0.6.15 binaries call the new `ensure_claim_batch_job_index` unchanged, and a 0.6.15 `awa migrate` leaves it in place. +Roll binaries back to 0.6.15 at any time and leave the patches installed: -For the batch insert fix, roll binaries back to 0.6.15 at any time and leave the patch in place: 0.6.15 binaries call the same function with the same arguments. Rolling the function back means applying the 0.6.15 `R__insert_jobs_compat_batch.sql`, which brings the deadlock back. +- 0.6.15 binaries call the new `insert_jobs_compat_batch` and `ensure_claim_batch_job_index` with the same arguments, and a 0.6.15 `awa migrate` treats both as applied, so the batch insert and claim index fixes stay in place. Restoring the 0.6.15 batch insert body means applying the 0.6.15 `R__insert_jobs_compat_batch.sql`, which brings the deadlock back. +- The `prune_controls` tables and functions are safe for older binaries, and replaying the patch does not resume the fleet. 0.6.15 ignores the control, even while a halt is stored, so fence every process that could prune or accept losing the halt guarantee before rolling back a potential leader. +- A 0.6.15 `QueueStorage` process can again deadlock with another producer when both are first to use a new lane; see [new queue lanes](#new-queue-lanes). From 86121e9f279dfd48df7966a579c6683e857e15f8 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:22:58 +1300 Subject: [PATCH 04/10] docs: cover the bounded rescue drain in the 0.6.16 upgrade guide The rescue section now comes first among the changes, says when rescued jobs become due (2^attempt seconds plus jitter, so at about the drain rate) and that slow Rescued hooks drop events past 1,024 outstanding. The upgrade steps and roll back section mention the leader's rescue rate, the lifecycle hooks page notes the drain, the README row lists the drain, and the configuration table describes zero for either pass setting as the code does: one batch per rescue interval with no early passes. --- README.md | 2 +- docs/configuration.md | 4 ++-- docs/lifecycle-hooks.md | 2 +- docs/upgrade-0.6.15-to-0.6.16.md | 32 +++++++++++++++++--------------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 5b3c71ba3..d8d637d62 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ All coordination through Postgres. The Rust runtime owns dispatch, leases, heart | [0.5 → 0.6 upgrade](docs/upgrade-0.5-to-0.6.md) | Step-by-step operator checklist for the staged storage transition | | [0.6.9 → 0.6.10 upgrade](docs/upgrade-0.6.9-to-0.6.10.md) | Rolling patch upgrade, rollback, and optional runtime privilege tightening | | [0.6.14 → 0.6.15 upgrade](docs/upgrade-0.6.14-to-0.6.15.md) | Rolling patch upgrade: bounded worker finalization, the claim batch index and the batch compatibility insert | -| [0.6.15 → 0.6.16 upgrade](docs/upgrade-0.6.15-to-0.6.16.md) | Rolling patch upgrade: batch compatibility inserts that give way to per-row producers, new-lane provisioning order, the operator prune halt and its grants, and claim batch index fixes | +| [0.6.15 → 0.6.16 upgrade](docs/upgrade-0.6.15-to-0.6.16.md) | Rolling patch upgrade: bounded rescue drains of abandoned claims, batch compatibility inserts that give way to per-row producers, new-lane provisioning order, the operator prune halt and its grants, and claim batch index fixes | | [Configuration reference](docs/configuration.md) | `QueueConfig`, `ClientBuilder`, Python `start()`, env vars | | [HTTP workers and callbacks](docs/http-callbacks.md) | Serverless dispatch, callback receiver endpoints, signature verification | | [Lifecycle hooks](docs/lifecycle-hooks.md) | React to job start/finish/retry/callback events for metrics, alerting, cleanup | diff --git a/docs/configuration.md b/docs/configuration.md index b6e8938a6..364513d5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -482,8 +482,8 @@ Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pa | Knob | Default | What it does | | --- | --- | --- | -| `ClientBuilder::rescue_pass_budget` (Rust) / `rescue_pass_budget_ms` (Python kwarg) | `1s` | How long one pass keeps starting batches. Capped at half the rescue interval. `Duration::ZERO` / `0` keeps every pass to one batch. | -| `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic), with up to one final batch beyond the cap. `0` keeps one batch per ordinary interval without early re-arms. | +| `ClientBuilder::rescue_pass_budget` (Rust) / `rescue_pass_budget_ms` (Python kwarg) | `1s` | How long one pass keeps starting batches. Capped at half the rescue interval. `Duration::ZERO` / `0` keeps one batch per rescue interval with no early passes, as in 0.6.15. | +| `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic), with up to one final batch beyond the cap. `0` keeps one batch per rescue interval with no early passes, as in 0.6.15. | In practice the database sets the rate, not the cap. On PostgreSQL 17, a batch of 500 row-local claims (claims with a deadline) spends about 0.1–0.2 s in its rescue statements and writes about 0.3 MB of WAL for its closures, so a default pass takes several batches before its second of rest. Rescue kinds run one after another on the leader, not in parallel. Measured with default intervals, 1.6 million abandoned row-local claims past their deadlines drained in about 17 minutes (1,600 a second), against about 33 a second (500 per kind per interval) without passes. Raising `rescue_pass_budget` towards its cap and `rescue_pass_max_rows` keeps rescue running for more of each second, but each pass holds the leader for its whole budget, so promotion, rotation and prune wait as long. After a large drain the rings take a minute or two to move past the slots the backlog pinned, while promotion admits the rescued retries. diff --git a/docs/lifecycle-hooks.md b/docs/lifecycle-hooks.md index 46ab87e66..b678d9493 100644 --- a/docs/lifecycle-hooks.md +++ b/docs/lifecycle-hooks.md @@ -77,7 +77,7 @@ Hooks are **best-effort, in-process notifications** — not a durable workflow m - They observe **committed** state. An outcome hook carries the post-transition `JobRow`; a stale/rejected finalization emits no outcome event (though a `Started` may already have fired for that attempt). - They do not delay a job's state transition, and `shutdown()` does not wait for them. They can be lost on process crash or shutdown. - Each attempt's hooks hold one of the worker's finalizer slots until they return, the same bounded slots that wait for completions to commit. Hooks slower than job throughput therefore slow claiming instead of piling up in memory. Keep hooks fast, and hand slow work to a job or follow-up instead. -- Maintenance dispatches `Rescued` events, and `Cancelled` for duplicates it supersedes, on detached tasks with at most 1,024 outstanding per worker. Events beyond that are dropped with a warning rather than holding up rescue. +- Maintenance dispatches `Rescued` events, and `Cancelled` for duplicates it supersedes, on detached tasks with at most 1,024 outstanding per worker. Events beyond that are dropped with a warning rather than holding up rescue. A queue-storage rescue pass can rescue up to `rescue_pass_max_rows` attempts (5,000 by default) while it [drains a backlog](configuration.md#rescue-backlog), so `Rescued` hooks slower than that lose events during a drain. - **Dispatch is not atomic with the state transition.** The hook runs _after_ the job's transition commits, as a separate in-process step — it is not a transactional outbox. If the process dies between the commit and the dispatch, the job state is durable but the event is lost. The same applies to callback resolution: `complete_external` / `resolve_callback` commit the outcome first, then dispatch. - `Started` runs alongside the handler, so a very fast job's state transition may commit before its `Started` hook completes. For outcomes the worker finalizes itself, the outcome hook runs only after the `Started` hook has returned. - A panicking hook is logged; later hooks for the same kind still run. diff --git a/docs/upgrade-0.6.15-to-0.6.16.md b/docs/upgrade-0.6.15-to-0.6.16.md index 7d78218c8..741e86773 100644 --- a/docs/upgrade-0.6.15-to-0.6.16.md +++ b/docs/upgrade-0.6.15-to-0.6.16.md @@ -1,6 +1,6 @@ # Upgrading from 0.6.15 to 0.6.16 -0.6.16 is a rolling patch upgrade: the schema stays at v40 and traffic keeps running throughout. Batch inserts give way to per-row producers instead of deadlocking with them, `QueueStorage` creates new queue lanes in the same lock order as every other producer, operators can halt and resume `maintenance.prune` across the fleet, and point reads of a job no longer miss it while prune moves it. +0.6.16 is a rolling patch upgrade: the schema stays at v40 and traffic keeps running throughout. Rescue drains a large backlog of abandoned claims in minutes instead of hours, batch inserts give way to per-row producers instead of deadlocking with them, `QueueStorage` creates new queue lanes in the same lock order as every other producer, operators can halt and resume `maintenance.prune` across the fleet, and point reads of a job no longer miss it while prune moves it. Two parts of the upgrade need ordering. The new prune control needs grants before 0.6.16 runtimes can read it, and a halt only holds once no 0.6.15 process can become maintenance leader. @@ -17,7 +17,7 @@ Two parts of the upgrade need ordering. The new prune control needs grants befor On a 0.6.15 database the output reports `Schema patch claim_batch_job_index: applied`, `Schema patch insert_jobs_compat_batch: applied` and `Schema patch prune_controls: applied`. Each patch commits in its own transaction and none takes a lock on an `awa` or queue-storage table; an interrupted run can be run again to apply the rest. The batch insert fix is in the function, so 0.6.15 producers get it as soon as its patch commits. 3. Apply the [runtime and operator grants](runtime-controls.md#install). The patch does not grant runtime or operator access. Runtime roles need `SELECT` on `awa.runtime_controls` and `SELECT, INSERT, UPDATE, DELETE` on `awa.runtime_control_observations`. Operators need `SELECT` on both tables and on `awa.runtime_instances`, plus explicit `EXECUTE` on `awa.halt_runtime_scope(text, text, text)` and `awa.resume_runtime_scope(text, uuid)`. Neither role should be able to write the control table directly: apply these exceptions after any blanket grants, and remove inherited mutation privileges from the granting role. -4. Roll workers, producers and admin processes to 0.6.16, in any order. Until every `QueueStorage` process runs 0.6.16, see [new queue lanes](#new-queue-lanes). +4. Roll workers, producers and admin processes to 0.6.16, in any order. Rescue drains faster as soon as the maintenance leader runs 0.6.16; on a fleet with abandoned claims, expect [a burst of retries](#rescue-drains-abandoned-claims-in-passes). Until every `QueueStorage` process runs 0.6.16, see [new queue lanes](#new-queue-lanes). 5. Upgrade or fence every remaining process that can become maintenance leader, standbys included. Then run `awa control status` on the operator connection: before relying on a halt, every live runtime must have a supported, recent report. Binaries can also go before the migration. A 0.6.16 runtime on a database without `prune_controls` keeps the existing reclamation policy and discovers the patch without a restart. Once the patch is installed, a runtime that cannot read the control stops admitting prune within five seconds, and a newly started one does not admit it at all, until the grants are in place; other maintenance continues. If 0.6.16 runtimes are already running, expect prune to pause between steps 2 and 3, and apply the grants before expecting it to resume. @@ -32,6 +32,20 @@ awa --database-url "$AWA_MIGRATOR_DATABASE_URL" migrate --extract-to ./sql/awa - Replace all `R__` files together with that set and keep previously applied `V` files unchanged. `--pending` lists only the patches the connected database lacks, so its output is not the full set. +## Rescue drains abandoned claims in passes + +This part needs no migration and applies once the maintenance leader runs 0.6.16. Queue-storage rescue no longer stops at 500 attempts per kind per interval. Each pass takes 500-attempt batches, each in its own short transaction, for up to `rescue_pass_budget` (default 1 s, at most half the rescue interval) or `rescue_pass_max_rows` (default 5,000) rescued attempts, and while a backlog remains the next pass runs a second later. A runtime killed while holding claims therefore stops pinning the queue and claim rings for hours: with default settings, 1.6 million abandoned row-local claims drained in about 17 minutes, where 0.6.15 needed about 13 hours. + +What changes on a fleet with a backlog: + +- Claims past their deadline drain from the first deadline pass after the leader starts. Stale-heartbeat passes, the only route for claims without a deadline (`deadline_duration` zero), take more than one batch only once runtime snapshots have shown the dead runtime gone for a full staleness window, about 3–3.5 minutes with default settings. +- Every rescued attempt counts against `max_attempts`. Its job returns to `retryable` with the ordinary backoff, 2^attempt seconds plus up to 25% (about 2 s for a first attempt), so rescued jobs become due a few seconds after rescue, at about the drain rate, and the workers that run them see the backlog arrive within minutes rather than over hours. `rescue_pass_max_rows` does not cap how fast promotion makes them available. On DLQ-enabled queues, exhausted attempts move to the DLQ just as quickly. +- `Rescued` lifecycle hooks run on detached tasks with at most 1,024 outstanding per worker. Hooks slower than the drain have further events dropped, with a warning, so keep them fast. +- Closures, retry rows and their WAL arrive at the drain rate, about 0.6 KB of closure WAL per claim plus the retry rows, which replicas and CDC consumers see as a burst. +- Each pass holds the maintenance leader for up to its budget plus one batch, so promotion, rotation and prune run between passes. + +`rescue_pass_budget` and `rescue_pass_max_rows` (Python `rescue_pass_budget_ms` and `rescue_pass_max_rows`) tune the drain; zero for either keeps the 0.6.15 cadence of one batch per rescue interval. Watch it with `awa.maintenance.rescue.pass_size` and `awa.maintenance.rescue.oldest_attempt_age`. See [draining a backlog of abandoned claims](configuration.md#rescue-backlog). + ## Batch inserts and per-row producers `insert_many_copy` and `insert_many_skip_unique_conflicts` insert through `awa.insert_jobs_compat_batch`. `awa migrate` replaces that function's body, which a 0.6.15 database reports as the pending `insert_jobs_compat_batch` patch. The patch is one `CREATE OR REPLACE FUNCTION` with no table DDL, and the function's signature, privileges and results are unchanged. @@ -107,23 +121,11 @@ As in 0.6.15 the patch replaces functions only: it builds no index and takes no `admin::get_job` and the other queue-storage point reads of one job read again in a new snapshot when the first read finds nothing. In 0.6.14 and 0.6.15 a read whose snapshot predated a queue prune could miss a failed job while prune moved its row out of the slot being pruned, and return `JobNotFound`. This part needs no migration and applies to each process once it runs 0.6.16. -## Rescue drains abandoned claims in passes - -This part needs no migration and applies once the maintenance leader runs 0.6.16. Queue-storage rescue no longer stops at 500 attempts per kind per interval. Each pass takes 500-attempt batches, each in its own short transaction, for up to `rescue_pass_budget` (default 1 s, at most half the rescue interval) or `rescue_pass_max_rows` (default 5,000) rescued attempts, and while a backlog remains the next pass runs a second later. A runtime killed while holding claims therefore stops pinning the queue and claim rings for hours: with default settings, 1.6 million abandoned row-local claims drained in about 17 minutes, where 0.6.15 needed about 13 hours. - -What changes on a fleet with a backlog: - -- Claims past their deadline drain from the first deadline pass after the leader starts. Stale-heartbeat passes, the only route for claims without a deadline (`deadline_duration` zero), take more than one batch only once runtime snapshots have shown the dead runtime gone for a full staleness window, about 3–3.5 minutes with default settings. -- Rescued jobs return to `retryable` with their ordinary backoff, so they become due in bursts and the workers that run them see the backlog arrive within minutes rather than over hours. On DLQ-enabled queues, exhausted attempts move to the DLQ just as quickly. -- Closures, retry rows and their WAL arrive at the drain rate, about 0.6 KB of closure WAL per claim plus the retry rows, which replicas and CDC consumers see as a burst. -- Each pass holds the maintenance leader for up to its budget plus one batch, so promotion, rotation and prune run between passes. - -`rescue_pass_budget` and `rescue_pass_max_rows` (Python `rescue_pass_budget_ms` and `rescue_pass_max_rows`) tune the drain; zero for either keeps the 0.6.15 cadence of one batch per interval. Watch it with `awa.maintenance.rescue.pass_size` and `awa.maintenance.rescue.oldest_attempt_age`. See [draining a backlog of abandoned claims](configuration.md#rescue-backlog). - ## Roll back Roll binaries back to 0.6.15 at any time and leave the patches installed: - 0.6.15 binaries call the new `insert_jobs_compat_batch` and `ensure_claim_batch_job_index` with the same arguments, and a 0.6.15 `awa migrate` treats both as applied, so the batch insert and claim index fixes stay in place. Restoring the 0.6.15 batch insert body means applying the 0.6.15 `R__insert_jobs_compat_batch.sql`, which brings the deadlock back. - The `prune_controls` tables and functions are safe for older binaries, and replaying the patch does not resume the fleet. 0.6.15 ignores the control, even while a halt is stored, so fence every process that could prune or accept losing the halt guarantee before rolling back a potential leader. +- Rolling the maintenance leader back restores the 0.6.15 rescue rate of one batch per kind per interval. - A 0.6.15 `QueueStorage` process can again deadlock with another producer when both are first to use a new lane; see [new queue lanes](#new-queue-lanes). From e2e99502de1b3684631128675a1881d024164cf4 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:28:25 +1300 Subject: [PATCH 05/10] docs: describe rescue_pass_max_rows as a stopping threshold in the hooks guide --- docs/lifecycle-hooks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lifecycle-hooks.md b/docs/lifecycle-hooks.md index b678d9493..355174b8f 100644 --- a/docs/lifecycle-hooks.md +++ b/docs/lifecycle-hooks.md @@ -77,7 +77,7 @@ Hooks are **best-effort, in-process notifications** — not a durable workflow m - They observe **committed** state. An outcome hook carries the post-transition `JobRow`; a stale/rejected finalization emits no outcome event (though a `Started` may already have fired for that attempt). - They do not delay a job's state transition, and `shutdown()` does not wait for them. They can be lost on process crash or shutdown. - Each attempt's hooks hold one of the worker's finalizer slots until they return, the same bounded slots that wait for completions to commit. Hooks slower than job throughput therefore slow claiming instead of piling up in memory. Keep hooks fast, and hand slow work to a job or follow-up instead. -- Maintenance dispatches `Rescued` events, and `Cancelled` for duplicates it supersedes, on detached tasks with at most 1,024 outstanding per worker. Events beyond that are dropped with a warning rather than holding up rescue. A queue-storage rescue pass can rescue up to `rescue_pass_max_rows` attempts (5,000 by default) while it [drains a backlog](configuration.md#rescue-backlog), so `Rescued` hooks slower than that lose events during a drain. +- Maintenance dispatches `Rescued` events, and `Cancelled` for duplicates it supersedes, on detached tasks with at most 1,024 outstanding per worker. Events beyond that are dropped with a warning rather than holding up rescue. While it [drains a backlog](configuration.md#rescue-backlog), a queue-storage rescue pass keeps rescuing until it reaches `rescue_pass_max_rows` (5,000 by default) and then finishes its current batch, so a pass can exceed that by nearly one batch, and `Rescued` hooks slower than the drain lose events. - **Dispatch is not atomic with the state transition.** The hook runs _after_ the job's transition commits, as a separate in-process step — it is not a transactional outbox. If the process dies between the commit and the dispatch, the job state is durable but the event is lost. The same applies to callback resolution: `complete_external` / `resolve_callback` commit the outcome first, then dispatch. - `Started` runs alongside the handler, so a very fast job's state transition may commit before its `Started` hook completes. For outcomes the worker finalizes itself, the outcome hook runs only after the `Started` hook has returned. - A panicking hook is logged; later hooks for the same kind still run. From aaa3a2e2d4fe755ceeea081bbda8e589e9c287a6 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:46:24 +1300 Subject: [PATCH 06/10] docs: cover 0.6.16 reads, index note and test coverage outside the release notes The troubleshooting note on transient admin reads now says that a read of one job which finds nothing reads once more in a new snapshot, so a failed job that prune moved is no longer reported missing. The claim batch index note in the migrations guide said released 0.6 binaries neither use nor build the index; 0.6.15 does both, so it now names binaries before 0.6.15. The test plan lists the bounded rescue drain, the durable prune halt and the batch insert and new-lane deadlock tests, and the lock-order model now covers new-lane provisioning. --- docs/migrations.md | 2 +- docs/test-plan.md | 5 ++++- docs/troubleshooting.md | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/migrations.md b/docs/migrations.md index df2a3951e..e4d11ffe7 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -247,7 +247,7 @@ That writes one `V__.sql` file per migration, plus one `R__.ensure_claim_batch_job_index(slot)` in every queue-storage schema and builds no index itself, so it takes no lock on any awa or queue-storage table and is safe inside a long transaction under live traffic. Claim prune calls that function right after truncating a claim partition, which builds the GIN index on `lease_claim_batches.job_ids` while the partition is empty, so the index covers the ring within one claim-ring cycle. Only a valid, non-partial GIN index on `job_ids` counts; a different index or relation already named `lease_claim_batches__job_ids_idx` is left in place with a warning naming it, and that slot keeps scanning until it is dropped or renamed. Do not build the index yourself on a populated partition: the build holds a `SHARE` lock, which blocks claim inserts into that partition, for about half a minute per 100k batches. Run the script as the owner of the queue-storage tables. Released 0.6 binaries neither use nor build the index and are unaffected by it, and binaries that use the index read the same rows, by scan, from a schema or partition without it. +`R__claim_batch_job_index.sql` indexes compact claim batches by job id. It installs `.ensure_claim_batch_job_index(slot)` in every queue-storage schema and builds no index itself, so it takes no lock on any awa or queue-storage table and is safe inside a long transaction under live traffic. Claim prune calls that function right after truncating a claim partition, which builds the GIN index on `lease_claim_batches.job_ids` while the partition is empty, so the index covers the ring within one claim-ring cycle. Only a valid, non-partial GIN index on `job_ids` counts; a different index or relation already named `lease_claim_batches__job_ids_idx` is left in place with a warning naming it, and that slot keeps scanning until it is dropped or renamed. Do not build the index yourself on a populated partition: the build holds a `SHARE` lock, which blocks claim inserts into that partition, for about half a minute per 100k batches. Run the script as the owner of the queue-storage tables. Binaries before 0.6.15 neither use nor build the index and are unaffected by it, and binaries that use the index read the same rows, by scan, from a schema or partition without it. `awa migrate` reports the schema version, each patch and what it updated, and storage routing before and after. It exits 1 if a patch is still missing somewhere, such as a queue-storage schema the migrating role cannot use, and names it. `awa::migrations::schema_patch_status` gives the same per-patch view from Rust. diff --git a/docs/test-plan.md b/docs/test-plan.md index dffcb3332..fa65aa835 100644 --- a/docs/test-plan.md +++ b/docs/test-plan.md @@ -14,6 +14,7 @@ Tests run against real Postgres 15+ (not managed services) using a dedicated tes | T4 | `kill -9` crash recovery | ✓ | | | T5 | Deadline rescue (hung job) | ✓ | | | T5a | Poison jobs (handler panic, crash loop, deadline loop) stop at `max_attempts` on both engines | ✓ | | +| T5b | Rescue drains a backlog of abandoned claims in bounded passes without rescuing live claims, and the queue and claim rings then rotate past the pinned slots | ✓ | | | T6 | Transactional atomicity | ✓ | | | T7 | Transactional atomicity (Python) | | ✓ | | T8 | Uniqueness under contention | ✓ | ✓ | @@ -59,6 +60,7 @@ Tests run against real Postgres 15+ (not managed services) using a dedicated tes | # | Test | Rust | Py | | --- | --- | --- | --- | | T13 | Queue pause/resume | ✓ | | +| T13a | Durable `maintenance.prune` halt survives a leader handoff and keeps job accounting exact; resume names the current halt | ✓ | | | T22 | Pool exhaustion resilience | ✓ | | | T26 | Migration idempotency | ✓ | | | T27 | Admin ops under load | ✓ | | @@ -167,6 +169,7 @@ _HTTPWorker is a Rust-only feature (ADR-018: serverless function dispatch). Not | --- | --- | --- | --- | | T47-T57a | COPY: empty, single, 1000, special chars, tags, unique, atomic | ✓ | ✓ | | QSC1-QSC2 | Queue storage COPY producer: ready/deferred direct COPY, unique-conflict rollback | ✓ | ✓ | +| QSC3 | Batch inserts give way to per-row producers that share their unique keys or lanes, and every producer pair creates a new lane without deadlock | ✓ | | ### Rate Limiting & Weighted Concurrency @@ -241,7 +244,7 @@ Concurrent lifecycle benchmark (1 queue × 128 workers, 20K jobs): | TLA9 | AwaShardedPrune | Cross-shard ready/terminal prune matching by `enqueue_shard` | | TLA10 | AwaSegmentedStorageRaces | Claim-vs-rotate/prune race exposure and checked-commit safety | | TLA11 | AwaSegmentedStorageTrace | Runtime trace acceptance for snooze, receipt rescue, cancel, callback wait, DLQ retry, and DLQ purge paths | -| TLA12 | AwaStorageLockOrder | Postgres lock ordering across claim, complete, cancel, rescue, rotate, and prune | +| TLA12 | AwaStorageLockOrder | Postgres lock ordering across enqueue, new-lane provisioning, claim, complete, cancel, rescue, rotate, and prune | | TLA13 | AwaStorageTransition | Storage-transition prepare, mixed-entry, finalize, and abort gates | | TLA14 | AwaDeadTupleContract | Hot-table reclaim-kind and partition-truncate contract | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1ceaf20b7..1d4d71ed3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -88,7 +88,7 @@ Interpretation: ### Transient Admin Reads During Partition Pruning -In 0.6, a job-detail read, including list-item hydration, uses several queries within one snapshot. It retains read locks on the located ready body, but does not lock all ring partitions before establishing that snapshot. Prune can truncate a ready partition before its body is located, or a lease or claim partition before that read phase reaches it. A completed job can then briefly appear available or missing. Retry with a fresh read before treating that result as missing durable work. Filtering tombstoned source rows after a queue or priority move does not close these separate windows. +In 0.6, a job-detail read, including list-item hydration, uses several queries within one snapshot. It retains read locks on the located ready body, but does not lock all ring partitions before establishing that snapshot. Prune can truncate a ready partition before its body is located, or a lease or claim partition before that read phase reaches it. A completed job can then briefly appear available or missing. A read of one job that finds nothing, including each item a list hydrates, reads once more in a new snapshot (from 0.6.16), so a job whose row prune moved to the live slot after the first snapshot was taken is found rather than reported as `JobNotFound`. A job that appears in an earlier state can still be stale; retry with a fresh read before acting on it. Filtering tombstoned source rows after a queue or priority move does not close these separate windows. ### Failed Jobs Rescued After Priority Aging From 282d9b404c5e76d8cd6bd6f55a54681a492a01d9 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:46:31 +1300 Subject: [PATCH 07/10] docs: describe bounded rescue passes and the prune halt in the architecture overview The recovery model now says that queue-storage rescue runs as passes of short 500-attempt batches, bounded by rescue_pass_budget and rescue_pass_max_rows and re-armed a second later while a backlog remains, with stale-heartbeat acceleration gated on runtime evidence. The reclamation section points at the maintenance.prune control: a halted leader stops admitting rotation, prune and housekeeping while enqueue, dispatch, promotion and rescue continue. --- docs/architecture.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 4139bd87b..80317e554 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -200,6 +200,8 @@ Awa has three rescue paths: - **Hard deadline rescue.** Per-queue deadlines write `deadline_at` onto row-local receipt claims or lease rows. The maintenance leader closes expired attempts and routes them through the normal retry/fail/DLQ path. - **Callback-timeout rescue.** Waiting attempts with expired callback timeouts are moved back through the same guarded finalization machinery. +On queue storage each rescue runs as a pass of 500-attempt batches, each in its own short transaction, oldest claim slot first. A pass stops when a batch comes back short, after `rescue_pass_budget` or once it has rescued `rescue_pass_max_rows`, and while a backlog remains the next pass runs a second later, so the open claims a killed runtime leaves behind stop pinning the queue and claim rings within minutes. Stale-heartbeat passes take more than one batch only once runtime snapshots show the claims' owners gone; deadline and callback passes need no such evidence. See [draining a backlog of abandoned claims](configuration.md#rescue-backlog). + Rescue closes the old attempt before making work available again. If the old handler later writes a completion, the `run_lease` guard rejects it as stale. When rescue happens in a process that still has the handler registered, the runtime also flips the in-memory cancellation flag. ## Partition Rotation And Reclamation @@ -214,6 +216,8 @@ Queue storage has three independent rings, each advanced by the elected maintena The maintenance tick for each ring is deliberately small: attempt one rotate, then attempt one prune. If a partition is busy, blocked by a lock, current, or still live, the tick records a skipped/blocked outcome and tries again on a future interval. +Operators can durably halt this reclamation with the `maintenance.prune` control (`awa control halt`). A leader that observes the halt stops admitting rotation, prune, terminal count rollup, cleanup, admin metadata refresh and descriptor publication, while enqueue, dispatch, promotion and rescue continue; the rings stay where they are until `awa control resume`. See [runtime controls](runtime-controls.md). + The common safety pattern is: 1. Lock the ring-state row with `FOR UPDATE`. @@ -292,7 +296,7 @@ Core safety invariants are modeled in TLA+: | [`AwaSegmentedStorageRaces`](../correctness/storage/AwaSegmentedStorageRaces.tla) | claim-vs-rotate/prune interleavings | | [`AwaSegmentedStorageTrace`](../correctness/storage/AwaSegmentedStorageTrace.tla) | concrete runtime trace acceptance for representative queue-storage flows | | [`AwaShardedPrune`](../correctness/storage/AwaShardedPrune.tla) | cross-shard ready/terminal prune matching by `enqueue_shard` | -| [`AwaStorageLockOrder`](../correctness/storage/AwaStorageLockOrder.tla) | Postgres lock ordering across claim, rotate, and prune | +| [`AwaStorageLockOrder`](../correctness/storage/AwaStorageLockOrder.tla) | Postgres lock ordering across enqueue, new-lane provisioning, claim, rescue, rotate, and prune | | [`AwaStorageTransition`](../correctness/storage/AwaStorageTransition.tla) | queue-storage transition prepare, mixed-entry, finalize, and abort gates | | [`AwaDeadTupleContract`](../correctness/storage/AwaDeadTupleContract.tla) | hot-table reclaim-kind contract for partition truncate and bounded warm tables | | [`AwaCbk`](../correctness/races/AwaCbk.tla) | callback registration/resume/finalization races | From 7fcda29d7411e9d868d02779302d5ecaf2db150e Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:46:31 +1300 Subject: [PATCH 08/10] fix(cli): describe the awa control halt and resume arguments in --help --scope, --mode, --reason and --halt had no help text. They now say which values are supported, that the reason is stored and shown by status, and where to find the halt UUID. --- awa-cli/src/control.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/awa-cli/src/control.rs b/awa-cli/src/control.rs index 3d94800ac..98e7cde6a 100644 --- a/awa-cli/src/control.rs +++ b/awa-cli/src/control.rs @@ -8,17 +8,22 @@ pub(crate) enum ControlCommands { Status, /// Stop new reclamation and housekeeping; let admitted work drain. Halt { + /// Control to halt; only `maintenance.prune` is supported #[arg(long, default_value = "maintenance.prune")] scope: String, + /// Halt mode; only `drain` is supported #[arg(long, default_value = "drain")] mode: String, + /// Why the halt is requested, stored with it and shown by `status` #[arg(long)] reason: String, }, /// Lift exactly the named current halt, without waiting for drain completion. Resume { + /// Control to resume; only `maintenance.prune` is supported #[arg(long, default_value = "maintenance.prune")] scope: String, + /// UUID of the current halt (`halt_id` in the `halt` or `status` output) #[arg(long)] halt: Uuid, }, From 96a1584042a49be2d56d44458f13ad167188ac69 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 19:46:44 +1300 Subject: [PATCH 09/10] test(correctness): model new-lane provisioning order in the lock-order spec AwaStorageLockOrder collapsed a lane's queue_lanes row and provisioning lock into its enqueue lock, so it could not see the #692 deadlock between a QueueStorage process and a SQL producer that are both first to use a lane. It now models both: ProvisionLanePlan takes the row, then the provisioning lock, then the enqueue lock, as every producer does, and the main config stays deadlock-free (245,414 distinct states). AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg keeps the earlier QueueStorage order and trips NoDeadlock in 5 steps. The TLC suite now runs that witness and the existing old striped-claim witness, which was documented but not run. MAPPING.md points the rescue actions at the batch entry points and records why bounded rescue passes, the stale-heartbeat owner-evidence gate, try-lock heartbeats and the prune halt leave every modelled invariant unchanged, and why batch insert contention, fresh-snapshot point reads and the claim batch index stay outside the models. --- correctness/README.md | 3 +- correctness/run-tlc-suite.sh | 2 + correctness/storage/AwaStorageLockOrder.tla | 79 +++++++++++++++++++ ...geLockOrderOldLaneProvisioningDeadlock.cfg | 21 +++++ correctness/storage/MAPPING.md | 22 +++++- correctness/storage/README.md | 8 +- 6 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 correctness/storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg diff --git a/correctness/README.md b/correctness/README.md index a874863ff..6f709fbfd 100644 --- a/correctness/README.md +++ b/correctness/README.md @@ -48,7 +48,7 @@ What is intentionally not modeled: - `storage/AwaSegmentedStorageRaces.tla` / `storage/AwaSegmentedStorageRaces.cfg` / `storage/AwaSegmentedStorageRacesSafe.cfg`: focused race-exposure spec that splits `Claim` into `BeginClaim` / `CommitClaim` to model the claim path's cursor-read-without-lock behaviour. The race config produces a counterexample trace (claim snapshots segment, rotate+prune fire, commit lands lease in pruned segment — simultaneously the claim-vs-rotate race and the prune check-then-act race). The safe config uses a checked commit and passes. Production uses CAS on ring state plus child-partition locks and busy checks, not `FOR SHARE` on `lease_ring_state` — see `storage/MAPPING.md` for the full analysis. - `storage/AwaShardedPrune.tla` / `storage/AwaShardedPrune.cfg` / `storage/AwaShardedPruneBroken.cfg`: focused ADR-025 regression model for queue-ring prune across enqueue shards. The passing config requires ready rows to match done rows by `(enqueue_shard, lane_seq)` before prune can drop a sealed queue slot. The broken config intentionally ignores `enqueue_shard` and produces the historical counterexample: shard 0 has a done row at `lane_seq = 1`, shard 1 still has a pending ready row at `lane_seq = 1`, and broken prune drops both. - `storage/AwaPartitionedQueueRouting.tla` / `storage/AwaPartitionedQueueRouting.cfg` / `storage/AwaPartitionedQueueRoutingBroken.cfg`: focused ADR-031 routing model for `PartitionedQueue`. Partitioned queues remain client-side routing over ordinary Awa queues, so this spec checks the refinement boundary above storage: abstract key hashes route to their selected partition, the same base hash still determines the enqueue shard inside that partition, lane sequence identity stays scoped to `(partition, shard)`, and the configured hash domain reaches every shard from every partition. The broken config reuses the base hash modulo at both routing levels and demonstrates the correlated-hash defect. -- `storage/AwaStorageLockOrder.tla` / `storage/AwaStorageLockOrder.cfg` / `storage/AwaStorageLockOrderDeadlockDemo.cfg` / `storage/AwaStorageLockOrderOldStripedClaimDeadlock.cfg`: lock-ordering protocol spec. Models each storage-engine transaction (enqueue, claim, complete, close-receipt, rescue-receipts, ensure-running, cancel, rotate, and prune) as an ordered sequence of Postgres lock acquisitions with a shared/exclusive compatibility matrix. The striped enqueue path takes multiple physical queue lanes in stable order; the current striped claim path takes at most one physical stripe per transaction. Checks `NoDeadlock` via a waits-for cycle detector. The demo configs use a deliberately cycle-creating plan pair and the historical old striped logical-claim plan to prove the detector catches real cycles. +- `storage/AwaStorageLockOrder.tla` / `storage/AwaStorageLockOrder.cfg` / `storage/AwaStorageLockOrderDeadlockDemo.cfg` / `storage/AwaStorageLockOrderOldStripedClaimDeadlock.cfg` / `storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg`: lock-ordering protocol spec. Models each storage-engine transaction (enqueue, first enqueue to a new lane, claim, complete, close-receipt, rescue-receipts, ensure-running, cancel, rotate, and prune) as an ordered sequence of Postgres lock acquisitions with a shared/exclusive compatibility matrix. The striped enqueue path takes multiple physical queue lanes in stable order; the current striped claim path takes at most one physical stripe per transaction. Checks `NoDeadlock` via a waits-for cycle detector. The first enqueue to a new lane takes its `queue_lanes` row, then its provisioning lock, then its enqueue lock, in every producer. The demo configs use a deliberately cycle-creating plan pair, the historical old striped logical-claim plan, and the historical `QueueStorage` lane-provisioning order (provisioning lock before the row) to prove the detector catches real cycles. - `storage/AwaCanonicalUniqueRescue.tla` / `storage/AwaCanonicalUniqueRescue.cfg` / `storage/AwaCanonicalUniqueRescueBatchOnly.cfg`: focused model for the canonical rescue sweeps versus the unique-claim trigger. A unique job with a pending-only `unique_states` mask is stuck running while a duplicate holds the claim; the batched rescue is blocked forever (BatchOnly config: expected `Convergence` liveness counterexample — the production wedge fixed in 0.6.1), while the per-row fallback rescues the innocent job and cancels the superseded one (main config passes, `ClaimConsistency` holds in both). - `storage/AwaStorageTransition.tla` / `storage/AwaStorageTransition.cfg` / `storage/AwaStorageTransitionCurrentGate.cfg`: focused model for the 0.5.x-to-0.6 storage transition control plane. It covers prepare, schema readiness, mixed-transition entry, canonical backlog drain, queue-storage routing, finalize, and abort interlocks. The desired config matches the v014 entry gate and v040 finalization gate: a live queue-storage target is required before routing flips, while drained pre-flip auto runtimes may remain idle through finalization. The `CurrentGate`-named config is a historical pre-v014 regression witness for the old capability-only entry gate. - `storage/AwaSegmentedStorageTrace.tla` / `storage/AwaSegmentedStorageTrace.cfg` / `storage/AwaSegmentedStorageTraceReceiptRescue.cfg` / `storage/AwaSegmentedStorageTraceLostClaimAdvance.cfg` / `storage/AwaSegmentedStorageTraceRunningCancel.cfg` / `storage/AwaSegmentedStorageTraceReceiptOnlyCancel.cfg` / `storage/AwaSegmentedStorageTraceCallbackWait.cfg` / `storage/AwaSegmentedStorageTraceDlqRetry.cfg` / `storage/AwaSegmentedStorageTraceDlqPurge.cfg` / `storage/AwaSegmentedStorageTraceBroken.cfg`: trace-validation harness. Takes hand-transcribed sequences of queue-storage runtime events and verifies each transition is accepted by the storage spec. Current positive traces cover snooze, receipt rescue, running cancel, receipt-only cancel, callback wait/resume, DLQ retry, and DLQ purge. A deliberately-broken variant trips deadlock at traceIdx = 2 to confirm the checker rejects invalid sequences. @@ -108,6 +108,7 @@ Expected counterexample or positive-witness configs: ./correctness/run-tlc.sh storage/AwaPartitionedQueueRouting.tla storage/AwaPartitionedQueueRoutingBroken.cfg ./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderDeadlockDemo.cfg ./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderOldStripedClaimDeadlock.cfg +./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg ./correctness/run-tlc.sh storage/AwaStorageTransition.tla storage/AwaStorageTransitionCurrentGate.cfg ./correctness/run-tlc.sh storage/AwaShardedPrune.tla storage/AwaShardedPruneBroken.cfg ``` diff --git a/correctness/run-tlc-suite.sh b/correctness/run-tlc-suite.sh index 05c320b84..007d27b82 100755 --- a/correctness/run-tlc-suite.sh +++ b/correctness/run-tlc-suite.sh @@ -32,6 +32,8 @@ AwaSegmentedStorageTrace.tla ;; AwaSegmentedStorageTraceDlqPurge.cfg ;; Dlq AwaSegmentedStorageTrace.tla ;; AwaSegmentedStorageTraceBroken.cfg ;; Deadlock reached ;; broken trace rejected AwaStorageLockOrder.tla ;; AwaStorageLockOrder.cfg ;; pass ;; lock ordering AwaStorageLockOrder.tla ;; AwaStorageLockOrderDeadlockDemo.cfg ;; NoDeadlock is violated ;; deadlock detector witness +AwaStorageLockOrder.tla ;; AwaStorageLockOrderOldStripedClaimDeadlock.cfg ;; NoDeadlock is violated ;; old striped-claim order witness +AwaStorageLockOrder.tla ;; AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg ;; NoDeadlock is violated ;; old QueueStorage lane-provisioning order witness AwaDeadTupleContract.tla ;; AwaDeadTupleContract.cfg ;; pass ;; architectural reclaim contract AwaStorageTransition.tla ;; - ;; pass ;; transition control plane + #456 reschedule drain AwaQuiescedTransition.tla ;; - ;; pass ;; quiesced routing flip serializes snapshot evidence diff --git a/correctness/storage/AwaStorageLockOrder.tla b/correctness/storage/AwaStorageLockOrder.tla index 02e711327..727d505fd 100644 --- a/correctness/storage/AwaStorageLockOrder.tla +++ b/correctness/storage/AwaStorageLockOrder.tla @@ -111,6 +111,15 @@ Compatible(held, wanted) == held = ModeShared /\ wanted = ModeShared \* deadlock; two transactions on the same lane and shard reduce to \* the existing `LaneResource(q, p)` model. LaneResource(q, p) == [k |-> "queue_lane", q |-> q, p |-> p] +\* Creating a new lane takes two more locks before the enqueue lock. The +\* `queue_lanes` row: an uncommitted insert of a `(queue, priority)` key +\* makes a second inserter of that key wait for its transaction. The +\* provisioning advisory lock `{schema}:{queue}:{priority}:{shard}`, under +\* which the lane's sequences and head rows are created. Once a lane +\* exists neither blocks, so modelling both as held to commit +\* over-approximates every enqueue after the first. +LaneRowResource(q, p) == [k |-> "queue_lane_row", q |-> q, p |-> p] +LaneProvisionResource(q, p) == [k |-> "queue_lane_provision", q |-> q, p |-> p] LeaseRingStateResource == [k |-> "lease_ring_state"] LeaseRingSlotResource(s) == [k |-> "lease_ring_slot", s |-> s] LeaseChildResource(s) == [k |-> "lease_child", s |-> s] @@ -136,6 +145,8 @@ ClosureChildResource(s) == [k |-> "closure_child", s |-> s] ClosureBatchChildResource(s) == [k |-> "closure_batch_child", s |-> s] LaneResources == { LaneResource(q, p) : q \in Queues, p \in Priorities } +LaneRowResources == { LaneRowResource(q, p) : q \in Queues, p \in Priorities } +LaneProvisionResources == { LaneProvisionResource(q, p) : q \in Queues, p \in Priorities } LeaseRingSlotResources == { LeaseRingSlotResource(s) : s \in LeaseSlots } LeaseChildResources == { LeaseChildResource(s) : s \in LeaseSlots } QueueRingSlotResources == { QueueRingSlotResource(s) : s \in ReadySlots } @@ -155,6 +166,8 @@ ClosureBatchChildResources == { ClosureBatchChildResource(s) : s \in ClaimSlots Resources == LaneResources \cup + LaneRowResources \cup + LaneProvisionResources \cup {LeaseRingStateResource} \cup LeaseRingSlotResources \cup LeaseChildResources \cup @@ -211,6 +224,31 @@ EnqueueTwoStripePlan(p) == << Step(LaneResource(StripeA, p), ModeExclusive), Step(LaneResource(StripeB, p), ModeExclusive) >> +\* First enqueue to a `(queue, priority)` lane, by any producer: +\* SQL inserts (`awa.insert_job`, `insert_jobs_compat_batch`): +\* INSERT INTO queue_lanes ... ON CONFLICT DO NOTHING +\* INSERT INTO queue_enqueue_heads, whose BEFORE INSERT trigger takes +\* the provisioning lock to create the lane's sequences +\* reserve_enqueue_seq takes the lane's enqueue lock +\* QueueStorage::ensure_lane_inserts (batch enqueue, promotion of +\* scheduled or retried jobs, prepare_queue): +\* INSERT INTO queue_lanes ... ON CONFLICT DO NOTHING +\* pg_advisory_xact_lock on the provisioning key, then provision_lane +\* or the head-row inserts +\* reserve_enqueue_seq takes the lane's enqueue lock +ProvisionLanePlan(q, p) == + << Step(LaneRowResource(q, p), ModeExclusive), + Step(LaneProvisionResource(q, p), ModeExclusive), + Step(LaneResource(q, p), ModeExclusive) >> + +\* QueueStorage::ensure_lane_inserts through 0.6.15 took the provisioning +\* lock before writing the `queue_lanes` row, the reverse of the SQL +\* inserts. Kept only as a negative regression harness. +OldQueueStorageProvisionLanePlan(q, p) == + << Step(LaneProvisionResource(q, p), ModeExclusive), + Step(LaneRowResource(q, p), ModeExclusive), + Step(LaneResource(q, p), ModeExclusive) >> + \* The claim CTE \* (`claim_runtime_batch_with_aging_for_instance` in queue_storage.rs) \* FOR UPDATE OF queue_claim_heads SKIP LOCKED (per-(queue,priority) row) @@ -581,6 +619,16 @@ StartEnqueueTwoStripe(t, p) == /\ txNextStep' = [txNextStep EXCEPT ![t] = 1] /\ UNCHANGED heldLocks +StartProvisionLane(t, q, p) == + /\ t \in TxIds + /\ txState[t] = "idle" + /\ q \in Queues + /\ p \in Priorities + /\ txState' = [txState EXCEPT ![t] = "running"] + /\ txPlan' = [txPlan EXCEPT ![t] = ProvisionLanePlan(q, p)] + /\ txNextStep' = [txNextStep EXCEPT ![t] = 1] + /\ UNCHANGED heldLocks + StartComplete(t, claimSlot, readySlot) == /\ t \in TxIds /\ txState[t] = "idle" @@ -767,6 +815,7 @@ Stutter == /\ UNCHANGED vars Next == \/ \E t \in TxIds, p \in Priorities : StartEnqueueTwoStripe(t, p) + \/ \E t \in TxIds, q \in Queues, p \in Priorities : StartProvisionLane(t, q, p) \/ \E t \in TxIds, q \in Queues, p \in Priorities, rs \in ReadySlots, cs \in ClaimSlots : StartClaimReceipts(t, q, p, rs, cs) @@ -877,6 +926,36 @@ NextOldStripedClaimDeadlockDemo == SpecOldStripedClaimDeadlockDemo == ConfigOK /\ Init /\ [][NextOldStripedClaimDeadlockDemo]_vars +\* ---- Historical lane-provisioning deadlock harness ---- +\* +\* A 0.6.15 QueueStorage process and a SQL producer that are both first to +\* use a lane: QueueStorage holds the provisioning lock and waits for the +\* SQL producer's uncommitted `queue_lanes` row, while the SQL producer +\* holds that row and waits for the provisioning lock. TLC is expected to +\* flag NoDeadlock on this spec. + +StartOldQueueStorageProvisionLane(t, q, p) == + /\ t \in TxIds + /\ txState[t] = "idle" + /\ q \in Queues + /\ p \in Priorities + /\ txState' = [txState EXCEPT ![t] = "running"] + /\ txPlan' = [txPlan EXCEPT ![t] = OldQueueStorageProvisionLanePlan(q, p)] + /\ txNextStep' = [txNextStep EXCEPT ![t] = 1] + /\ UNCHANGED heldLocks + +NextOldLaneProvisioningDeadlockDemo == + \/ \E t \in TxIds, q \in Queues, p \in Priorities : StartProvisionLane(t, q, p) + \/ \E t \in TxIds, q \in Queues, p \in Priorities : + StartOldQueueStorageProvisionLane(t, q, p) + \/ \E t \in TxIds : AcquireNext(t) + \/ \E t \in TxIds : Commit(t) + \/ \E t \in TxIds : Recycle(t) + \/ Stutter + +SpecOldLaneProvisioningDeadlockDemo == + ConfigOK /\ Init /\ [][NextOldLaneProvisioningDeadlockDemo]_vars + \* ---- Invariants ---- TypeOK == diff --git a/correctness/storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg b/correctness/storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg new file mode 100644 index 000000000..45bd412ef --- /dev/null +++ b/correctness/storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg @@ -0,0 +1,21 @@ +\* Historical lane-provisioning harness. A 0.6.15 QueueStorage process +\* took a new lane's provisioning lock before its queue_lanes row, while +\* SQL inserts write the row first. TLC is expected to flag NoDeadlock +\* when one of each is the first to use the same lane. +CONSTANTS + TxIds = {"t1", "t2"} + Queues = {"q1#0", "q1#1"} + StripeA = "q1#0" + StripeB = "q1#1" + Priorities = {1} + LeaseSlots = {1} + ReadySlots = {1} + ClaimSlots = {1} + +SPECIFICATION SpecOldLaneProvisioningDeadlockDemo + +INVARIANTS + TypeOK + LockCompatibility + HeldOnlyByRunningTxs + NoDeadlock diff --git a/correctness/storage/MAPPING.md b/correctness/storage/MAPPING.md index bd350018f..c132f7e03 100644 --- a/correctness/storage/MAPPING.md +++ b/correctness/storage/MAPPING.md @@ -69,8 +69,8 @@ The TLA+ lifecycle model does not represent the completed-history rollup cache, | `PruneLeaseSegment` | `QueueStorage::prune_oldest_leases` | `FOR UPDATE` on `lease_ring_state` and `lease_ring_slots[slot]`, then `LOCK TABLE ... ACCESS EXCLUSIVE` with a short transaction-local `lock_timeout`, recheck active rows, then `TRUNCATE` the selected `leases_N` child | | `RotateClaimSegments` | maintenance `QueueStorage::rotate_claims`, wired via `Maintenance::rotate_queue_storage_claims` at the `claim_rotate_interval` tick | `FOR UPDATE` on `claim_ring_state`, busy-check the three child partitions, then `UPDATE claim_ring_state SET current_slot = next, generation = next_gen` with compare-and-swap on `(current_slot, generation)` | | `PruneClaimSegment(seg)` | `QueueStorage::prune_oldest_claims` | `FOR UPDATE` on `claim_ring_state`, `FOR UPDATE` on `claim_ring_slots[slot]`, proof that every row-local claim or compact claim-batch item in the sealed slot has durable closure evidence, then `LOCK TABLE` `lease_claims_N`, `lease_claim_batches_N`, `lease_claim_closures_N`, and `lease_claim_closure_batches_N` `IN ACCESS EXCLUSIVE MODE` with a short transaction-local `lock_timeout`, recheck not-current, recheck the open-claim proof, reset that slot's stale-rescue cursor, then `TRUNCATE` the four children. The proof compares exact child-row counts (`lease_claims` rows plus `lease_claim_batches.claimed_count` totals versus explicit closures plus compact `closed_count` totals). If counts do not prove closure, prune returns `SkippedActive` and tries again later instead of running an unbounded per-claim anti-join over retained compact batches. | -| `RescueStaleReceipt(j, r)` | `rescue_stale_receipt_claims_tx`, invoked from maintenance `rescue_stale_heartbeats`. Excludes claims already materialized into `leases` so the lease-side rescue path owns those. The spec takes the explicit `(j, r)` so concurrent rescue / re-claim races are reachable: rescue can fire on an old attempt's `(j, r_old)` receipt while `(j, r_old + 1)` already has an open receipt in a newer partition. | reads `claim_ring_state` to try the oldest initialized claim slot first, then scans remaining slots with the same global rescue budget; each slot uses a cyclic bounded scan of `lease_claims_` from `claim_ring_slots.rescue_cursor_*`, wrapping to the start of the partition when it reaches the end, but only includes claims whose `claimed_at` is older than the rescue cutoff; anti-joins explicit `lease_claim_closures_`, compact `lease_claim_closure_batches_`, and `leases`, takes the per-`(job_id, run_lease)` advisory key with `pg_try_advisory_xact_lock`, locks stale candidates with `FOR UPDATE SKIP LOCKED`, closes stragglers with the rescue outcome `'rescued'` — row-local `lease_claims` claims append to `lease_claim_closures` and mark `lease_claims.closed_at`, while compact `lease_claim_batches` claims (which have no `lease_claims` row to mark) append a `lease_claim_closure_batches` row carrying `ready_slot` / `ready_generation` so both prune count proofs can see the closure — and advances the sweep cursor across closed, lease-managed, old heartbeat-fresh, or newly rescued claims while stopping before the first stale open claim not closed by the transaction | -| Receipt deadline rescue | `rescue_expired_receipt_deadlines_tx`, invoked from maintenance `rescue_expired_deadlines` after the lease-side scan. Excludes claims already materialized into `leases` so the lease-side deadline path owns those. | reads `claim_ring_state` to try the oldest initialized claim slot first, then scans remaining slots with the same global rescue budget; each active claim slot has an independent bounded cursor over `(deadline_at, job_id, run_lease)` stored on `claim_ring_slots.deadline_cursor_*`, but the candidate window includes only deadlines that have already passed; anti-joins explicit `lease_claim_closures_`, compact `lease_claim_closure_batches_`, and `leases`, takes the per-`(job_id, run_lease)` advisory key with `pg_try_advisory_xact_lock`, locks expired candidates with `FOR UPDATE SKIP LOCKED`, closes them by appending `lease_claim_closures(outcome = 'deadline_expired')` and marking `lease_claims.closed_at`, and advances over closed or lease-managed claims and newly inserted deadline closures. The cursor exists so a long MVCC horizon cannot make every deadline tick repeatedly re-prove millions of already-completed receipt claims, and the expired-only window avoids repeatedly proving future-deadline claims. | +| `RescueStaleReceipt(j, r)` | `rescue_stale_receipt_claims_tx`, invoked through `QueueStorage::rescue_stale_heartbeats_batch` from each batch of the maintenance leader's stale-heartbeat pass (`rescue_queue_storage_pass`). Excludes claims already materialized into `leases` so the lease-side rescue path owns those. The spec takes the explicit `(j, r)` so concurrent rescue / re-claim races are reachable: rescue can fire on an old attempt's `(j, r_old)` receipt while `(j, r_old + 1)` already has an open receipt in a newer partition. | reads `claim_ring_state` to try the oldest initialized claim slot first (every fourth batch of a pass visits the other slots first, and slots an earlier batch of the pass found idle are skipped), then scans remaining slots with the same 500-attempt batch budget; each slot uses a cyclic bounded index seek over `lease_claims_` and `lease_claim_batches_` from `claim_ring_slots.rescue_cursor_*`, wrapping to the start of the partition when it reaches the end, but only includes claims whose `claimed_at` is older than the rescue cutoff; anti-joins explicit `lease_claim_closures_`, compact `lease_claim_closure_batches_`, and `leases`, takes the per-`(job_id, run_lease)` advisory key with `pg_try_advisory_xact_lock`, locks stale candidates with `FOR UPDATE SKIP LOCKED`, closes stragglers with the rescue outcome `'rescued'` — row-local `lease_claims` claims append to `lease_claim_closures` and mark `lease_claims.closed_at`, while compact `lease_claim_batches` claims (which have no `lease_claims` row to mark) append a `lease_claim_closure_batches` row carrying `ready_slot` / `ready_generation` so both prune count proofs can see the closure — and advances the sweep cursor across closed, lease-managed, old heartbeat-fresh, or newly rescued claims while stopping before the first stale open claim not closed by the transaction | +| Receipt deadline rescue | `rescue_expired_receipt_deadlines_tx`, invoked through `QueueStorage::rescue_expired_deadlines_batch` after the lease-side scan, from each batch of the maintenance leader's deadline pass (`rescue_queue_storage_pass`). Excludes claims already materialized into `leases` so the lease-side deadline path owns those. | reads `claim_ring_state` to try the oldest initialized claim slot first, then scans remaining slots with the same global rescue budget; each active claim slot has an independent bounded cursor over `(deadline_at, job_id, run_lease)` stored on `claim_ring_slots.deadline_cursor_*`, but the candidate window includes only deadlines that have already passed; anti-joins explicit `lease_claim_closures_`, compact `lease_claim_closure_batches_`, and `leases`, takes the per-`(job_id, run_lease)` advisory key with `pg_try_advisory_xact_lock`, locks expired candidates with `FOR UPDATE SKIP LOCKED`, closes them by appending `lease_claim_closures(outcome = 'deadline_expired')` and marking `lease_claims.closed_at`, and advances over closed or lease-managed claims and newly inserted deadline closures. The cursor exists so a long MVCC horizon cannot make every deadline tick repeatedly re-prove millions of already-completed receipt claims, and the expired-only window avoids repeatedly proving future-deadline claims. | | `CancelRunningToTerminal(j)` | `QueueStorage::cancel_job_tx` lease branch | `DELETE FROM leases ... RETURNING`, `insert_done_rows_tx` (state = `cancelled`), `close_receipt_tx` → `close_receipt_pairs_tx` (writes the `'cancelled'` closure into the matching claim partition: row-local claims into `lease_claim_closures`, compact batch claims into `lease_claim_closure_batches`), `pg_notify('awa:cancel', ...)` | | `CancelReceiptOnlyToTerminal(j)` | `QueueStorage::cancel_job_tx` receipt-only branch | `SELECT ... FROM lease_claims` or `lease_claim_batches` `FOR UPDATE ... SKIP LOCKED` → `insert_done_rows_tx` → close the claim (row-local claim → `INSERT INTO lease_claim_closures`; compact batch claim → `INSERT INTO lease_claim_closure_batches` with `ready_slot` / `ready_generation`) → defensive `DELETE FROM leases` (sweeps any concurrent materialization) → `pg_notify` | @@ -224,6 +224,8 @@ The spec plus `AwaSegmentedStorageRaces.cfg` (race-exposing) and `AwaSegmentedSt `AwaStorageLockOrder.tla` (see [`README.md`](./README.md)) is the complementary positive artifact: it models the Postgres locks directly and checks that no interleaving of claim / complete / close-receipt / rescue-receipts / ensure-running / cancel / rotate-leases / prune-leases / rotate-ready / prune-ready / rotate-claims / prune-claims transactions produces a waits-for cycle. It also models the striped producer path that updates multiple physical queue lanes in a stable order, while the current runtime claim path claims only one physical stripe per transaction. A deliberately-broken demo config (`AwaStorageLockOrderDeadlockDemo.cfg`) confirms the deadlock detector fires when a cycle exists, and `AwaStorageLockOrderOldStripedClaimDeadlock.cfg` captures the historical unsafe shape where one logical claim transaction walked multiple physical stripes in the opposite order from enqueue. +`ProvisionLanePlan` models the first enqueue to a new `(queue, priority)` lane, which every producer now takes in one order: the `queue_lanes` row (SQL inserts in `awa.insert_job` and `insert_jobs_compat_batch`; `QueueStorage::ensure_lane_inserts` for batch enqueue, promotion and `prepare_queue`), then the provisioning advisory lock (the head-table trigger, or `ensure_lane_inserts` before `provision_lane`), then the lane's enqueue lock. `AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg` keeps the earlier `QueueStorage` order, provisioning lock before the row, and must violate `NoDeadlock` against a SQL producer of the same new lane (#692). + Together the two specs cover complementary risks: - `AwaSegmentedStorageRaces` catches data-level races that would occur if the locks were removed — proves the locks are necessary @@ -309,8 +311,9 @@ Model checking results: - `AwaSegmentedStorageRacesMultiWorker.cfg`: safe commit with 3 workers, 332 distinct states, clean. - `AwaShardedPrune.cfg`: 1,028 distinct states, clean. This checks the fixed queue-ring prune predicate that matches ready rows to terminal rows by `(enqueue_shard, lane_seq)`. - `AwaShardedPruneBroken.cfg`: expected failure. It ignores `enqueue_shard` in the ready/terminal match and produces the counterexample where shard 0's completed `lane_seq = 1` row masks shard 1's still-pending `lane_seq = 1` row. -- `AwaStorageLockOrder.cfg`: 153,515 distinct states, clean. Models the receipts and legacy claim modes separately, plus `CloseReceiptPlan`, `RescueReceiptsPlan`, `EnsureRunningPlan`, `CancelReceiptOnlyPlan`, and `CancelRunningPlan`. +- `AwaStorageLockOrder.cfg`: 245,414 distinct states, clean. Models the receipts and legacy claim modes separately, plus `CloseReceiptPlan`, `RescueReceiptsPlan`, `EnsureRunningPlan`, `CancelReceiptOnlyPlan`, `CancelRunningPlan`, and `ProvisionLanePlan`. - `AwaStorageLockOrderDeadlockDemo.cfg`: trips `NoDeadlock` in 5 steps, confirming the detector works. +- `AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg`: trips `NoDeadlock` in 5 steps, one transaction holding the new lane's `queue_lanes` row and the other its provisioning lock. - `AwaDeadTupleContract.cfg`: 1 distinct state, clean. TLC generated 2 states because the contract is an ASSUME-style architectural check. The checks (`HotTablesAreNotRowVacuum`, `PartitionTruncateTablesAreReclaimed`, `WarmTablesDocumentTheirBound`, `BacklogRowVacuumTablesDocumentTheirBound`, `OnlyBoundedKindsHaveBoundedBy`, `AppendOnlyAcceptsOnlyInsert`, `VacuumKindTablesNotTruncated`) all hold for the current schema and transaction list. Workflow: when adding a new table to `prepare_schema()` or a new SQL site to queue_storage.rs, register it in `AwaDeadTupleContract.tla` with the correct reclaim kind, hotness, optional `bounded_by`, and mutation list. An `open_receipt_claims`-style proposal (hot table, RowVacuum reclaim, INSERT+DELETE traffic) fires `HotTablesAreNotRowVacuum` at parse time; a `Warm` table without a declared `bounded_by` fires `WarmTablesDocumentTheirBound`. ### Receipt-plane shape @@ -328,6 +331,19 @@ Model checking results: `receipt_plane_chaos_test` (4 `#[ignore]`-d nightly tests) covers flood / concurrency / lock-order scenarios: rescue throughput under overload, prune-skips-active under concurrent traffic, the bounded `ACCESS EXCLUSIVE` barrier between TRUNCATE and concurrent inserts, and admin-cancel-during-materialize orphan-lease cleanup. +## Rescue passes, prune halt and enqueue contention + +These implementation behaviours refine existing actions or sit outside the models; none adds a modelled action or invariant. + +- **Bounded rescue passes.** The leader's `rescue_queue_storage_pass` runs a stale-heartbeat, deadline or callback rescue as a sequence of batches, each its own transaction of up to 500 attempts per source, until a batch comes back short, the pass budget or `rescue_pass_max_rows` is reached, and re-arms a second later while a backlog remains. Each rescued attempt is still one `RescueToReady`, `RescueStaleReceipt`, `TimeoutWaitingToReady` or `TimeoutWaitingToDlq` step; more batches per pass are more such steps sooner, so the safety invariants are unchanged. The models are untimed, so the drain rate they enable is not a modelled property. +- **Stale-heartbeat owner evidence.** Batches after the first in a stale-heartbeat pass run only while runtime snapshots show every runtime fresh with a live heartbeat loop or silent for the staleness window, and they use a heartbeat cutoff at least as old as the ordinary staleness cutoff. That only strengthens the per-attempt staleness guard behind `RescueToReady` and `RescueStaleReceipt`; a stalled live runtime whose attempts are rescued is the existing `LoseHeartbeat` case, and the `run_lease` guard still rejects its late completions. +- **Heartbeats skip locked attempts.** Heartbeat and progress writes take the per-attempt advisory key with `pg_try_advisory_xact_lock` and lock lease rows with `SKIP LOCKED`, and compact claims are heartbeated by claim batch and item index without a batch row lock. Heartbeats are not a plan in `AwaStorageLockOrder`; non-blocking acquisitions add no waits-for edge, so they cannot join a cycle. +- **Rescue visits several claim slots per transaction.** `RescueReceiptsPlan` and `RescueReceiptDeadlinesPlan` model one slot, while a rescue batch locks the `claim_ring_slots` row of each slot it visits, in an order that varies between batches. The leader runs one rescue batch at a time, so two rescue transactions meet only if a deposed leader's pass is still in flight; PostgreSQL would then abort one batch with `40P01`, and a later pass retries it. +- **Prune halt.** A `maintenance.prune` halt stops a compatible leader from admitting rotation and prune of the queue, lease and claim rings (`Rotate*Segments`, `Prune*Segment`) along with other housekeeping that has no model action. Halting removes behaviours, so every safety invariant still holds. No model states a liveness property about ring rotation (`AwaSegmentedStorage` checks safety only, and `AwaExtended` does not model prune), so none is conditioned on the absence of a halt. +- **Batch insert contention.** `insert_jobs_compat_batch` takes a batch's unique claims before its lanes and, for a bounded phase, gives way to a per-row producer by limiting each lock wait and rolling back to retry. Unique-claim keys are not modelled (see [Unique-claim keys](#unique-claim-keys)), and `AwaStorageLockOrder` deliberately excludes lock timeouts and aborts, so this protocol is covered by `awa/tests/insert_many_batch_test.rs` rather than TLC. +- **Point reads during prune.** `QueueStorage::load_job` reads once more in a new snapshot when the first finds nothing, because a `TRUNCATE` that commits after a read's snapshot hides both the old slot and the row prune moved out of it. Snapshot visibility is outside the models, whose reads see the current state. +- **Claim batch index.** Whether a `lease_claim_batches` slot has its GIN index on `job_ids` changes only how by-job lookups find compact claims; claim prune's count proof and the closure semantics are unchanged. + ## Dispatcher scheduling (#510) The shared LISTEN hub, capped empty-poll backoff and runtime semaphore change scheduling, not `Claim` ownership. Durable SQL claim evidence remains authoritative. A saturated semaphore is cancellable and refunds worker permits/rate tokens. A claim already in progress can finish during shutdown; each spawned execution observes the dispatcher cancellation token, including before in-flight registration. Dispatcher exit and worker drain share a timeout, after which pending dispatchers are aborted and unexecuted claims rely on lease rescue. diff --git a/correctness/storage/README.md b/correctness/storage/README.md index 8f5cd9934..f28ab0ed7 100644 --- a/correctness/storage/README.md +++ b/correctness/storage/README.md @@ -138,7 +138,7 @@ Run the broken correlated-hash witness with: ## Lock-order companion spec -[`AwaStorageLockOrder.tla`](./AwaStorageLockOrder.tla) models each storage-engine transaction (claim, rotate-leases, prune-leases, rotate-ready, prune-ready) as an ordered sequence of Postgres lock acquisitions, with a simplified shared/exclusive compatibility matrix that captures the cases relevant to deadlock analysis. Invariants: +[`AwaStorageLockOrder.tla`](./AwaStorageLockOrder.tla) models each storage-engine transaction (enqueue, first enqueue to a new lane, claim, complete, rescue, cancel, and rotate / prune of each ring) as an ordered sequence of Postgres lock acquisitions, with a simplified shared/exclusive compatibility matrix that captures the cases relevant to deadlock analysis. Invariants: - `NoDeadlock`: the waits-for graph is acyclic - `LockCompatibility`: no two incompatible locks on the same resource @@ -147,14 +147,18 @@ Run the broken correlated-hash witness with: Configs: -- [`AwaStorageLockOrder.cfg`](./AwaStorageLockOrder.cfg): main run against the real Rust lock plans. Models claim (receipts and legacy modes), complete, close-receipt, stale receipt rescue, receipt deadline rescue, ensure-running, terminal-delete closure materialization, the two cancel branches, plus rotate / prune for the queue, lease, and claim rings. This is the positive artifact saying the current SQL lock ordering is deadlock-free and the lock compatibility contract holds. +- [`AwaStorageLockOrder.cfg`](./AwaStorageLockOrder.cfg): main run against the real Rust lock plans. Models enqueue, the first enqueue to a new lane (`queue_lanes` row, provisioning lock, enqueue lock), claim (receipts and legacy modes), complete, close-receipt, stale receipt rescue, receipt deadline rescue, ensure-running, terminal-delete closure materialization, the two cancel branches, plus rotate / prune for the queue, lease, and claim rings. This is the positive artifact saying the current SQL lock ordering is deadlock-free and the lock compatibility contract holds. - [`AwaStorageLockOrderDeadlockDemo.cfg`](./AwaStorageLockOrderDeadlockDemo.cfg): sanity harness using a deliberately cycle-creating pair of plans — **NoDeadlock tripped in 5 steps** (confirms the checker works). +- [`AwaStorageLockOrderOldStripedClaimDeadlock.cfg`](./AwaStorageLockOrderOldStripedClaimDeadlock.cfg): the historical claim shape that walked several physical stripes against the enqueue order — **NoDeadlock tripped**. +- [`AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg`](./AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg): the historical `QueueStorage` order for a new lane, provisioning lock before the `queue_lanes` row, against a SQL producer of the same lane — **NoDeadlock tripped in 5 steps** (#692). Run: ```bash ./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla ./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderDeadlockDemo.cfg +./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderOldStripedClaimDeadlock.cfg +./correctness/run-tlc.sh storage/AwaStorageLockOrder.tla storage/AwaStorageLockOrderOldLaneProvisioningDeadlock.cfg ``` Coverage note: the plans model the lock steps that actually appear in the Rust SQL (`FOR UPDATE` / `FOR SHARE` / `LOCK TABLE ACCESS EXCLUSIVE` / the implicit AccessShare of SELECT on partition children). They do NOT model implicit table-level locks beyond what is named, or Postgres's lock-timeout / deadlock-detector abort choice. The spec treats a waits-for cycle as a safety violation, which is conservative — Postgres would abort one transaction and let the other proceed. For our purposes "this sequence of lock requests could produce a cycle" is the thing we want to catch, regardless of how the runtime resolves it. From 199a02a7d474fb0221f53300288580e28887fbbf Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Tue, 10 Nov 2026 06:51:07 +1300 Subject: [PATCH 10/10] docs: date the 0.6.16 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19089048b..e6880965a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [Unreleased] -## [0.6.16] — 2026-11-09 +## [0.6.16] — 2026-11-10 Recommended for everyone on 0.6.15 using queue storage. Rescue drains a large backlog of abandoned claims, which a killed runtime leaves behind and which stops the queue and claim rings from rotating, in minutes instead of the hours or days 0.6.15 took. Batch inserts (`insert_many_copy`, `insert_many_skip_unique_conflicts`) give way to per-row producers that share their keys or lanes instead of deadlocking with them, and creating a new queue lane no longer deadlocks between `QueueStorage` and SQL producers. Operators can durably halt and resume `maintenance.prune` across the fleet, `admin::get_job` no longer misses a failed job while prune moves it, and `awa migrate` re-applies the claim batch index and batch insert patches with fixes.