From e8609bd6c1aec279175db21b2b69e8a492a7431f Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 06:23:50 +1300 Subject: [PATCH 1/7] chore: bump the Rust and Python packages to 0.6.15 --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 12 ++++++------ awa-cli/Cargo.toml | 4 ++-- awa-cli/pyproject.toml | 2 +- awa-python/Cargo.lock | 10 +++++----- awa-python/Cargo.toml | 6 +++--- awa-python/pyproject.toml | 4 ++-- awa-python/uv.lock | 4 ++-- awa-seaorm/Cargo.toml | 2 +- awa/Cargo.toml | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 656f4a77d..9aa90294f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,7 +383,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "awa" -version = "0.6.14" +version = "0.6.15" dependencies = [ "async-trait", "awa-macros", @@ -409,7 +409,7 @@ dependencies = [ [[package]] name = "awa-cli" -version = "0.6.14" +version = "0.6.15" dependencies = [ "assert_cmd", "awa-model", @@ -431,7 +431,7 @@ dependencies = [ [[package]] name = "awa-macros" -version = "0.6.14" +version = "0.6.15" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -441,7 +441,7 @@ dependencies = [ [[package]] name = "awa-metrics" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa-model", "opentelemetry", @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "awa-model" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa-macros", "blake3", @@ -471,7 +471,7 @@ dependencies = [ [[package]] name = "awa-seaorm" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa", "awa-testing", @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "awa-testing" -version = "0.6.14" +version = "0.6.15" dependencies = [ "async-trait", "awa-model", @@ -499,7 +499,7 @@ dependencies = [ [[package]] name = "awa-ui" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa-metrics", "awa-model", @@ -522,7 +522,7 @@ dependencies = [ [[package]] name = "awa-worker" -version = "0.6.14" +version = "0.6.15" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 2121e4041..12453de53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ exclude = ["awa-python", "spike"] [workspace.package] -version = "0.6.14" +version = "0.6.15" edition = "2021" license = "MIT OR Apache-2.0" description = "Postgres-native background job queue — transactional enqueue, heartbeat crash recovery, SKIP LOCKED dispatch" @@ -25,11 +25,11 @@ categories = ["database", "asynchronous", "web-programming"] [workspace.dependencies] # Internal crates -awa-model = { path = "awa-model", version = "0.6.14" } -awa-macros = { path = "awa-macros", version = "0.6.14" } -awa-metrics = { path = "awa-metrics", version = "0.6.14" } -awa-worker = { path = "awa-worker", version = "0.6.14" } -awa = { path = "awa", version = "0.6.14" } +awa-model = { path = "awa-model", version = "0.6.15" } +awa-macros = { path = "awa-macros", version = "0.6.15" } +awa-metrics = { path = "awa-metrics", version = "0.6.15" } +awa-worker = { path = "awa-worker", version = "0.6.15" } +awa = { path = "awa", version = "0.6.15" } # Database sea-orm = { version = "=2.0.0-rc.38", default-features = false, features = ["sqlx-postgres", "runtime-tokio-rustls", "with-chrono", "with-json"] } diff --git a/awa-cli/Cargo.toml b/awa-cli/Cargo.toml index da4141211..96bffb3e5 100644 --- a/awa-cli/Cargo.toml +++ b/awa-cli/Cargo.toml @@ -14,7 +14,7 @@ path = "src/main.rs" [dependencies] awa-model.workspace = true awa-worker.workspace = true -awa-ui = { path = "../awa-ui", version = "0.6.14" } +awa-ui = { path = "../awa-ui", version = "0.6.15" } axum.workspace = true sqlx.workspace = true tokio.workspace = true @@ -28,5 +28,5 @@ hex.workspace = true uuid.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.14" } +awa-testing = { path = "../awa-testing", version = "0.6.15" } assert_cmd = "2" diff --git a/awa-cli/pyproject.toml b/awa-cli/pyproject.toml index 8b3be9b01..3609b14c2 100644 --- a/awa-cli/pyproject.toml +++ b/awa-cli/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "awa-cli" -version = "0.6.14" +version = "0.6.15" description = "CLI for the Awa Postgres-native job queue (migrations, admin, serve)" readme = "README.md" requires-python = ">=3.10" diff --git a/awa-python/Cargo.lock b/awa-python/Cargo.lock index eaf424b46..f29fa6999 100644 --- a/awa-python/Cargo.lock +++ b/awa-python/Cargo.lock @@ -95,7 +95,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "awa-macros" -version = "0.6.14" +version = "0.6.15" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "awa-metrics" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa-model", "opentelemetry", @@ -113,7 +113,7 @@ dependencies = [ [[package]] name = "awa-model" -version = "0.6.14" +version = "0.6.15" dependencies = [ "awa-macros", "blake3", @@ -133,7 +133,7 @@ dependencies = [ [[package]] name = "awa-python" -version = "0.6.14" +version = "0.6.15" dependencies = [ "async-trait", "awa-model", @@ -156,7 +156,7 @@ dependencies = [ [[package]] name = "awa-worker" -version = "0.6.14" +version = "0.6.15" dependencies = [ "async-trait", "awa-macros", diff --git a/awa-python/Cargo.toml b/awa-python/Cargo.toml index d81c4d54e..be022c1a4 100644 --- a/awa-python/Cargo.toml +++ b/awa-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "awa-python" -version = "0.6.14" +version = "0.6.15" edition = "2021" [lib] @@ -12,8 +12,8 @@ default = ["cel"] cel = ["awa-model/cel"] [dependencies] -awa-model = { path = "../awa-model", version = "0.6.14" } -awa-worker = { path = "../awa-worker", version = "0.6.14", features = ["__python-bridge"] } +awa-model = { path = "../awa-model", version = "0.6.15" } +awa-worker = { path = "../awa-worker", version = "0.6.15", features = ["__python-bridge"] } pyo3 = { version = "0.29", features = ["macros", "abi3-py310", "chrono"] } pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono", "json"] } diff --git a/awa-python/pyproject.toml b/awa-python/pyproject.toml index 38e7cef76..76d77df44 100644 --- a/awa-python/pyproject.toml +++ b/awa-python/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "awa-pg" requires-python = ">=3.10" -version = "0.6.14" +version = "0.6.15" description = "Postgres-native background job queue — Python SDK with async/sync workers, transactional enqueue, and progress tracking. Install awa-pg[ui] to add the bundled web dashboard." readme = "README.md" license = {text = "MIT OR Apache-2.0"} @@ -34,7 +34,7 @@ classifiers = [ # # Kept opt-in so the default `awa-pg` install stays small — workers and # producers don't need the ~10 MB UI bundle. -ui = ["awa-cli==0.6.14"] +ui = ["awa-cli==0.6.15"] [project.urls] Homepage = "https://github.com/hardbyte/awa" diff --git a/awa-python/uv.lock b/awa-python/uv.lock index c18f79069..063ed5887 100644 --- a/awa-python/uv.lock +++ b/awa-python/uv.lock @@ -88,12 +88,12 @@ wheels = [ [[package]] name = "awa-cli" -version = "0.6.14" +version = "0.6.15" source = { directory = "../awa-cli" } [[package]] name = "awa-pg" -version = "0.6.14" +version = "0.6.15" source = { editable = "." } [package.optional-dependencies] diff --git a/awa-seaorm/Cargo.toml b/awa-seaorm/Cargo.toml index 336c4de99..50a1c6688 100644 --- a/awa-seaorm/Cargo.toml +++ b/awa-seaorm/Cargo.toml @@ -14,6 +14,6 @@ serde_json.workspace = true sqlx.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.14" } +awa-testing = { path = "../awa-testing", version = "0.6.15" } serde = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/awa/Cargo.toml b/awa/Cargo.toml index 60275e5c9..56a1d98f8 100644 --- a/awa/Cargo.toml +++ b/awa/Cargo.toml @@ -20,7 +20,7 @@ awa-macros.workspace = true awa-worker.workspace = true [dev-dependencies] -awa-testing = { path = "../awa-testing", version = "0.6.14" } +awa-testing = { path = "../awa-testing", version = "0.6.15" } sqlx.workspace = true serde.workspace = true serde_json.workspace = true From 6469d39533253b741f68c3686a2f0fb73b5e59fe Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 06:24:21 +1300 Subject: [PATCH 2/7] docs: cover the claim_batch_job_index patch in the 0.6.15 upgrade guide --- docs/upgrade-0.6.14-to-0.6.15.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/upgrade-0.6.14-to-0.6.15.md b/docs/upgrade-0.6.14-to-0.6.15.md index ff7f0d430..9e0b8d816 100644 --- a/docs/upgrade-0.6.14-to-0.6.15.md +++ b/docs/upgrade-0.6.14-to-0.6.15.md @@ -1,6 +1,6 @@ # Upgrading from 0.6.14 to 0.6.15 -0.6.15 is a rolling patch upgrade. Traffic keeps running throughout, binaries and `awa migrate` can go in either order, and you can roll binaries back to 0.6.14 with the schema patch left in place. +0.6.15 is a rolling patch upgrade. Traffic keeps running throughout, binaries and `awa migrate` can go in either order, and you can roll binaries back to 0.6.14 with the schema patches left in place. ## Upgrade @@ -12,15 +12,27 @@ awa --database-url "$AWA_MIGRATOR_DATABASE_URL" migrate ``` - The output reports `Schema patch insert_jobs_compat_batch: applied`. + The output reports `Schema patch claim_batch_job_index: applied` and `Schema patch insert_jobs_compat_batch: applied`. 3. Roll workers, producers and admin processes to 0.6.15, in any order. -Steps 2 and 3 can also run the other way round. A 0.6.15 producer checks for the patch on every `insert_many_copy` call, so it switches to the batch function on its next call after the patch commits, without a restart. +Steps 2 and 3 can also run the other way round. A 0.6.15 producer checks for the batch function on every `insert_many_copy` call, so it switches to it on its next call after the patch commits, without a restart. Claim prune starts indexing claim partitions on its next pass after the patch commits. ## What `awa migrate` changes -The schema stays at v40. The `insert_jobs_compat_batch` patch adds one function, `awa.insert_jobs_compat_batch`, and changes nothing else: no table DDL, no table or sequence locks, no change to `awa.insert_job_compat` or to storage routing. It applies as one transaction, so an interrupted run leaves nothing applied; run it again. +The schema stays at v40. `awa migrate` applies two repeatable schema patches in one transaction, so an interrupted run leaves nothing applied; run it again. Neither takes a lock on any `awa` or queue-storage table, so both are safe under live traffic. + +### `claim_batch_job_index` + +The patch adds `.ensure_claim_batch_job_index(slot)` to every queue-storage schema and builds no index itself. Claim prune calls it right after truncating a claim partition, which builds a GIN index on `lease_claim_batches.job_ids` while the partition is empty. Every partition gains the index within one claim-ring cycle, and live claims never wait behind a build. Don't build the index yourself on a populated partition: the build holds a `SHARE` lock that blocks claim inserts into that partition, for about half a minute per 100k batches. + +With the index, by-job lookups of zero-deadline (compact) receipt claims stop reading every retained claim batch. These are heartbeat and progress flush, completion fallbacks, receipt close, cancellation, rescue and `load_job`. Under load those scans could pick parallel plans and exhaust PostgreSQL's dynamic shared memory. A partition without the index yet returns the same rows by scan. + +The function is `SECURITY DEFINER` and owned by the owner of `lease_claim_batches`. `EXECUTE` is reset to exactly the roles holding `TRUNCATE` on that table each time the patch runs; see [Claim batch index](security.md#claim-batch-index). A runtime that cannot execute it skips the build and keeps claiming. + +### `insert_jobs_compat_batch` + +The patch adds one function, `awa.insert_jobs_compat_batch`, and changes nothing else: no table DDL, no table or sequence locks, no change to `awa.insert_job_compat` or to storage routing. The function inserts a whole batch the way `awa.insert_job_compat` inserts one job, with the same validation, unique claims, lane assignment and storage routing. It does everything that does not depend on a lane first (job ids, unique claims in key order, scheduled rows, the result rows). Then, in `(queue, priority, shard)` order, it registers each lane it is the first to use and reserves each lane's range of positions once, as every other producer registers and reserves lane by lane, and writes the ready rows. Each lane's enqueue lock is therefore held only for the reservation, the ready-row write and the commit, instead of from the batch's first ready row to the commit. Concurrent producers and promotion on the same lane wait for that much shorter window. @@ -33,11 +45,11 @@ What callers see: 0.6.14 and earlier binaries never call the new function. They keep calling `awa.insert_job_compat`, which takes the same lane lock, so old and new producers share a lane without reordering it. -### Unique keys shared with per-row producers +#### Unique keys shared with per-row producers The batch function takes all of a batch's unique claims before any lane lock. Per-row inserts (generic `insert_many`, single-job inserts, and 0.6.14 and earlier binaries) take each job's claim and its lane together, lane by lane. When a per-row transaction already holds a lane the batch needs and then inserts a job whose unique key the batch has claimed but not yet committed, each waits on the other. PostgreSQL aborts one of them with `deadlock detected` (SQLSTATE `40P01`), where before the per-row insert would have waited and then skipped the job or failed with a unique violation (`23505`). Retry the aborted transaction: by then the key is either committed, giving the old outcome, or free. Two batches through the function take claims in key order and lanes in lane order, so they do not deadlock with each other this way. -### Privileges +#### Privileges New functions are executable by `PUBLIC` unless your deployment revokes that through default privileges. If it does and the runtime role's default privileges do not cover functions created by the migration role, grant it explicitly: @@ -52,10 +64,10 @@ Without `EXECUTE`, `insert_many_copy` keeps the per-row path and `insert_many_sk ### External migration runners -`awa migrate --sql --pending` prints the `R__insert_jobs_compat_batch` patch on a 0.6.14 database. To write it as a file instead, run `awa migrate --extract-to --pending` and apply `R__insert_jobs_compat_batch.sql` as a repeatable script after `V40`. It is a single `CREATE OR REPLACE FUNCTION`, safe inside the runner's transaction and safe to re-run. Run it as the role that owns the existing `awa` functions, normally the schema owner. +`awa migrate --sql --pending` prints the `R__claim_batch_job_index` and `R__insert_jobs_compat_batch` patches on a 0.6.14 database. To write them as files instead, run `awa migrate --extract-to --pending` and apply `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql` as repeatable scripts after `V40`. Both are safe inside the runner's transaction and safe to re-run. Run them as the schema owner, which owns the existing `awa` functions and the queue-storage tables. Replace the runner's `R__` files with the set the 0.6.15 CLI extracts, together, and keep the `V` files exactly as originally applied. ## Roll back -Roll binaries back to 0.6.14 at any time. Leave the patch in place: released binaries do not call the function, and it holds no state. +Roll binaries back to 0.6.14 at any time. Leave the patches in place: 0.6.14 binaries neither call the functions nor use the claim batch index, and neither patch holds state. From 55949f64e79c6c59642e2da4d7adcc55cbfe31c7 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 06:37:36 +1300 Subject: [PATCH 3/7] docs: write the 0.6.15 release notes and cover the finalization bound in the upgrade guide --- CHANGELOG.md | 35 ++++++++++++++++++++++---------- docs/upgrade-0.6.14-to-0.6.15.md | 12 +++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9228e5a2..1a6a53aa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,31 +4,44 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [Unreleased] +## [0.6.15] — 2026-11-09 + +Recommended for everyone on 0.6.14 or 0.6.13 using queue storage. A worker whose completions fall behind stops claiming instead of holding every finished job in memory until it runs out, by-job lookups of compact claims no longer read every retained claim batch, and large COPY batches hold a queue lane's enqueue lock only while they write that lane. + ### Upgrading -- The schema stays at v40. Run `awa migrate` as the schema owner, before or after rolling binaries, while traffic keeps running. It applies the repeatable `insert_jobs_compat_batch` patch, which adds one function and changes no existing object. Binaries can roll back to 0.6.14 with the patch left in place. See the [upgrade guide](docs/upgrade-0.6.14-to-0.6.15.md). -- External migration runners: apply `R__insert_jobs_compat_batch.sql` after `V40` (`awa migrate --extract-to --pending`), replacing all `R__` files together. +- The schema stays at v40. Run `awa migrate` once as the schema owner, before or after rolling binaries, while traffic keeps running. It applies two repeatable patches, `claim_batch_job_index` and `insert_jobs_compat_batch`, which add functions and take no table locks. Binaries can roll back to 0.6.14 with the patches left in place. See the [upgrade guide](docs/upgrade-0.6.14-to-0.6.15.md). +- External migration runners: apply `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql` after `V40` (`awa migrate --extract-to --pending`), replacing all `R__` files together. - Deployments that revoke `EXECUTE` on new functions from `PUBLIC` should grant it on `awa.insert_jobs_compat_batch` to runtime roles; without it, `insert_many_copy` keeps its per-row path. +### Fixed + +**Worker memory** + +- Worker finalization is bounded to the larger of the runtime's worker capacity and twice the completion batch size times the completion shard count (1,024 attempts with the queue-storage defaults). A finished handler keeps its worker slot while it waits for finalizer capacity, so when durable completion falls behind the runtime stops claiming instead of holding an unbounded backlog of finished attempts and their payloads in memory. Before, a 120-second completion stall could leave about 77,000 finished attempts and 1 GB in one worker. Attempt heartbeats continue until finalization settles. Lowering `AWA_COMPLETION_BATCH_SIZE` lowers the bound in proportion, for jobs with large payloads. + +**Claim lookups** + +- By-job lookups of zero-deadline (compact) receipt claims — heartbeat and progress flush, completion fallbacks, receipt close, cancellation, rescue and `load_job` — use a GIN index on `lease_claim_batches.job_ids` instead of reading every retained claim batch. Under load those scans could pick parallel plans and exhaust dynamic shared memory. The `claim_batch_job_index` patch installs the index function and takes no table locks; claim prune builds the index on each claim partition right after emptying it, so every partition gains it within one claim-ring cycle and live claims are never blocked behind a build. +- Retried queue-storage jobs and jobs carrying metadata complete by their compact claim batch and item index, instead of reading every retained claim batch. + +**Rescue, events and admin reads** + +- Heartbeat, deadline and callback rescue of lease-backed attempts reach the stale attempts behind a full batch of attempts whose retained ready bodies are ambiguous. Before, every pass selected the same ambiguous attempts first, left them running, and rescued nothing else. +- A `Completed` lifecycle event, and the job a `Completed` follow-up enqueue spec receives, no longer carry the finished attempt's `heartbeat_at` and `deadline_at`; both are `None`, as a read of the completed job returns them. +- Admin job lists and kind counts work on a queue-storage schema installed with more claim slots than the default, instead of failing with `invalid physical claim slot`, and no longer read other claim slots' receipt closure partitions when they read one slot's open claims. +- Queue-storage job lists without a kind filter select receipt candidate IDs before loading their ready bodies. Kind filters still apply before the limit. + ### Changed - Compatibility COPY inserts (`insert_many_copy` in Rust and Python) hold a queue lane's enqueue lock only while they write that lane, instead of from the batch's first ready row until commit. Many producers enqueuing large batches into one queue no longer queue behind each other's whole batches, and promotion into a busy queue waits less. The batch goes through the new `awa.insert_jobs_compat_batch` function, which prepares job ids, unique claims and payloads first, then registers each new lane and reserves each lane once, in the sorted `(queue, priority, shard)` order every other producer uses. Each call checks its connection for the function and keeps the per-row path where it is missing or not executable. Results are unchanged: input order, with jobs whose unique key is already claimed skipped. - A batch through `awa.insert_jobs_compat_batch` takes all its unique claims before any lane lock, while per-row inserts (generic `insert_many`, single-job inserts, 0.6.14 and earlier binaries) take each job's claim and lane together, lane by lane. A per-row transaction that already holds a lane the batch needs and then inserts a job whose unique key the batch has claimed but not committed now fails with `deadlock detected` (SQLSTATE `40P01`) instead of skipping the job or failing with a unique violation (`23505`). Retry the aborted transaction. Batches through the function do not deadlock with each other this way. -- Queue-storage job lists without a kind filter select receipt candidate IDs before loading their ready bodies. Kind filters still apply before the limit. ### Added - `insert_many_skip_unique_conflicts` (Rust) inserts a batch through `awa.insert_jobs_compat_batch`, skipping jobs whose unique key is already claimed, and returns the inserted rows in input order. Use it instead of retrying a failed `insert_many` one job at a time, which takes the lane lock once per job. It needs the `insert_jobs_compat_batch` patch. Generic `insert_many` is unchanged. - `enqueue_lane_contention_test` benchmarks many producers on one lane, with a nightly gate on per-lane throughput and lock hold time. -### Fixed - -- Point lookups of zero-deadline (compact) receipt claims by job ID — heartbeat and progress flush, completion fallbacks, receipt close, cancellation, rescue and `load_job` — use a GIN index on `lease_claim_batches.job_ids` instead of reading every retained claim batch. Under load those scans could pick parallel plans and exhaust dynamic shared memory. The repeatable `claim_batch_job_index` schema patch installs the index function and takes no table locks; claim prune builds the index on each claim partition right after emptying it, so every partition gains it within one claim-ring cycle after the patch and live claims are never blocked behind a build. The schema remains v40. -- Admin job lists and kind counts work on a queue-storage schema installed with more claim slots than the default, instead of failing with `invalid physical claim slot`. -- Heartbeat, deadline and callback rescue of lease-backed attempts reach the stale attempts behind a full batch of attempts whose retained ready bodies are ambiguous. Before, every pass selected the same ambiguous attempts first, left them running, and rescued nothing else. A pass now sets all of a batch's ambiguous attempts aside at once. -- Admin job lists and kind counts no longer read other claim slots' receipt closure partitions when they read one slot's open claims. -- A `Completed` lifecycle event, and the job a `Completed` follow-up enqueue spec receives, no longer carry the finished attempt's `heartbeat_at` and `deadline_at`; both are `None`, as a read of the completed job returns them. - ## [0.6.14] — 2026-11-08 Recommended for everyone on 0.6.13 using queue storage. A busy queue no longer stalls promotion of the others, prune backs off cleanly under lock contention, new queue lanes are usable by every runtime role, and jobs no longer stay running, run again or get misreported after priority aging. diff --git a/docs/upgrade-0.6.14-to-0.6.15.md b/docs/upgrade-0.6.14-to-0.6.15.md index 9e0b8d816..57e63d8fd 100644 --- a/docs/upgrade-0.6.14-to-0.6.15.md +++ b/docs/upgrade-0.6.14-to-0.6.15.md @@ -68,6 +68,18 @@ Without `EXECUTE`, `insert_many_copy` keeps the per-row path and `insert_many_sk Replace the runner's `R__` files with the set the 0.6.15 CLI extracts, together, and keep the `V` files exactly as originally applied. +## Worker memory when completions fall behind + +A 0.6.15 worker bounds how many finished attempts may wait for their completion to commit: the larger of its worker capacity and twice the completion batch size times the completion shard count. With the queue-storage defaults (`AWA_COMPLETION_BATCH_SIZE=512`, one shard) and fewer than 512 workers, that is 1,024 attempts. A handler that finishes while the bound is reached keeps its worker slot until a completion commits, so the worker stops claiming new jobs instead of holding the backlog, and each waiting attempt's payload, in memory. + +0.6.14 and earlier admitted every finished attempt. When completions fell behind handlers, for example under lock contention on the completion tables, worker memory grew by roughly 10 KB per waiting attempt plus its payload until the process was killed, and did not shrink afterwards. + +What changes in practice: + +- A worker whose completions stall shows its workers busy and its claim rate dropping to its completion rate, instead of its memory climbing. Attempts keep heartbeating while they wait, so they are not rescued as stale. +- Jobs with large payloads hold more memory per waiting attempt. Lower `AWA_COMPLETION_BATCH_SIZE` to lower the bound in proportion. +- If you reduced worker counts to keep 0.6.14 workers below their completion rate, you can restore them. + ## Roll back Roll binaries back to 0.6.14 at any time. Leave the patches in place: 0.6.14 binaries neither call the functions nor use the claim batch index, and neither patch holds state. From 3add531217dc7294d30caca478b2227b12f012c7 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 06:43:29 +1300 Subject: [PATCH 4/7] docs: each 0.6.15 schema patch commits on its own, and extract the full repeatable set for external runners --- CHANGELOG.md | 2 +- docs/upgrade-0.6.14-to-0.6.15.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a6a53aa2..9b6dadb80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Recommended for everyone on 0.6.14 or 0.6.13 using queue storage. A worker whose ### Upgrading - The schema stays at v40. Run `awa migrate` once as the schema owner, before or after rolling binaries, while traffic keeps running. It applies two repeatable patches, `claim_batch_job_index` and `insert_jobs_compat_batch`, which add functions and take no table locks. Binaries can roll back to 0.6.14 with the patches left in place. See the [upgrade guide](docs/upgrade-0.6.14-to-0.6.15.md). -- External migration runners: apply `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql` after `V40` (`awa migrate --extract-to --pending`), replacing all `R__` files together. +- External migration runners: apply `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql` after `V40`. Replace all `R__` files together with the full set from `awa migrate --extract-to --from 40`; `--pending` lists only the patches a database lacks. - Deployments that revoke `EXECUTE` on new functions from `PUBLIC` should grant it on `awa.insert_jobs_compat_batch` to runtime roles; without it, `insert_many_copy` keeps its per-row path. ### Fixed diff --git a/docs/upgrade-0.6.14-to-0.6.15.md b/docs/upgrade-0.6.14-to-0.6.15.md index 57e63d8fd..b9c7ba194 100644 --- a/docs/upgrade-0.6.14-to-0.6.15.md +++ b/docs/upgrade-0.6.14-to-0.6.15.md @@ -20,7 +20,7 @@ Steps 2 and 3 can also run the other way round. A 0.6.15 producer checks for the ## What `awa migrate` changes -The schema stays at v40. `awa migrate` applies two repeatable schema patches in one transaction, so an interrupted run leaves nothing applied; run it again. Neither takes a lock on any `awa` or queue-storage table, so both are safe under live traffic. +The schema stays at v40. `awa migrate` applies two repeatable schema patches, each in its own transaction. An interrupted run can leave the first applied and the second not; run it again to apply the rest. Neither takes a lock on any `awa` or queue-storage table, so both are safe under live traffic. ### `claim_batch_job_index` @@ -64,9 +64,9 @@ Without `EXECUTE`, `insert_many_copy` keeps the per-row path and `insert_many_sk ### External migration runners -`awa migrate --sql --pending` prints the `R__claim_batch_job_index` and `R__insert_jobs_compat_batch` patches on a 0.6.14 database. To write them as files instead, run `awa migrate --extract-to --pending` and apply `R__claim_batch_job_index.sql` and `R__insert_jobs_compat_batch.sql` as repeatable scripts after `V40`. Both are safe inside the runner's transaction and safe to re-run. Run them as the schema owner, which owns the existing `awa` functions and the queue-storage tables. +`awa migrate --sql --pending` prints the `R__claim_batch_job_index` and `R__insert_jobs_compat_batch` patches on a 0.6.14 database. Apply them as repeatable scripts after `V40`. Both are safe inside the runner's transaction and safe to re-run. Run them as the schema owner, which owns the existing `awa` functions and the queue-storage tables. -Replace the runner's `R__` files with the set the 0.6.15 CLI extracts, together, and keep the `V` files exactly as originally applied. +`--pending` lists only the patches that database lacks, so its output is not the full repeatable set. Write the full set with `awa migrate --extract-to --from 40`, which extracts every 0.6.15 `R__` file and no `V` files. Replace the runner's `R__` files with that set, together, and keep the `V` files exactly as originally applied. ## Worker memory when completions fall behind From 443b0f0f24a4acb6a94565b6157f095a76d319d3 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 08:20:26 +1300 Subject: [PATCH 5/7] docs: cover the lifecycle hook bound in the 0.6.15 notes and upgrade guide --- CHANGELOG.md | 3 ++- docs/upgrade-0.6.14-to-0.6.15.md | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6dadb80..d3b436a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [0.6.15] — 2026-11-09 -Recommended for everyone on 0.6.14 or 0.6.13 using queue storage. A worker whose completions fall behind stops claiming instead of holding every finished job in memory until it runs out, by-job lookups of compact claims no longer read every retained claim batch, and large COPY batches hold a queue lane's enqueue lock only while they write that lane. +Recommended for everyone on 0.6.14 or 0.6.13 using queue storage. A worker whose completions or lifecycle hooks fall behind stops claiming instead of holding every finished job in memory until it runs out, by-job lookups of compact claims no longer read every retained claim batch, and large COPY batches hold a queue lane's enqueue lock only while they write that lane. ### Upgrading @@ -19,6 +19,7 @@ Recommended for everyone on 0.6.14 or 0.6.13 using queue storage. A worker whose **Worker memory** - Worker finalization is bounded to the larger of the runtime's worker capacity and twice the completion batch size times the completion shard count (1,024 attempts with the queue-storage defaults). A finished handler keeps its worker slot while it waits for finalizer capacity, so when durable completion falls behind the runtime stops claiming instead of holding an unbounded backlog of finished attempts and their payloads in memory. Before, a 120-second completion stall could leave about 77,000 finished attempts and 1 GB in one worker. Attempt heartbeats continue until finalization settles. Lowering `AWA_COMPLETION_BATCH_SIZE` lowers the bound in proportion, for jobs with large payloads. +- Lifecycle hooks (`on_event`, `on_event_kind`) no longer accumulate without bound when they are slower than job throughput. Each attempt's `Started` and outcome hooks now hold its finalizer slot until they return, so slow hooks slow claiming instead of growing worker memory, and a worker-finalized outcome hook runs after that attempt's `Started` hook has returned. Maintenance's `Rescued` and duplicate `Cancelled` events allow 1,024 outstanding hook tasks per worker and drop further events with a warning. **Claim lookups** diff --git a/docs/upgrade-0.6.14-to-0.6.15.md b/docs/upgrade-0.6.14-to-0.6.15.md index b9c7ba194..66da7e59f 100644 --- a/docs/upgrade-0.6.14-to-0.6.15.md +++ b/docs/upgrade-0.6.14-to-0.6.15.md @@ -79,6 +79,8 @@ What changes in practice: - A worker whose completions stall shows its workers busy and its claim rate dropping to its completion rate, instead of its memory climbing. Attempts keep heartbeating while they wait, so they are not rescued as stale. - Jobs with large payloads hold more memory per waiting attempt. Lower `AWA_COMPLETION_BATCH_SIZE` to lower the bound in proportion. - If you reduced worker counts to keep 0.6.14 workers below their completion rate, you can restore them. +- Lifecycle hooks (`on_event`, `on_event_kind`) run under the same bound: each attempt's `Started` and outcome hooks hold its slot until they return. A hook slower than job throughput now slows claiming, where 0.6.14 kept claiming and accumulated hook tasks in memory. Check that hooks return quickly, and move slow work into a job or follow-up. +- Maintenance's `Rescued` and duplicate `Cancelled` hooks still run detached so they cannot hold up rescue, with at most 1,024 outstanding per worker; further events are dropped with a warning. ## Roll back From 1c24938ac08b35315c356538b2fe6885731856d0 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 08:25:08 +1300 Subject: [PATCH 6/7] docs: describe the finalization bound in configuration and troubleshooting --- docs/configuration.md | 2 ++ docs/troubleshooting.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index c0ec2a062..2f2dd6cad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -621,6 +621,8 @@ Queue storage defaults `AWA_COMPLETION_SHARDS` to `1` for ordinary runtimes and `AWA_COMPLETION_BATCH_SIZE` defaults to `512` and `AWA_COMPLETION_FLUSH_MS` defaults to `1`. The queue-storage short-job path fuses receipt closure and terminal insertion into one statement, so the lower flush interval reduces the worker capacity tied up waiting for durable completion while the batch size still amortises the claim/complete path under load. That pairing is what moves the lower WAL budget into durable throughput. +A runtime lets at most the larger of its worker capacity and `2 × AWA_COMPLETION_BATCH_SIZE × AWA_COMPLETION_SHARDS` finished attempts wait for their completion to commit: 1,024 with the queue-storage defaults below 512 workers. Each attempt's lifecycle hooks run within that allowance. A handler that finishes while it is used up keeps its worker slot until a completion commits, so a runtime whose completions or hooks fall behind claims more slowly instead of holding the backlog in memory. Each waiting attempt holds its job, payload included; for large payloads, lower `AWA_COMPLETION_BATCH_SIZE` to lower the bound in proportion. + For very high-throughput no-op queues, size `max_workers` for durable completion latency, not handler CPU alone. The `10k/s` offered-rate reference shape needs 1,024 in-flight worker permits to absorb the load consistently; real jobs with non-trivial handler time need workload-specific sizing. The terminal write path is already narrow for running/waiting jobs: successful receipt completions use compact `receipt_completion_batches_*` rows, and other retained terminal facts use `done_entries_*` without duplicating immutable ready body fields when the ready row is still retained. Both surfaces hydrate through `terminal_jobs`, so the default completion settings stay conservative without giving up durable terminal history. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5dd8a9dc4..ff76df346 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -146,6 +146,22 @@ To recover: In all cases the cancel itself is durable — only the early wake-up is lost. Long-running handlers that don't poll `ctx.is_cancelled()` between heartbeats won't notice the cancel until they finish or heartbeat-rescue fires. +## Claiming Slows While Handlers Are Idle + +### What It Usually Means + +Durable completion, or lifecycle hooks, are behind the handlers. A runtime lets only a bounded number of finished attempts wait for their completion to commit (see [completion settings](configuration.md#hot-queue-claim-control)). Once that allowance is used up, finished handlers keep their worker slots, `awa_job_in_flight` sits at `max_workers` while handler CPU is idle, and the runtime claims only as fast as completions commit. Worker memory stays flat. Before 0.6.15 the runtime kept claiming instead, and its memory grew with every finished attempt until the process was killed. + +### What To Check + +- `awa_completion_flush_duration_seconds`: long flushes point at the database, usually lock waits on the completion or terminal tables. Look for long transactions and lock waits on the queue-storage schema, as in [Inspect Long Transactions](#inspect-long-transactions). +- `awa_completion_deadlock_retry_total`: a steady rate means completions are re-running after deadlocks. +- Lifecycle hooks (`on_event`, `on_event_kind`): a hook that waits on a slow service or on the same database holds its attempt's slot until it returns. Move slow work into a job or a follow-up. + +### Fixes + +Clear the lock contention or long transaction; claiming recovers as soon as completions commit again. Raising `max_workers` does not help while completion is the bottleneck. For jobs with large payloads, lowering `AWA_COMPLETION_BATCH_SIZE` reduces how much memory waiting attempts can hold. + ## Rescue Fails With `idx_awa_jobs_unique` Symptom: maintenance repeats, every tick (canonical engine — the queue-storage engine wedges identically but logs `error: unique conflict` instead): From 85ef60be9ab7290da5807c049d03de18ef43e9f7 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 08:38:44 +1300 Subject: [PATCH 7/7] docs: describe the whole 0.6.15 upgrade in the README guide table --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bcd74cb1..0710c117c 100644 --- a/README.md +++ b/README.md @@ -389,7 +389,7 @@ All coordination through Postgres. The Rust runtime owns dispatch, leases, heart | [Migration guide](docs/migrations.md) | Fresh installs, upgrades, extracted SQL, rollback strategy | | [0.5 → 0.6 upgrade](docs/upgrade-0.5-to-0.6.md) | Step-by-step operator checklist for the staged storage transition | | [0.6.9 → 0.6.10 upgrade](docs/upgrade-0.6.9-to-0.6.10.md) | Rolling patch upgrade, rollback, and optional runtime privilege tightening | -| [0.6.14 → 0.6.15 upgrade](docs/upgrade-0.6.14-to-0.6.15.md) | Rolling patch upgrade for the batch compatibility insert | +| [0.6.14 → 0.6.15 upgrade](docs/upgrade-0.6.14-to-0.6.15.md) | Rolling patch upgrade: bounded worker finalization, the claim batch index and the batch compatibility insert | | [Configuration reference](docs/configuration.md) | `QueueConfig`, `ClientBuilder`, Python `start()`, env vars | | [HTTP workers and callbacks](docs/http-callbacks.md) | Serverless dispatch, callback receiver endpoints, signature verification | | [Lifecycle hooks](docs/lifecycle-hooks.md) | React to job start/finish/retry/callback events for metrics, alerting, cleanup |