Skip to content

fix(queue-storage): preserve moved-attempt reads and snapshots (0.6) #652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
hardbyte wants to merge 2 commits into
base: release/0.6.6
Choose a base branch
from
Draft

fix(queue-storage): preserve moved-attempt reads and snapshots (0.6) #652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a41c85b
Fix moved-attempt hydration and protect point-read snapshots
hardbyte Nov 6, 2026
9b05fe2
fix: lock ready ring before point-read snapshots
hardbyte Nov 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ Notable changes between releases. Detailed migration notes for storage transitio

### Fixed

- Lock the ready ring before point reads establish their snapshot, so
concurrent queue pruning cannot hide a failed job carried to a live slot.
- Exclude tombstoned source rows when hydrating moved running jobs, including compact receipts, and preserve point-read snapshots against concurrent claim/lease partition pruning.
- **Ring prune and terminal-count rollup report `Blocked` when any of their
statements times out on a lock.** Prune sets a short `lock_timeout` (10 ms
by default) before it takes `ACCESS EXCLUSIVE` on a slot's partitions, but
Expand Down
45 changes: 45 additions & 0 deletions awa-model/src/admin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,15 @@ pub fn queue_storage_current_jobs_cte(schema: &str) -> String {
AND ready.run_lease + 1 = leases.run_lease
AND ready.enqueue_shard = leases.enqueue_shard
AND ready.lane_seq = leases.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
UNION ALL
SELECT job_id, kind, queue, state, created_at, run_at, finalized_at
FROM {schema}.terminal_jobs
Expand Down Expand Up @@ -734,6 +743,15 @@ fn queue_storage_recent_jobs_cte(schema: &str, since: &str) -> String {
AND ready.run_lease + 1 = leases.run_lease
AND ready.enqueue_shard = leases.enqueue_shard
AND ready.lane_seq = leases.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
WHERE ready.created_at >= {since}
UNION ALL
-- A queue or priority move copies the row with its times, so
Expand Down Expand Up @@ -856,6 +874,15 @@ fn queue_storage_kind_counts_cte(schema: &str) -> String {
AND ready.run_lease + 1 = leases.run_lease
AND ready.enqueue_shard = leases.enqueue_shard
AND ready.lane_seq = leases.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
GROUP BY ready.kind, leases.queue
),
deferred_by_kind AS MATERIALIZED (
Expand Down Expand Up @@ -1090,6 +1117,15 @@ fn queue_storage_job_id_candidates_sql(
AND ready.run_lease + 1 = leases.run_lease
AND ready.enqueue_shard = leases.enqueue_shard
AND ready.lane_seq = leases.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
WHERE TRUE{predicates}
ORDER BY leases.job_id DESC
LIMIT {limit}
Expand Down Expand Up @@ -1552,6 +1588,15 @@ fn unique_key_candidate_sql(schema: &str, scoped: bool) -> String {
AND ready.run_lease + 1 = leases.run_lease
AND ready.enqueue_shard = leases.enqueue_shard
AND ready.lane_seq = leases.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
AND ready.unique_key = $1
OFFSET 0
) AS ready ON TRUE
Expand Down
60 changes: 60 additions & 0 deletions awa-model/src/queue_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11866,6 +11866,20 @@ impl QueueStorage {
.await
.map_err(map_sqlx_error)?;

// Protect all rings prune can truncate before the first data query
// establishes this repeatable-read snapshot: the query can take its
// snapshot before acquiring its relation locks. Queue prune takes the
// ready child NOWAIT before terminal children, so ready also fences
// those. Claim-family order agrees with claim prune.
sqlx::query(&format!(
"LOCK TABLE {schema}.ready_entries, {schema}.leases, {schema}.lease_claims, \
{schema}.lease_claim_batches, {schema}.lease_claim_closures, \
{schema}.lease_claim_closure_batches IN ACCESS SHARE MODE"
))
.execute(tx.as_mut())
.await
.map_err(map_sqlx_error)?;

let ready_rows: Vec<ReadyJobRow> = sqlx::query_as(&format!(
r#"
SELECT
Expand Down Expand Up @@ -11979,6 +11993,15 @@ impl QueueStorage {
AND ready.run_lease + 1 = lease.run_lease
AND ready.enqueue_shard = lease.enqueue_shard
AND ready.lane_seq = lease.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
LEFT JOIN {schema}.attempt_state AS attempt
ON attempt.job_id = lease.job_id
AND attempt.run_lease = lease.run_lease
Expand Down Expand Up @@ -12039,6 +12062,15 @@ impl QueueStorage {
AND ready.run_lease + 1 = claims.run_lease
AND ready.enqueue_shard = claims.enqueue_shard
AND ready.lane_seq = claims.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
LEFT JOIN {schema}.attempt_state AS attempt
ON attempt.job_id = claims.job_id
AND attempt.run_lease = claims.run_lease
Expand Down Expand Up @@ -12115,6 +12147,16 @@ impl QueueStorage {
AND ready.enqueue_shard = claim_batches.enqueue_shard
AND ready.lane_seq = items.lane_seq
AND ready.job_id = items.job_id
AND ready.run_lease + 1 = items.run_lease
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
LEFT JOIN {schema}.attempt_state AS attempt
ON attempt.job_id = items.job_id
AND attempt.run_lease = items.run_lease
Expand Down Expand Up @@ -12567,6 +12609,15 @@ impl QueueStorage {
AND ready.run_lease + 1 = lease.run_lease
AND ready.enqueue_shard = lease.enqueue_shard
AND ready.lane_seq = lease.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
LEFT JOIN {schema}.attempt_state AS attempt
ON attempt.job_id = lease.job_id
AND attempt.run_lease = lease.run_lease
Expand Down Expand Up @@ -12766,6 +12817,15 @@ impl QueueStorage {
AND ready.run_lease + 1 = lease.run_lease
AND ready.enqueue_shard = lease.enqueue_shard
AND ready.lane_seq = lease.lane_seq
AND NOT EXISTS (
SELECT 1 FROM {schema}.ready_tombstones AS tomb
WHERE tomb.ready_slot = ready.ready_slot
AND tomb.ready_generation = ready.ready_generation
AND tomb.queue = ready.queue
AND tomb.priority = ready.priority
AND tomb.enqueue_shard = ready.enqueue_shard
AND tomb.lane_seq = ready.lane_seq
)
LEFT JOIN {schema}.attempt_state AS attempt
ON attempt.job_id = lease.job_id
AND attempt.run_lease = lease.run_lease
Expand Down
Loading
Loading