From fa1ac71d2c4c3d19dbdbcf640b780caf1d020862 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Wed, 4 Nov 2026 21:18:34 +1300 Subject: [PATCH 1/3] perf(queue-storage): drain a queue in bulk chunks (0.6) Backport of #598. admin::drain_queue on queue storage listed the queue's jobs through the current-jobs view, which reads every retained ready row, then cancelled them one by one inside a single transaction. Scheduled, retryable and available jobs are now cancelled 1,000 at a time, one transaction per chunk, and a striped queue drains every persisted stripe. --- CHANGELOG.md | 13 +- awa-model/src/admin.rs | 19 +- awa-model/src/queue_storage.rs | 340 ++++++++++++++++++++++++ awa/tests/queue_storage_runtime_test.rs | 238 +++++++++++++++++ 4 files changed, 595 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ffefa404..41a400772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [0.6.12] — Unreleased -Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit. No migration and no public API change. +Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit, and draining a queue cancels its backlog in bulk. No migration and no public API change. - **Completing a large batch and moving many failed jobs to the DLQ no longer fail on queue storage** (backport of the writer approach in @@ -21,6 +21,17 @@ Patch release: batch completion and bulk "move failed to DLQ" on queue storage n `AWA_COMPLETION_BATCH_SIZE` is raised above the default of 512. Both writers now send one array per column through a fixed-text `INSERT ... SELECT FROM unnest(...)`. +- **`admin::drain_queue` on queue storage cancels a backlog in bulk and + drains every stripe of a striped queue.** It found the queue's jobs through + a lookup that read every retained ready row, then cancelled them one at a + time, several statements each, inside one transaction, so draining a large + backlog took minutes and held its row locks throughout. Scheduled, + retryable and available jobs are now cancelled 1,000 at a time, each chunk + one short transaction, with available rows read from each lane's claim + cursor; waiting jobs are still cancelled one by one and running jobs are + left alone. It also matched only the logical queue name, so draining a + striped queue cancelled nothing; it now covers every stripe. `awa queue + drain` and the Python `drain_queue` use this path. ## [0.6.11] — 2026-11-04 diff --git a/awa-model/src/admin.rs b/awa-model/src/admin.rs index fd16120bf..899f45274 100644 --- a/awa-model/src/admin.rs +++ b/awa-model/src/admin.rs @@ -1631,20 +1631,11 @@ where /// Drain a queue: cancel all non-running, non-terminal jobs. pub async fn drain_queue(pool: &PgPool, queue: &str) -> Result { if let Some(store) = active_queue_storage(pool).await? { - let sql = format!( - "{} \ - SELECT job_id \ - FROM current_jobs \ - WHERE queue = $1 \ - AND state IN ('available', 'scheduled', 'retryable', 'waiting_external') \ - ORDER BY job_id ASC", - queue_storage_current_jobs_cte(store.schema()) - ); - let ids: Vec = sqlx::query_scalar(&sql).bind(queue).fetch_all(pool).await?; - return store - .cancel_jobs_by_ids(pool, &ids) - .await - .map(|rows| rows.len() as u64); + let queues = { + let mut conn = pool.acquire().await?; + persisted_physical_queues(&mut conn, store.schema(), queue).await? + }; + return store.drain_queues(pool, &queues).await; } let result = sqlx::query( diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 42c59ede2..1b36f1e63 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -25,6 +25,7 @@ const COPY_NULL_SENTINEL: &str = "__AWA_NULL__"; const COPY_CHUNK_TARGET_BYTES: usize = 256 * 1024; const TERMINAL_COUNTER_BUCKETS: i16 = 256; const RECEIPT_RESCUE_BATCH_LIMIT: i64 = 500; +const DRAIN_BATCH_SIZE: i64 = 1_000; const RECEIPT_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const DEFAULT_PRUNE_LOCK_TIMEOUT: Duration = Duration::from_millis(10); @@ -8407,6 +8408,345 @@ impl QueueStorage { Ok(rows) } + /// Cancel every available, scheduled, retryable and waiting job in the + /// physical `queues`, returning how many were cancelled. + /// + /// Scheduled, retryable and available jobs are cancelled a chunk at a + /// time, each chunk in its own transaction, so a large backlog costs a + /// few statements per chunk rather than per job and holds row locks for + /// one chunk at a time. Waiting jobs go through [`Self::cancel_jobs_by_ids`]. + /// Running jobs are left alone. + pub async fn drain_queues(&self, pool: &PgPool, queues: &[String]) -> Result { + let mut drained = 0u64; + + loop { + let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let cancelled = self.cancel_deferred_chunk_tx(&mut tx, queues).await?; + tx.commit().await.map_err(map_sqlx_error)?; + drained += cancelled as u64; + if (cancelled as i64) < DRAIN_BATCH_SIZE { + break; + } + } + + // After the deferred pass, so jobs promoted while it ran are drained + // from the ready lanes. + loop { + let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let (cancelled, advances) = self.cancel_available_chunk_tx(&mut tx, queues).await?; + tx.commit().await.map_err(map_sqlx_error)?; + self.advance_claim_cursors(pool, &advances).await; + drained += cancelled as u64; + if (cancelled as i64) < DRAIN_BATCH_SIZE { + break; + } + } + + let schema = self.schema(); + let waiting: Vec = sqlx::query_scalar(&format!( + r#" + SELECT job_id + FROM {schema}.leases + WHERE queue = ANY($1) + AND state = 'waiting_external' + ORDER BY job_id + "# + )) + .bind(queues) + .fetch_all(pool) + .await + .map_err(map_sqlx_error)?; + drained += self.cancel_jobs_by_ids(pool, &waiting).await?.len() as u64; + + Ok(drained) + } + + async fn cancel_deferred_chunk_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + queues: &[String], + ) -> Result { + let schema = self.schema(); + let deferred: Vec = sqlx::query_as(&format!( + r#" + DELETE FROM {schema}.deferred_jobs + WHERE job_id IN ( + SELECT job_id + FROM {schema}.deferred_jobs + WHERE queue = ANY($1) + AND state IN ('scheduled', 'retryable') + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + RETURNING + job_id, + kind, + queue, + args, + state, + priority, + attempt, + run_lease, + max_attempts, + run_at, + attempted_at, + finalized_at, + created_at, + unique_key, + unique_states, + COALESCE(payload, '{{}}'::jsonb) AS payload + "# + )) + .bind(queues) + .bind(DRAIN_BATCH_SIZE) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + + if deferred.is_empty() { + return Ok(0); + } + + let cancelled = deferred.len(); + let (ready_slot, ready_generation) = self.current_queue_ring(tx).await?; + let lanes: BTreeSet<(String, i16)> = deferred + .iter() + .map(|row| (row.queue.clone(), row.priority)) + .collect(); + for (queue, priority) in &lanes { + self.ensure_lane(tx, queue, *priority, 0).await?; + } + + let finalized_at = Utc::now(); + let mut by_state: HashMap> = HashMap::new(); + for deferred in deferred { + // Same shape as a single deferred cancel: shard 0 and a negative + // lane_seq keep the done_entries key unique. + let done = DoneJobRow { + ready_slot, + ready_generation, + job_id: deferred.job_id, + kind: deferred.kind, + queue: deferred.queue, + args: deferred.args, + state: JobState::Cancelled, + priority: deferred.priority, + attempt: deferred.attempt, + run_lease: deferred.run_lease, + max_attempts: deferred.max_attempts, + lane_seq: -deferred.job_id, + enqueue_shard: 0, + run_at: deferred.run_at, + attempted_at: deferred.attempted_at, + finalized_at, + created_at: deferred.created_at, + unique_key: deferred.unique_key, + unique_states: deferred.unique_states, + payload: deferred.payload, + }; + by_state.entry(deferred.state).or_default().push(done); + } + for (old_state, rows) in by_state { + self.insert_done_rows_tx(tx, &rows, Some(old_state)).await?; + } + + Ok(cancelled) + } + + async fn cancel_available_chunk_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + queues: &[String], + ) -> Result<(usize, Vec), AwaError> { + let schema = self.schema(); + // Unclaimed rows are found the way a claim finds them: from each + // lane's claim cursor through its segments, then along the ready key + // within each segment's ring slot. The ready key leads with the ring + // slot, so a lane range without it scans every retained ready row. + // Only rows whose tombstone this statement inserts are cancelled, so a + // concurrent cancel of the same row cannot produce a second done row. + let ready: Vec = sqlx::query_as(&format!( + r#" + WITH heads AS ( + SELECT + claims.queue, + claims.priority, + claims.enqueue_shard, + {schema}.sequence_next_value(claims.seq_name) AS claim_seq, + {schema}.sequence_next_value(enqueues.seq_name) AS next_seq + FROM {schema}.queue_claim_heads AS claims + JOIN {schema}.queue_enqueue_heads AS enqueues + ON enqueues.queue = claims.queue + AND enqueues.priority = claims.priority + AND enqueues.enqueue_shard = claims.enqueue_shard + WHERE claims.queue = ANY($1) + ), + candidates AS ( + SELECT + ready.ready_slot, + ready.ready_generation, + ready.queue, + ready.priority, + ready.enqueue_shard, + ready.lane_seq + FROM heads + CROSS JOIN LATERAL ( + SELECT + segment.ready_slot, + segment.ready_generation, + GREATEST(segment.first_lane_seq, heads.claim_seq) AS from_seq, + LEAST(segment.next_lane_seq, heads.next_seq) AS to_seq + FROM {schema}.ready_segments AS segment + WHERE segment.queue = heads.queue + AND segment.priority = heads.priority + AND segment.enqueue_shard = heads.enqueue_shard + AND segment.next_lane_seq > heads.claim_seq + AND segment.first_lane_seq < heads.next_seq + ORDER BY segment.next_lane_seq + ) AS segment + CROSS JOIN LATERAL ( + SELECT + ready.ready_slot, + ready.ready_generation, + ready.queue, + ready.priority, + ready.enqueue_shard, + ready.lane_seq + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = heads.queue + AND ready.priority = heads.priority + AND ready.enqueue_shard = heads.enqueue_shard + AND ready.lane_seq >= segment.from_seq + AND ready.lane_seq < segment.to_seq + ORDER BY ready.lane_seq + OFFSET 0 + ) AS ready + WHERE NOT EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + LIMIT $2 + ), + target AS ( + SELECT ready.* + FROM {schema}.ready_entries AS ready + JOIN candidates + ON candidates.ready_slot = ready.ready_slot + AND candidates.ready_generation = ready.ready_generation + AND candidates.queue = ready.queue + AND candidates.priority = ready.priority + AND candidates.enqueue_shard = ready.enqueue_shard + AND candidates.lane_seq = ready.lane_seq + FOR UPDATE OF ready SKIP LOCKED + ), + tombstone AS ( + INSERT INTO {schema}.ready_tombstones ( + ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq, job_id + ) + SELECT ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq, job_id + FROM target + ON CONFLICT DO NOTHING + RETURNING ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq + ) + SELECT + target.ready_slot, + target.ready_generation, + target.job_id, + target.kind, + target.queue, + target.args, + target.priority, + target.attempt, + target.run_lease, + target.max_attempts, + target.lane_seq, + target.enqueue_shard, + target.run_at, + target.attempted_at, + target.created_at, + target.unique_key, + target.unique_states, + COALESCE(target.payload, '{{}}'::jsonb) AS payload + FROM target + JOIN tombstone + ON tombstone.ready_slot = target.ready_slot + AND tombstone.ready_generation = target.ready_generation + AND tombstone.queue = target.queue + AND tombstone.priority = target.priority + AND tombstone.enqueue_shard = target.enqueue_shard + AND tombstone.lane_seq = target.lane_seq + "# + )) + .bind(queues) + .bind(DRAIN_BATCH_SIZE) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + + if ready.is_empty() { + return Ok((0, Vec::new())); + } + + // One advance per run of consecutive cancelled positions on a lane: + // it applies only while the cursor sits at the start of the run, and + // every position it skips was tombstoned above. + let mut ready = ready; + ready.sort_unstable_by(|left, right| { + ( + &left.queue, + left.priority, + left.enqueue_shard, + left.lane_seq, + ) + .cmp(&( + &right.queue, + right.priority, + right.enqueue_shard, + right.lane_seq, + )) + }); + let mut advances: Vec = Vec::new(); + for row in &ready { + if let Some(last) = advances.last_mut() { + if last.queue == row.queue + && last.priority == row.priority + && last.enqueue_shard == row.enqueue_shard + && last.next_seq == row.lane_seq + { + last.next_seq = row.lane_seq + 1; + continue; + } + } + advances.push(ClaimCursorAdvance { + queue: row.queue.clone(), + priority: row.priority, + enqueue_shard: row.enqueue_shard, + next_seq: row.lane_seq + 1, + only_if_current: Some(row.lane_seq), + }); + } + + let finalized_at = Utc::now(); + let done_rows: Vec = ready + .into_iter() + .map(|row| { + let payload = row.payload.clone(); + row.into_done_row(JobState::Cancelled, finalized_at, payload) + }) + .collect(); + self.insert_done_rows_tx(tx, &done_rows, Some(JobState::Available)) + .await?; + + Ok((done_rows.len(), advances)) + } + pub async fn set_priority( &self, pool: &PgPool, diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index c4235d75e..78739e950 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -13854,3 +13854,241 @@ async fn test_queue_storage_cancel_by_unique_key_reads_live_work_only() { {scoped_finished_claims} with retained claims" ); } + +async fn queue_deferred_count(pool: &sqlx::PgPool, store: &QueueStorage, queue: &str) -> i64 { + sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.deferred_jobs WHERE queue = $1", + store.schema() + )) + .bind(queue) + .fetch_one(pool) + .await + .expect("count deferred jobs") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_cancels_a_backlog_larger_than_one_chunk() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_backlog"; + let other_queue = "qs_drain_other"; + let store = create_store(&pool, "awa_qs_drain_backlog").await; + + store + .enqueue_batch(&pool, queue, 2, 3_000) + .await + .expect("enqueue history"); + loop { + let claimed = store + .claim_batch(&pool, queue, 1_000) + .await + .expect("claim history"); + if claimed.is_empty() { + break; + } + store + .complete_batch(&pool, &claimed) + .await + .expect("complete history"); + } + + store + .enqueue_batch(&pool, queue, 2, 3) + .await + .expect("enqueue retry jobs"); + let to_retry = store + .claim_runtime_batch(&pool, queue, 3, Duration::from_secs(30)) + .await + .expect("claim retry jobs"); + assert_eq!(to_retry.len(), 3); + for claimed in &to_retry { + store + .retry_after( + &pool, + claimed.job.id, + claimed.job.run_lease, + Duration::from_secs(3_600), + None, + ) + .await + .expect("retry job") + .expect("running job should become retryable"); + } + + store + .enqueue_batch(&pool, queue, 2, 1) + .await + .expect("enqueue running job"); + let running = store + .claim_runtime_batch(&pool, queue, 1, Duration::from_secs(30)) + .await + .expect("claim running job"); + assert_eq!(running.len(), 1); + let running_id = running[0].job.id; + + store + .enqueue_batch(&pool, queue, 2, 2_500) + .await + .expect("enqueue available backlog"); + let scheduled: Vec<_> = (0..2_500) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue scheduled backlog"); + let unique_opts = InsertOpts { + queue: queue.to_string(), + unique: Some(UniqueOpts::default()), + ..Default::default() + }; + for id in 0..3 { + enqueue_job( + &pool, + &store, + &CompleteJob { id: 900_000 + id }, + unique_opts.clone(), + ) + .await; + } + + store + .enqueue_batch(&pool, other_queue, 2, 10) + .await + .expect("enqueue other available"); + let other_scheduled: Vec<_> = (0..10) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: other_queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("other scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &other_scheduled) + .await + .expect("enqueue other scheduled"); + + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 3 + 2_500 + 2_500 + 3); + + let counts = store + .queue_counts(&pool, queue) + .await + .expect("queue counts after drain"); + assert_eq!(counts.available, 0); + assert_eq!(counts.running, 1); + assert_eq!(queue_deferred_count(&pool, &store, queue).await, 0); + let dispatcher_counts = store + .queue_counts_fast(&pool, queue) + .await + .expect("dispatcher counts after drain"); + assert_eq!( + dispatcher_counts.available, 0, + "the claim cursor should move past the drained rows" + ); + let cancelled: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.done_entries WHERE queue = $1 AND state = 'cancelled'", + store.schema() + )) + .bind(queue) + .fetch_one(&pool) + .await + .expect("count cancelled jobs"); + assert_eq!(cancelled, drained as i64); + let running_job = store + .load_job(&pool, running_id) + .await + .expect("load running job") + .expect("running job should exist"); + assert_eq!(running_job.state, JobState::Running); + + let other_counts = store + .queue_counts(&pool, other_queue) + .await + .expect("other queue counts"); + assert_eq!(other_counts.available, 10); + assert_eq!(queue_deferred_count(&pool, &store, other_queue).await, 10); + + // The drained unique jobs released their keys, and the lane's claim + // cursor moved past the cancelled rows. + let requeued = enqueue_job(&pool, &store, &CompleteJob { id: 900_000 }, unique_opts).await; + let claimed = store + .claim_runtime_batch(&pool, queue, 10, Duration::from_secs(30)) + .await + .expect("claim after drain"); + let claimed_ids: Vec = claimed.iter().map(|claimed| claimed.job.id).collect(); + assert_eq!(claimed_ids, vec![requeued]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_covers_every_stripe() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_striped"; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: "awa_qs_drain_striped".to_string(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 2, + lease_claim_receipts: false, + ..Default::default() + }, + ) + .await; + + store + .enqueue_batch(&pool, queue, 2, 20) + .await + .expect("enqueue striped available"); + let scheduled: Vec<_> = (0..20) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue striped scheduled"); + + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain striped queue"); + assert_eq!(drained, 40); + for stripe in 0..2 { + let physical = format!("{queue}#{stripe}"); + let counts = store + .queue_counts(&pool, &physical) + .await + .expect("stripe counts"); + assert_eq!(counts.available, 0, "{physical} available"); + assert_eq!( + queue_deferred_count(&pool, &store, &physical).await, + 0, + "{physical} deferred" + ); + } +} From 07166fcae47d386cb30590594e52e0a502417fb5 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Thu, 5 Nov 2026 03:20:27 +1300 Subject: [PATCH 2/3] fix(queue-storage): bound the drain and leave claimed rows alone (0.6) Backport of the follow-up on #598. The drain reads every lane's enqueue head once and job ids below the next id at the start, so work enqueued while it runs is left for a later call. Each chunk locks rows before applying its limit and the next chunk resumes after the last cancelled row. Stripes are matched by name in every pass, which covers stripes holding only scheduled jobs. Cancel, move and reprioritise skip ready rows a claim has recorded but the claim cursor has not yet passed, so they no longer cancel or re-enqueue a job that is already running. --- CHANGELOG.md | 19 +- awa-model/src/admin.rs | 9 +- awa-model/src/queue_storage.rs | 271 +++++++++++++++++------- awa/tests/queue_storage_runtime_test.rs | 170 +++++++++++++++ 4 files changed, 380 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41a400772..f7ce76b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [0.6.12] — Unreleased -Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit, and draining a queue cancels its backlog in bulk. No migration and no public API change. +Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit, draining a queue cancels its backlog in bulk, and cancelling or moving an available job no longer acts on one a worker has already claimed. No migration and no public API change. - **Completing a large batch and moving many failed jobs to the DLQ no longer fail on queue storage** (backport of the writer approach in @@ -29,9 +29,20 @@ Patch release: batch completion and bulk "move failed to DLQ" on queue storage n retryable and available jobs are now cancelled 1,000 at a time, each chunk one short transaction, with available rows read from each lane's claim cursor; waiting jobs are still cancelled one by one and running jobs are - left alone. It also matched only the logical queue name, so draining a - striped queue cancelled nothing; it now covers every stripe. `awa queue - drain` and the Python `drain_queue` use this path. + left alone. Jobs enqueued after the drain starts are left for a later + call, so draining a queue that keeps receiving work still returns. It also + matched only the logical queue name, so draining a striped queue cancelled + nothing; it now covers every stripe. `awa queue drain` and the Python + `drain_queue` use this path. +- **Cancelling, moving or reprioritising an available job on queue storage + no longer acts on a job a worker has already claimed.** A claim moves its + lane's claim cursor past the rows it took only after it commits, so + briefly, or until the next claim if that advance failed, a claimed row + still looked available. A cancel in that window wrote a cancelled terminal + row while the attempt kept running, and a move re-enqueued a job that was + already executing. These paths now skip rows a claim has recorded, the + same evidence claims use to skip them, so a cancel reaches the running + attempt instead. ## [0.6.11] — 2026-11-04 diff --git a/awa-model/src/admin.rs b/awa-model/src/admin.rs index 899f45274..baf0a3db4 100644 --- a/awa-model/src/admin.rs +++ b/awa-model/src/admin.rs @@ -1629,13 +1629,12 @@ where } /// Drain a queue: cancel all non-running, non-terminal jobs. +/// +/// On queue storage this also drains the queue's stripes: queues named +/// `#`. Jobs enqueued after the drain starts are left alone. pub async fn drain_queue(pool: &PgPool, queue: &str) -> Result { if let Some(store) = active_queue_storage(pool).await? { - let queues = { - let mut conn = pool.acquire().await?; - persisted_physical_queues(&mut conn, store.schema(), queue).await? - }; - return store.drain_queues(pool, &queues).await; + return store.drain_queue(pool, queue).await; } let result = sqlx::query( diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 1b36f1e63..17e6c0580 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -531,6 +531,16 @@ fn claim_child_name(schema: &str, slot: usize) -> String { format!("{schema}.lease_claims_{slot}") } +/// SQL true when `column` names queue `param` or one of its stripes +/// (`queue#N`). +fn queue_and_stripes_sql(column: &str, param: &str) -> String { + format!( + "({column} = {param} \ + OR (left({column}, char_length({param}) + 1) = {param} || '{QUEUE_STRIPE_DELIMITER}' \ + AND substr({column}, char_length({param}) + 2) ~ '^[0-9]+$'))" + ) +} + fn claim_batch_child_name(schema: &str, slot: usize) -> String { format!("{schema}.lease_claim_batches_{slot}") } @@ -1564,6 +1574,15 @@ struct ReadyTransitionRow { payload: serde_json::Value, } +#[derive(Debug, Clone, sqlx::FromRow)] +struct DrainLane { + queue: String, + priority: i16, + enqueue_shard: i16, + from_seq: i64, + to_seq: i64, +} + #[derive(Debug, Clone)] struct ClaimCursorAdvance { queue: String, @@ -7826,12 +7845,53 @@ impl QueueStorage { Ok(()) } + /// SQL true when the ready row `alias` has been claimed. A claim records + /// its attempt in its own transaction but moves the lane's claim cursor + /// past the row only after commit, so a claimed row can still sit at or + /// after the cursor; claims skip such rows on the same evidence. Receipt + /// claims leave an attempt batch, and materialized claims a later attempt + /// of the job, which is checked whatever this store's mode, since admin + /// callers build the store with default configuration. + fn ready_row_claimed_sql(&self, alias: &str) -> String { + let schema = self.schema(); + let mut sql = format!( + "EXISTS ( + SELECT 1 FROM {schema}.ready_claim_attempt_batches AS attempts + WHERE attempts.ready_slot = {alias}.ready_slot + AND attempts.ready_generation = {alias}.ready_generation + AND attempts.queue = {alias}.queue + AND attempts.priority = {alias}.priority + AND attempts.enqueue_shard = {alias}.enqueue_shard + AND attempts.first_lane_seq <= {alias}.lane_seq + AND attempts.next_lane_seq > {alias}.lane_seq + AND attempts.lane_ranges @> int8range({alias}.lane_seq, {alias}.lane_seq + 1, '[)') + )" + ); + for table in [ + "leases", + "done_entries", + "deferred_jobs", + "dlq_entries", + "ready_entries", + ] { + sql.push_str(&format!( + " OR EXISTS ( + SELECT 1 FROM {schema}.{table} AS attempt + WHERE attempt.job_id = {alias}.job_id + AND attempt.run_lease = {alias}.run_lease + 1 + )" + )); + } + sql + } + async fn cancel_job_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, job_id: i64, ) -> Result, AwaError> { let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" WITH target AS ( @@ -7852,6 +7912,7 @@ impl QueueStorage { AND tomb.ready_slot = ready.ready_slot AND tomb.ready_generation = ready.ready_generation ) + AND NOT ({claimed}) ORDER BY ready.lane_seq DESC LIMIT 1 FOR UPDATE OF ready SKIP LOCKED @@ -8408,51 +8469,101 @@ impl QueueStorage { Ok(rows) } - /// Cancel every available, scheduled, retryable and waiting job in the - /// physical `queues`, returning how many were cancelled. + /// Cancel every available, scheduled, retryable and waiting job in + /// `queue` and its stripes (`queue#N`), returning how many were cancelled. /// - /// Scheduled, retryable and available jobs are cancelled a chunk at a - /// time, each chunk in its own transaction, so a large backlog costs a + /// Jobs enqueued after the drain starts are left alone, and so are running + /// jobs. Scheduled, retryable and available jobs are cancelled a chunk at + /// a time, each chunk in its own transaction, so a large backlog costs a /// few statements per chunk rather than per job and holds row locks for - /// one chunk at a time. Waiting jobs go through [`Self::cancel_jobs_by_ids`]. - /// Running jobs are left alone. - pub async fn drain_queues(&self, pool: &PgPool, queues: &[String]) -> Result { - let mut drained = 0u64; + /// one chunk at a time. Rows another transaction has locked are skipped. + /// Waiting jobs go through [`Self::cancel_jobs_by_ids`]. + pub(crate) async fn drain_queue(&self, pool: &PgPool, queue: &str) -> Result { + let schema = self.schema(); + let job_id_bound: i64 = sqlx::query_scalar(&format!( + "SELECT {schema}.sequence_next_value('job_id_seq')" + )) + .fetch_one(pool) + .await + .map_err(map_sqlx_error)?; + let mut drained = 0u64; + let mut after_job_id = i64::MIN; loop { let mut tx = pool.begin().await.map_err(map_sqlx_error)?; - let cancelled = self.cancel_deferred_chunk_tx(&mut tx, queues).await?; + let (cancelled, last_job_id) = self + .cancel_deferred_chunk_tx(&mut tx, queue, after_job_id, job_id_bound) + .await?; tx.commit().await.map_err(map_sqlx_error)?; drained += cancelled as u64; if (cancelled as i64) < DRAIN_BATCH_SIZE { break; } + after_job_id = last_job_id; } - // After the deferred pass, so jobs promoted while it ran are drained - // from the ready lanes. - loop { + // Read after the deferred pass, so jobs promoted while it ran are + // drained from the ready lanes. + let mut lanes: Vec = sqlx::query_as(&format!( + r#" + SELECT + claims.queue, + claims.priority, + claims.enqueue_shard, + {schema}.sequence_next_value(claims.seq_name) AS from_seq, + {schema}.sequence_next_value(enqueues.seq_name) AS to_seq + FROM {schema}.queue_claim_heads AS claims + JOIN {schema}.queue_enqueue_heads AS enqueues + ON enqueues.queue = claims.queue + AND enqueues.priority = claims.priority + AND enqueues.enqueue_shard = claims.enqueue_shard + WHERE {queues} + "#, + queues = queue_and_stripes_sql("claims.queue", "$1"), + )) + .bind(queue) + .fetch_all(pool) + .await + .map_err(map_sqlx_error)?; + lanes.retain(|lane| lane.from_seq < lane.to_seq); + + while !lanes.is_empty() { let mut tx = pool.begin().await.map_err(map_sqlx_error)?; - let (cancelled, advances) = self.cancel_available_chunk_tx(&mut tx, queues).await?; + let (cancelled, advances) = self.cancel_available_chunk_tx(&mut tx, &lanes).await?; tx.commit().await.map_err(map_sqlx_error)?; self.advance_claim_cursors(pool, &advances).await; drained += cancelled as u64; if (cancelled as i64) < DRAIN_BATCH_SIZE { break; } + // The next chunk resumes after the rows this one cancelled, so + // each lane is read once even when its claim cursor cannot move + // past a claimed or locked row. + for advance in &advances { + if let Some(lane) = lanes.iter_mut().find(|lane| { + lane.queue == advance.queue + && lane.priority == advance.priority + && lane.enqueue_shard == advance.enqueue_shard + }) { + lane.from_seq = lane.from_seq.max(advance.next_seq); + } + } + lanes.retain(|lane| lane.from_seq < lane.to_seq); } - let schema = self.schema(); let waiting: Vec = sqlx::query_scalar(&format!( r#" SELECT job_id FROM {schema}.leases - WHERE queue = ANY($1) + WHERE {queues} AND state = 'waiting_external' + AND job_id < $2 ORDER BY job_id - "# + "#, + queues = queue_and_stripes_sql("queue", "$1"), )) - .bind(queues) + .bind(queue) + .bind(job_id_bound) .fetch_all(pool) .await .map_err(map_sqlx_error)?; @@ -8461,11 +8572,16 @@ impl QueueStorage { Ok(drained) } + /// Cancel up to [`DRAIN_BATCH_SIZE`] scheduled and retryable jobs of + /// `queue` with ids in `(after_job_id, job_id_bound)`, returning how many + /// were cancelled and the largest cancelled id. async fn cancel_deferred_chunk_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, - queues: &[String], - ) -> Result { + queue: &str, + after_job_id: i64, + job_id_bound: i64, + ) -> Result<(usize, i64), AwaError> { let schema = self.schema(); let deferred: Vec = sqlx::query_as(&format!( r#" @@ -8473,9 +8589,12 @@ impl QueueStorage { WHERE job_id IN ( SELECT job_id FROM {schema}.deferred_jobs - WHERE queue = ANY($1) + WHERE {queues} AND state IN ('scheduled', 'retryable') - LIMIT $2 + AND job_id > $2 + AND job_id < $3 + ORDER BY job_id + LIMIT $4 FOR UPDATE SKIP LOCKED ) RETURNING @@ -8495,17 +8614,20 @@ impl QueueStorage { unique_key, unique_states, COALESCE(payload, '{{}}'::jsonb) AS payload - "# + "#, + queues = queue_and_stripes_sql("queue", "$1"), )) - .bind(queues) + .bind(queue) + .bind(after_job_id) + .bind(job_id_bound) .bind(DRAIN_BATCH_SIZE) .fetch_all(tx.as_mut()) .await .map_err(map_sqlx_error)?; - if deferred.is_empty() { - return Ok(0); - } + let Some(last_job_id) = deferred.iter().map(|row| row.job_id).max() else { + return Ok((0, after_job_id)); + }; let cancelled = deferred.len(); let (ready_slot, ready_generation) = self.current_queue_ring(tx).await?; @@ -8550,45 +8672,45 @@ impl QueueStorage { self.insert_done_rows_tx(tx, &rows, Some(old_state)).await?; } - Ok(cancelled) + Ok((cancelled, last_job_id)) } + /// Cancel up to [`DRAIN_BATCH_SIZE`] available jobs from `lanes`, + /// returning how many were cancelled and the claim-cursor advances that + /// skip them. async fn cancel_available_chunk_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, - queues: &[String], + lanes: &[DrainLane], ) -> Result<(usize, Vec), AwaError> { let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); // Unclaimed rows are found the way a claim finds them: from each // lane's claim cursor through its segments, then along the ready key // within each segment's ring slot. The ready key leads with the ring // slot, so a lane range without it scans every retained ready row. - // Only rows whose tombstone this statement inserts are cancelled, so a - // concurrent cancel of the same row cannot produce a second done row. + // Rows are locked before the limit applies, so a row another + // transaction holds does not shorten the chunk. Only rows whose + // tombstone this statement inserts are cancelled, so a concurrent + // cancel of the same row cannot produce a second done row. let ready: Vec = sqlx::query_as(&format!( r#" WITH heads AS ( SELECT - claims.queue, - claims.priority, - claims.enqueue_shard, - {schema}.sequence_next_value(claims.seq_name) AS claim_seq, - {schema}.sequence_next_value(enqueues.seq_name) AS next_seq - FROM {schema}.queue_claim_heads AS claims - JOIN {schema}.queue_enqueue_heads AS enqueues - ON enqueues.queue = claims.queue - AND enqueues.priority = claims.priority - AND enqueues.enqueue_shard = claims.enqueue_shard - WHERE claims.queue = ANY($1) + lanes.queue, + lanes.priority, + lanes.enqueue_shard, + GREATEST(lanes.from_seq, {schema}.sequence_next_value(claims.seq_name)) AS claim_seq, + lanes.to_seq AS next_seq + FROM unnest($1::text[], $2::smallint[], $3::smallint[], $4::bigint[], $5::bigint[]) + AS lanes(queue, priority, enqueue_shard, from_seq, to_seq) + JOIN {schema}.queue_claim_heads AS claims + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard ), - candidates AS ( - SELECT - ready.ready_slot, - ready.ready_generation, - ready.queue, - ready.priority, - ready.enqueue_shard, - ready.lane_seq + target AS ( + SELECT ready.* FROM heads CROSS JOIN LATERAL ( SELECT @@ -8605,13 +8727,7 @@ impl QueueStorage { ORDER BY segment.next_lane_seq ) AS segment CROSS JOIN LATERAL ( - SELECT - ready.ready_slot, - ready.ready_generation, - ready.queue, - ready.priority, - ready.enqueue_shard, - ready.lane_seq + SELECT ready.* FROM {schema}.ready_entries AS ready WHERE ready.ready_slot = segment.ready_slot AND ready.ready_generation = segment.ready_generation @@ -8620,31 +8736,20 @@ impl QueueStorage { AND ready.enqueue_shard = heads.enqueue_shard AND ready.lane_seq >= segment.from_seq AND ready.lane_seq < segment.to_seq + AND NOT EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + AND NOT ({claimed}) ORDER BY ready.lane_seq - OFFSET 0 + FOR UPDATE OF ready SKIP LOCKED ) AS ready - WHERE NOT EXISTS ( - SELECT 1 FROM {schema}.ready_tombstones AS tomb - WHERE tomb.ready_slot = ready.ready_slot - AND tomb.ready_generation = ready.ready_generation - AND tomb.queue = ready.queue - AND tomb.priority = ready.priority - AND tomb.enqueue_shard = ready.enqueue_shard - AND tomb.lane_seq = ready.lane_seq - ) - LIMIT $2 - ), - target AS ( - SELECT ready.* - FROM {schema}.ready_entries AS ready - JOIN candidates - ON candidates.ready_slot = ready.ready_slot - AND candidates.ready_generation = ready.ready_generation - AND candidates.queue = ready.queue - AND candidates.priority = ready.priority - AND candidates.enqueue_shard = ready.enqueue_shard - AND candidates.lane_seq = ready.lane_seq - FOR UPDATE OF ready SKIP LOCKED + LIMIT $6 ), tombstone AS ( INSERT INTO {schema}.ready_tombstones ( @@ -8684,7 +8789,11 @@ impl QueueStorage { AND tombstone.lane_seq = target.lane_seq "# )) - .bind(queues) + .bind(lanes.iter().map(|lane| lane.queue.as_str()).collect::>()) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind(lanes.iter().map(|lane| lane.enqueue_shard).collect::>()) + .bind(lanes.iter().map(|lane| lane.from_seq).collect::>()) + .bind(lanes.iter().map(|lane| lane.to_seq).collect::>()) .bind(DRAIN_BATCH_SIZE) .fetch_all(tx.as_mut()) .await @@ -8949,6 +9058,7 @@ impl QueueStorage { priority: Option, ) -> Result { let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" WITH target AS ( @@ -8969,6 +9079,7 @@ impl QueueStorage { AND tomb.ready_slot = ready.ready_slot AND tomb.ready_generation = ready.ready_generation ) + AND NOT ({claimed}) ORDER BY ready.lane_seq DESC LIMIT 1 FOR UPDATE OF ready SKIP LOCKED diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index 78739e950..c70c07c1c 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -14092,3 +14092,173 @@ async fn test_queue_storage_drain_queue_covers_every_stripe() { ); } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_covers_stripes_holding_only_scheduled_jobs() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_scheduled_stripes"; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: "awa_qs_drain_scheduled_stripes".to_string(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 4, + ..Default::default() + }, + ) + .await; + + let scheduled: Vec<_> = (0..40) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue striped scheduled"); + let stripes: i64 = sqlx::query_scalar(&format!( + "SELECT count(DISTINCT queue)::bigint FROM {}.deferred_jobs", + store.schema() + )) + .fetch_one(&pool) + .await + .expect("count scheduled stripes"); + assert!(stripes > 1, "scheduled jobs should spread across stripes"); + + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain striped queue"); + assert_eq!(drained, 40); + let remaining: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.deferred_jobs", + store.schema() + )) + .fetch_one(&pool) + .await + .expect("count remaining scheduled"); + assert_eq!(remaining, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_and_cancel_leave_claims_the_cursor_has_not_passed() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_claim_window"; + let store = create_store(&pool, "awa_qs_drain_claim_window").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 5) + .await + .expect("enqueue available"); + let (seq_name, cursor_before): (String, i64) = sqlx::query_as(&format!( + "SELECT seq_name, {schema}.sequence_next_value(seq_name) \ + FROM {schema}.queue_claim_heads WHERE queue = $1" + )) + .bind(queue) + .fetch_one(&pool) + .await + .expect("read claim cursor"); + let claimed = store + .claim_runtime_batch(&pool, queue, 2, Duration::from_secs(30)) + .await + .expect("claim two jobs"); + assert_eq!(claimed.len(), 2); + // The claim committed but its post-commit cursor advance has not run. + sqlx::query("SELECT setval(format('%I.%I', $1::text, $2::text)::regclass, $3, false)") + .bind(&schema) + .bind(&seq_name) + .bind(cursor_before) + .execute(&pool) + .await + .expect("rewind claim cursor"); + + let cancelled = store + .cancel_job(&pool, claimed[0].job.id) + .await + .expect("cancel a claimed job"); + assert!( + cancelled.is_none_or(|row| row.run_lease == claimed[0].job.run_lease), + "cancel must act on the running attempt, not the claimed ready row" + ); + let moved = store + .move_queue( + &pool, + claimed[1].job.id, + "qs_drain_claim_window_moved", + None, + ) + .await + .expect("move a claimed job"); + assert!(!moved, "a claimed job must not be re-enqueued elsewhere"); + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 3); + + store + .complete_runtime_batch(&pool, &claimed[1..]) + .await + .expect("complete the other claimed job"); + let done_states: Vec = sqlx::query_scalar(&format!( + "SELECT state::text FROM {schema}.done_entries WHERE job_id = $1" + )) + .bind(claimed[1].job.id) + .fetch_all(&pool) + .await + .expect("load done rows"); + assert_eq!(done_states, vec!["completed".to_string()]); + let duplicated: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM ( + SELECT job_id FROM {schema}.done_entries GROUP BY job_id HAVING count(*) > 1 + ) AS duplicated" + )) + .fetch_one(&pool) + .await + .expect("count duplicated done rows"); + assert_eq!(duplicated, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_skips_a_locked_row_without_stopping() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_locked_row"; + let store = create_store(&pool, "awa_qs_drain_locked_row").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 2_500) + .await + .expect("enqueue available backlog"); + let mut holder = pool.begin().await.expect("begin lock holder"); + let locked_id: i64 = sqlx::query_scalar(&format!( + "SELECT job_id FROM {schema}.ready_entries WHERE queue = $1 \ + ORDER BY lane_seq LIMIT 1 FOR UPDATE" + )) + .bind(queue) + .fetch_one(holder.as_mut()) + .await + .expect("lock the first ready row"); + + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 2_499); + holder.commit().await.expect("release lock"); + + let locked_job = store + .load_job(&pool, locked_id) + .await + .expect("load locked job") + .expect("locked job should exist"); + assert_eq!(locked_job.state, JobState::Available); + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain the released row"); + assert_eq!(drained, 1); +} From a1531abc111e4262d58bf441bfec7e9d3621b862 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Thu, 5 Nov 2026 06:36:13 +1300 Subject: [PATCH 3/3] fix(queue-storage): keep a drain off in-flight claims and move the cursor past it (0.6) A noisy-neighbour benchmark (a 200/s workload on one queue while another is drained, including a drain of a queue with live workers and a producer) found three problems in the bulk drain: - A claim records its attempt in the transaction that holds its lane's claim head, so a chunk read while that claim was in flight saw the row as available and cancelled a job a worker then ran. Each chunk now locks the claim heads of its lanes first, and cancel_job / move / cancel_jobs_by_ids lock the heads of the jobs' lanes, in the byte order cursor advances use. - The drain advanced a lane's cursor only if it still sat at the start of the cancelled run. With workers claiming, it rarely did, and claims then crossed the tombstoned range one claim batch per poll: about 64k jobs enqueued during a 100k drain were still unclaimed three minutes later. After each chunk the drain now moves the cursor over tombstoned rows and committed claims, under the claim-head lock. - Chunk statements were JIT-compiled (about 3 s each against milliseconds of execution), and the tombstone NOT EXISTS was planned as an anti join that re-read the lane's tombstones for every segment. Drain transactions turn JIT off, and the tombstone check is kept a per-row probe. At 100k available plus 100k scheduled the drain now takes 11 s quiet and 11 s with live workers (previously 340 s and 321 s), with no running job cancelled, no job left stranded and the neighbouring queue's latency at baseline. Backport of the follow-up on #598. --- CHANGELOG.md | 13 +- awa-model/src/queue_storage.rs | 338 +++++++++++++++++++----- awa/tests/queue_storage_runtime_test.rs | 58 ++++ 3 files changed, 339 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ce76b42..0172b8e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,10 @@ Patch release: batch completion and bulk "move failed to DLQ" on queue storage n one short transaction, with available rows read from each lane's claim cursor; waiting jobs are still cancelled one by one and running jobs are left alone. Jobs enqueued after the drain starts are left for a later - call, so draining a queue that keeps receiving work still returns. It also - matched only the logical queue name, so draining a striped queue cancelled + call, so draining a queue that keeps receiving work still returns, and + the drain moves each lane's claim cursor past the rows it cancelled, so + workers reach the jobs behind them on their next poll. It also matched + only the logical queue name, so draining a striped queue cancelled nothing; it now covers every stripe. `awa queue drain` and the Python `drain_queue` use this path. - **Cancelling, moving or reprioritising an available job on queue storage @@ -40,9 +42,10 @@ Patch release: batch completion and bulk "move failed to DLQ" on queue storage n briefly, or until the next claim if that advance failed, a claimed row still looked available. A cancel in that window wrote a cancelled terminal row while the attempt kept running, and a move re-enqueued a job that was - already executing. These paths now skip rows a claim has recorded, the - same evidence claims use to skip them, so a cancel reaches the running - attempt instead. + already executing. A claim still in progress was missed the same way. + These paths now wait for any claim in progress on the job's lane and skip + rows a claim has recorded, the same evidence claims use to skip them, so a + cancel reaches the running attempt instead. ## [0.6.11] — 2026-11-04 diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 17e6c0580..cdeb344de 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -26,6 +26,10 @@ const COPY_CHUNK_TARGET_BYTES: usize = 256 * 1024; const TERMINAL_COUNTER_BUCKETS: i16 = 256; const RECEIPT_RESCUE_BATCH_LIMIT: i64 = 500; const DRAIN_BATCH_SIZE: i64 = 1_000; +// The planner overestimates the drain statements enough to JIT-compile them, +// which takes seconds against milliseconds of execution. +const DRAIN_TRANSACTION_BEGIN: &str = "BEGIN; SET LOCAL jit = off"; +const DRAIN_CLAIM_CHECK_ROWS: i64 = 64; const RECEIPT_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const DEFAULT_PRUNE_LOCK_TIMEOUT: Duration = Duration::from_millis(10); @@ -7891,6 +7895,7 @@ impl QueueStorage { job_id: i64, ) -> Result, AwaError> { let schema = self.schema(); + self.lock_claim_heads_of_jobs_tx(tx, &[job_id]).await?; let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" @@ -8453,6 +8458,7 @@ impl QueueStorage { } let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + self.lock_claim_heads_of_jobs_tx(&mut tx, ids).await?; let mut rows = Vec::with_capacity(ids.len()); let mut claim_cursor_advances = Vec::new(); for job_id in ids { @@ -8490,7 +8496,10 @@ impl QueueStorage { let mut drained = 0u64; let mut after_job_id = i64::MIN; loop { - let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; let (cancelled, last_job_id) = self .cancel_deferred_chunk_tx(&mut tx, queue, after_job_id, job_id_bound) .await?; @@ -8527,29 +8536,23 @@ impl QueueStorage { .map_err(map_sqlx_error)?; lanes.retain(|lane| lane.from_seq < lane.to_seq); + let drained_lanes = lanes.clone(); while !lanes.is_empty() { - let mut tx = pool.begin().await.map_err(map_sqlx_error)?; - let (cancelled, advances) = self.cancel_available_chunk_tx(&mut tx, &lanes).await?; + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; + self.lock_claim_heads_tx(&mut tx, &lanes).await?; + let cancelled = self.cancel_available_chunk_tx(&mut tx, &mut lanes).await?; tx.commit().await.map_err(map_sqlx_error)?; - self.advance_claim_cursors(pool, &advances).await; + self.skip_spent_claim_prefixes(pool, &lanes).await?; drained += cancelled as u64; if (cancelled as i64) < DRAIN_BATCH_SIZE { break; } - // The next chunk resumes after the rows this one cancelled, so - // each lane is read once even when its claim cursor cannot move - // past a claimed or locked row. - for advance in &advances { - if let Some(lane) = lanes.iter_mut().find(|lane| { - lane.queue == advance.queue - && lane.priority == advance.priority - && lane.enqueue_shard == advance.enqueue_shard - }) { - lane.from_seq = lane.from_seq.max(advance.next_seq); - } - } lanes.retain(|lane| lane.from_seq < lane.to_seq); } + self.skip_spent_claim_prefixes(pool, &drained_lanes).await?; let waiting: Vec = sqlx::query_scalar(&format!( r#" @@ -8681,8 +8684,8 @@ impl QueueStorage { async fn cancel_available_chunk_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, - lanes: &[DrainLane], - ) -> Result<(usize, Vec), AwaError> { + lanes: &mut [DrainLane], + ) -> Result { let schema = self.schema(); let claimed = self.ready_row_claimed_sql("ready"); // Unclaimed rows are found the way a claim finds them: from each @@ -8690,7 +8693,11 @@ impl QueueStorage { // within each segment's ring slot. The ready key leads with the ring // slot, so a lane range without it scans every retained ready row. // Rows are locked before the limit applies, so a row another - // transaction holds does not shorten the chunk. Only rows whose + // transaction holds does not shorten the chunk. The tombstone check + // shares one NOT with the claim check: on its own, NOT EXISTS is + // planned as an anti join that reads the lane's tombstones again for + // every segment, and a lane enqueued one job at a time has a segment + // per row. Only rows whose // tombstone this statement inserts are cancelled, so a concurrent // cancel of the same row cannot produce a second done row. let ready: Vec = sqlx::query_as(&format!( @@ -8736,16 +8743,18 @@ impl QueueStorage { AND ready.enqueue_shard = heads.enqueue_shard AND ready.lane_seq >= segment.from_seq AND ready.lane_seq < segment.to_seq - AND NOT EXISTS ( - SELECT 1 FROM {schema}.ready_tombstones AS tomb - WHERE tomb.ready_slot = ready.ready_slot - AND tomb.ready_generation = ready.ready_generation - AND tomb.queue = ready.queue - AND tomb.priority = ready.priority - AND tomb.enqueue_shard = ready.enqueue_shard - AND tomb.lane_seq = ready.lane_seq + AND NOT ( + EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + OR {claimed} ) - AND NOT ({claimed}) ORDER BY ready.lane_seq FOR UPDATE OF ready SKIP LOCKED ) AS ready @@ -8800,46 +8809,19 @@ impl QueueStorage { .map_err(map_sqlx_error)?; if ready.is_empty() { - return Ok((0, Vec::new())); + return Ok(0); } - // One advance per run of consecutive cancelled positions on a lane: - // it applies only while the cursor sits at the start of the run, and - // every position it skips was tombstoned above. - let mut ready = ready; - ready.sort_unstable_by(|left, right| { - ( - &left.queue, - left.priority, - left.enqueue_shard, - left.lane_seq, - ) - .cmp(&( - &right.queue, - right.priority, - right.enqueue_shard, - right.lane_seq, - )) - }); - let mut advances: Vec = Vec::new(); + // The next chunk resumes after the rows this one cancelled, so each + // lane is read once even where a claimed or locked row stays behind. for row in &ready { - if let Some(last) = advances.last_mut() { - if last.queue == row.queue - && last.priority == row.priority - && last.enqueue_shard == row.enqueue_shard - && last.next_seq == row.lane_seq - { - last.next_seq = row.lane_seq + 1; - continue; - } + if let Some(lane) = lanes.iter_mut().find(|lane| { + lane.queue == row.queue + && lane.priority == row.priority + && lane.enqueue_shard == row.enqueue_shard + }) { + lane.from_seq = lane.from_seq.max(row.lane_seq + 1); } - advances.push(ClaimCursorAdvance { - queue: row.queue.clone(), - priority: row.priority, - enqueue_shard: row.enqueue_shard, - next_seq: row.lane_seq + 1, - only_if_current: Some(row.lane_seq), - }); } let finalized_at = Utc::now(); @@ -8853,7 +8835,232 @@ impl QueueStorage { self.insert_done_rows_tx(tx, &done_rows, Some(JobState::Available)) .await?; - Ok((done_rows.len(), advances)) + Ok(done_rows.len()) + } + + /// Lock the claim heads of the lanes holding ready rows of `job_ids`, so + /// the transaction's next statement sees any claim of those rows that was + /// in progress. See [`Self::lock_claim_heads_tx`]. + async fn lock_claim_heads_of_jobs_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + job_ids: &[i64], + ) -> Result<(), AwaError> { + let schema = self.schema(); + sqlx::query(&format!( + r#" + SELECT 1 + FROM {schema}.queue_claim_heads AS claims + WHERE (claims.queue, claims.priority, claims.enqueue_shard) IN ( + SELECT ready.queue, ready.priority, ready.enqueue_shard + FROM {schema}.ready_entries AS ready + WHERE ready.job_id = ANY($1) + ) + ORDER BY claims.queue COLLATE "C", claims.priority, claims.enqueue_shard + FOR UPDATE OF claims + "# + )) + .bind(job_ids) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(()) + } + + /// Lock the claim heads of `lanes`. A claim holds its lane's head until + /// it commits, so once this returns the transaction's next statement sees + /// every claim on these lanes. Heads are locked in the byte order that + /// cursor advances use. + async fn lock_claim_heads_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + lanes: &[DrainLane], + ) -> Result<(), AwaError> { + let schema = self.schema(); + sqlx::query(&format!( + r#" + SELECT 1 + FROM {schema}.queue_claim_heads AS claims + JOIN unnest($1::text[], $2::smallint[], $3::smallint[]) + AS lanes(queue, priority, enqueue_shard) + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard + ORDER BY claims.queue COLLATE "C", claims.priority, claims.enqueue_shard + FOR UPDATE OF claims + "# + )) + .bind( + lanes + .iter() + .map(|lane| lane.queue.as_str()) + .collect::>(), + ) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind( + lanes + .iter() + .map(|lane| lane.enqueue_shard) + .collect::>(), + ) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(()) + } + + /// Move each lane's claim cursor over cancelled and claimed rows, up to + /// the lane's `to_seq`. A claim crosses cancelled rows only one claim + /// batch per poll, so without this the jobs behind a drained range wait + /// for many polls before a worker reaches them. + async fn skip_spent_claim_prefixes( + &self, + pool: &PgPool, + lanes: &[DrainLane], + ) -> Result<(), AwaError> { + let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); + let mut lanes = lanes.to_vec(); + while !lanes.is_empty() { + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; + self.lock_claim_heads_tx(&mut tx, &lanes).await?; + // Each window is a range of positions, so a live row inside it is + // always seen. Untombstoned rows are checked for a claim only from + // the first, a few at a time: claims past the cursor are the few + // whose claimer has not yet moved it. The tombstone check is a + // column behind OFFSET 0, so it is not pushed down as a NOT EXISTS, + // for the reason given in `cancel_available_chunk_tx`. A gap before a visible row + // is an enqueue that rolled back: enqueues on a lane commit in + // position order. + let unfinished: Vec<(String, i16, i16)> = sqlx::query_as(&format!( + r#" + WITH windows AS ( + SELECT + lanes.queue, + lanes.priority, + lanes.enqueue_shard, + claims.seq_name, + {schema}.sequence_next_value(claims.seq_name) AS claim_seq, + LEAST({schema}.sequence_next_value(claims.seq_name) + $5, lanes.to_seq) AS window_end, + lanes.to_seq + FROM unnest($1::text[], $2::smallint[], $3::smallint[], $4::bigint[]) + AS lanes(queue, priority, enqueue_shard, to_seq) + JOIN {schema}.queue_claim_heads AS claims + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard + ), + scanned AS ( + SELECT + windows.*, + unclaimed.first_live, + COALESCE( + unclaimed.first_live, + CASE + WHEN unclaimed.checked = $6 THEN unclaimed.last_checked + 1 + ELSE visible.last_seen + 1 + END + ) AS next_seq + FROM windows + CROSS JOIN LATERAL ( + SELECT max(ready.lane_seq) AS last_seen + FROM {schema}.ready_segments AS segment + CROSS JOIN LATERAL ( + SELECT ready.lane_seq + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = windows.queue + AND ready.priority = windows.priority + AND ready.enqueue_shard = windows.enqueue_shard + AND ready.lane_seq >= GREATEST(segment.first_lane_seq, windows.claim_seq) + AND ready.lane_seq < LEAST(segment.next_lane_seq, windows.window_end) + ORDER BY ready.lane_seq + ) AS ready + WHERE segment.queue = windows.queue + AND segment.priority = windows.priority + AND segment.enqueue_shard = windows.enqueue_shard + AND segment.next_lane_seq > windows.claim_seq + AND segment.first_lane_seq < windows.window_end + ) AS visible + CROSS JOIN LATERAL ( + SELECT + min(ready.lane_seq) FILTER (WHERE NOT ({claimed})) AS first_live, + max(ready.lane_seq) AS last_checked, + count(*) AS checked + FROM ( + SELECT ready.* + FROM {schema}.ready_segments AS segment + CROSS JOIN LATERAL ( + SELECT + ready.*, + EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) AS tombstoned + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = windows.queue + AND ready.priority = windows.priority + AND ready.enqueue_shard = windows.enqueue_shard + AND ready.lane_seq >= GREATEST(segment.first_lane_seq, windows.claim_seq) + AND ready.lane_seq < LEAST(segment.next_lane_seq, windows.window_end) + ORDER BY ready.lane_seq + OFFSET 0 + ) AS ready + WHERE segment.queue = windows.queue + AND segment.priority = windows.priority + AND segment.enqueue_shard = windows.enqueue_shard + AND segment.next_lane_seq > windows.claim_seq + AND segment.first_lane_seq < windows.window_end + AND NOT ready.tombstoned + ORDER BY ready.lane_seq + LIMIT $6 + ) AS ready + ) AS unclaimed + WHERE windows.claim_seq < windows.to_seq + ), + advanced AS MATERIALIZED ( + SELECT + scanned.*, + {schema}.set_sequence_next(scanned.seq_name, scanned.next_seq) AS moved + FROM scanned + WHERE scanned.next_seq > scanned.claim_seq + ) + SELECT queue, priority, enqueue_shard + FROM advanced + WHERE first_live IS NULL + AND next_seq < to_seq + "# + )) + .bind(lanes.iter().map(|lane| lane.queue.as_str()).collect::>()) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind(lanes.iter().map(|lane| lane.enqueue_shard).collect::>()) + .bind(lanes.iter().map(|lane| lane.to_seq).collect::>()) + .bind(DRAIN_BATCH_SIZE) + .bind(DRAIN_CLAIM_CHECK_ROWS) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + tx.commit().await.map_err(map_sqlx_error)?; + lanes.retain(|lane| { + unfinished.iter().any(|(queue, priority, enqueue_shard)| { + &lane.queue == queue + && lane.priority == *priority + && lane.enqueue_shard == *enqueue_shard + }) + }); + } + Ok(()) } pub async fn set_priority( @@ -9058,6 +9265,7 @@ impl QueueStorage { priority: Option, ) -> Result { let schema = self.schema(); + self.lock_claim_heads_of_jobs_tx(tx, &[job_id]).await?; let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index c70c07c1c..183a77c3c 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -14224,6 +14224,64 @@ async fn test_queue_storage_drain_and_cancel_leave_claims_the_cursor_has_not_pas .await .expect("count duplicated done rows"); assert_eq!(duplicated, 0); + + store + .enqueue_batch(&pool, queue, 2, 1) + .await + .expect("enqueue after the drain"); + let next = store + .claim_runtime_batch(&pool, queue, 1, Duration::from_secs(30)) + .await + .expect("claim after the drain"); + assert_eq!( + next.len(), + 1, + "a job enqueued after a drain must be claimable on the next poll" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_waits_for_a_claim_in_flight() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_claim_in_flight"; + let store = create_store(&pool, "awa_qs_drain_claim_in_flight").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 5) + .await + .expect("enqueue available"); + let mut claim_tx = pool.begin().await.expect("begin claim"); + let claimed: Vec = sqlx::query_scalar(&format!( + "SELECT job_id FROM {schema}.claim_ready_runtime($1, 2, 0::float8, 0::float8)" + )) + .bind(queue) + .fetch_all(claim_tx.as_mut()) + .await + .expect("claim two jobs"); + assert_eq!(claimed.len(), 2); + + let drain = tokio::spawn({ + let pool = pool.clone(); + async move { admin::drain_queue(&pool, queue).await } + }); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + !drain.is_finished(), + "the drain must wait for the claim on its lane" + ); + claim_tx.commit().await.expect("commit claim"); + let drained = drain.await.expect("join drain").expect("drain queue"); + assert_eq!(drained, 3); + + let cancelled_claims: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {schema}.done_entries WHERE job_id = ANY($1)" + )) + .bind(&claimed) + .fetch_one(&pool) + .await + .expect("count done rows of claimed jobs"); + assert_eq!(cancelled_claims, 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)]