diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ffefa404..0172b8e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [0.6.12] — Unreleased -Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit. No migration and no public API change. +Patch release: batch completion and bulk "move failed to DLQ" on queue storage no longer fail once a batch grows past PostgreSQL's bind-parameter limit, draining a queue cancels its backlog in bulk, and cancelling or moving an available job no longer acts on one a worker has already claimed. No migration and no public API change. - **Completing a large batch and moving many failed jobs to the DLQ no longer fail on queue storage** (backport of the writer approach in @@ -21,6 +21,31 @@ Patch release: batch completion and bulk "move failed to DLQ" on queue storage n `AWA_COMPLETION_BATCH_SIZE` is raised above the default of 512. Both writers now send one array per column through a fixed-text `INSERT ... SELECT FROM unnest(...)`. +- **`admin::drain_queue` on queue storage cancels a backlog in bulk and + drains every stripe of a striped queue.** It found the queue's jobs through + a lookup that read every retained ready row, then cancelled them one at a + time, several statements each, inside one transaction, so draining a large + backlog took minutes and held its row locks throughout. Scheduled, + retryable and available jobs are now cancelled 1,000 at a time, each chunk + one short transaction, with available rows read from each lane's claim + cursor; waiting jobs are still cancelled one by one and running jobs are + left alone. Jobs enqueued after the drain starts are left for a later + call, so draining a queue that keeps receiving work still returns, and + the drain moves each lane's claim cursor past the rows it cancelled, so + workers reach the jobs behind them on their next poll. It also matched + only the logical queue name, so draining a striped queue cancelled + nothing; it now covers every stripe. `awa queue drain` and the Python + `drain_queue` use this path. +- **Cancelling, moving or reprioritising an available job on queue storage + no longer acts on a job a worker has already claimed.** A claim moves its + lane's claim cursor past the rows it took only after it commits, so + briefly, or until the next claim if that advance failed, a claimed row + still looked available. A cancel in that window wrote a cancelled terminal + row while the attempt kept running, and a move re-enqueued a job that was + already executing. A claim still in progress was missed the same way. + These paths now wait for any claim in progress on the job's lane and skip + rows a claim has recorded, the same evidence claims use to skip them, so a + cancel reaches the running attempt instead. ## [0.6.11] — 2026-11-04 diff --git a/awa-model/src/admin.rs b/awa-model/src/admin.rs index fd16120bf..baf0a3db4 100644 --- a/awa-model/src/admin.rs +++ b/awa-model/src/admin.rs @@ -1629,22 +1629,12 @@ where } /// Drain a queue: cancel all non-running, non-terminal jobs. +/// +/// On queue storage this also drains the queue's stripes: queues named +/// `#`. Jobs enqueued after the drain starts are left alone. pub async fn drain_queue(pool: &PgPool, queue: &str) -> Result { if let Some(store) = active_queue_storage(pool).await? { - let sql = format!( - "{} \ - SELECT job_id \ - FROM current_jobs \ - WHERE queue = $1 \ - AND state IN ('available', 'scheduled', 'retryable', 'waiting_external') \ - ORDER BY job_id ASC", - queue_storage_current_jobs_cte(store.schema()) - ); - let ids: Vec = sqlx::query_scalar(&sql).bind(queue).fetch_all(pool).await?; - return store - .cancel_jobs_by_ids(pool, &ids) - .await - .map(|rows| rows.len() as u64); + return store.drain_queue(pool, queue).await; } let result = sqlx::query( diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 42c59ede2..cdeb344de 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -25,6 +25,11 @@ const COPY_NULL_SENTINEL: &str = "__AWA_NULL__"; const COPY_CHUNK_TARGET_BYTES: usize = 256 * 1024; const TERMINAL_COUNTER_BUCKETS: i16 = 256; const RECEIPT_RESCUE_BATCH_LIMIT: i64 = 500; +const DRAIN_BATCH_SIZE: i64 = 1_000; +// The planner overestimates the drain statements enough to JIT-compile them, +// which takes seconds against milliseconds of execution. +const DRAIN_TRANSACTION_BEGIN: &str = "BEGIN; SET LOCAL jit = off"; +const DRAIN_CLAIM_CHECK_ROWS: i64 = 64; const RECEIPT_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const DEFAULT_PRUNE_LOCK_TIMEOUT: Duration = Duration::from_millis(10); @@ -530,6 +535,16 @@ fn claim_child_name(schema: &str, slot: usize) -> String { format!("{schema}.lease_claims_{slot}") } +/// SQL true when `column` names queue `param` or one of its stripes +/// (`queue#N`). +fn queue_and_stripes_sql(column: &str, param: &str) -> String { + format!( + "({column} = {param} \ + OR (left({column}, char_length({param}) + 1) = {param} || '{QUEUE_STRIPE_DELIMITER}' \ + AND substr({column}, char_length({param}) + 2) ~ '^[0-9]+$'))" + ) +} + fn claim_batch_child_name(schema: &str, slot: usize) -> String { format!("{schema}.lease_claim_batches_{slot}") } @@ -1563,6 +1578,15 @@ struct ReadyTransitionRow { payload: serde_json::Value, } +#[derive(Debug, Clone, sqlx::FromRow)] +struct DrainLane { + queue: String, + priority: i16, + enqueue_shard: i16, + from_seq: i64, + to_seq: i64, +} + #[derive(Debug, Clone)] struct ClaimCursorAdvance { queue: String, @@ -7825,12 +7849,54 @@ impl QueueStorage { Ok(()) } + /// SQL true when the ready row `alias` has been claimed. A claim records + /// its attempt in its own transaction but moves the lane's claim cursor + /// past the row only after commit, so a claimed row can still sit at or + /// after the cursor; claims skip such rows on the same evidence. Receipt + /// claims leave an attempt batch, and materialized claims a later attempt + /// of the job, which is checked whatever this store's mode, since admin + /// callers build the store with default configuration. + fn ready_row_claimed_sql(&self, alias: &str) -> String { + let schema = self.schema(); + let mut sql = format!( + "EXISTS ( + SELECT 1 FROM {schema}.ready_claim_attempt_batches AS attempts + WHERE attempts.ready_slot = {alias}.ready_slot + AND attempts.ready_generation = {alias}.ready_generation + AND attempts.queue = {alias}.queue + AND attempts.priority = {alias}.priority + AND attempts.enqueue_shard = {alias}.enqueue_shard + AND attempts.first_lane_seq <= {alias}.lane_seq + AND attempts.next_lane_seq > {alias}.lane_seq + AND attempts.lane_ranges @> int8range({alias}.lane_seq, {alias}.lane_seq + 1, '[)') + )" + ); + for table in [ + "leases", + "done_entries", + "deferred_jobs", + "dlq_entries", + "ready_entries", + ] { + sql.push_str(&format!( + " OR EXISTS ( + SELECT 1 FROM {schema}.{table} AS attempt + WHERE attempt.job_id = {alias}.job_id + AND attempt.run_lease = {alias}.run_lease + 1 + )" + )); + } + sql + } + async fn cancel_job_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, job_id: i64, ) -> Result, AwaError> { let schema = self.schema(); + self.lock_claim_heads_of_jobs_tx(tx, &[job_id]).await?; + let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" WITH target AS ( @@ -7851,6 +7917,7 @@ impl QueueStorage { AND tomb.ready_slot = ready.ready_slot AND tomb.ready_generation = ready.ready_generation ) + AND NOT ({claimed}) ORDER BY ready.lane_seq DESC LIMIT 1 FOR UPDATE OF ready SKIP LOCKED @@ -8391,6 +8458,7 @@ impl QueueStorage { } let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + self.lock_claim_heads_of_jobs_tx(&mut tx, ids).await?; let mut rows = Vec::with_capacity(ids.len()); let mut claim_cursor_advances = Vec::new(); for job_id in ids { @@ -8407,6 +8475,594 @@ impl QueueStorage { Ok(rows) } + /// Cancel every available, scheduled, retryable and waiting job in + /// `queue` and its stripes (`queue#N`), returning how many were cancelled. + /// + /// Jobs enqueued after the drain starts are left alone, and so are running + /// jobs. Scheduled, retryable and available jobs are cancelled a chunk at + /// a time, each chunk in its own transaction, so a large backlog costs a + /// few statements per chunk rather than per job and holds row locks for + /// one chunk at a time. Rows another transaction has locked are skipped. + /// Waiting jobs go through [`Self::cancel_jobs_by_ids`]. + pub(crate) async fn drain_queue(&self, pool: &PgPool, queue: &str) -> Result { + let schema = self.schema(); + let job_id_bound: i64 = sqlx::query_scalar(&format!( + "SELECT {schema}.sequence_next_value('job_id_seq')" + )) + .fetch_one(pool) + .await + .map_err(map_sqlx_error)?; + + let mut drained = 0u64; + let mut after_job_id = i64::MIN; + loop { + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; + let (cancelled, last_job_id) = self + .cancel_deferred_chunk_tx(&mut tx, queue, after_job_id, job_id_bound) + .await?; + tx.commit().await.map_err(map_sqlx_error)?; + drained += cancelled as u64; + if (cancelled as i64) < DRAIN_BATCH_SIZE { + break; + } + after_job_id = last_job_id; + } + + // Read after the deferred pass, so jobs promoted while it ran are + // drained from the ready lanes. + let mut lanes: Vec = sqlx::query_as(&format!( + r#" + SELECT + claims.queue, + claims.priority, + claims.enqueue_shard, + {schema}.sequence_next_value(claims.seq_name) AS from_seq, + {schema}.sequence_next_value(enqueues.seq_name) AS to_seq + FROM {schema}.queue_claim_heads AS claims + JOIN {schema}.queue_enqueue_heads AS enqueues + ON enqueues.queue = claims.queue + AND enqueues.priority = claims.priority + AND enqueues.enqueue_shard = claims.enqueue_shard + WHERE {queues} + "#, + queues = queue_and_stripes_sql("claims.queue", "$1"), + )) + .bind(queue) + .fetch_all(pool) + .await + .map_err(map_sqlx_error)?; + lanes.retain(|lane| lane.from_seq < lane.to_seq); + + let drained_lanes = lanes.clone(); + while !lanes.is_empty() { + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; + self.lock_claim_heads_tx(&mut tx, &lanes).await?; + let cancelled = self.cancel_available_chunk_tx(&mut tx, &mut lanes).await?; + tx.commit().await.map_err(map_sqlx_error)?; + self.skip_spent_claim_prefixes(pool, &lanes).await?; + drained += cancelled as u64; + if (cancelled as i64) < DRAIN_BATCH_SIZE { + break; + } + lanes.retain(|lane| lane.from_seq < lane.to_seq); + } + self.skip_spent_claim_prefixes(pool, &drained_lanes).await?; + + let waiting: Vec = sqlx::query_scalar(&format!( + r#" + SELECT job_id + FROM {schema}.leases + WHERE {queues} + AND state = 'waiting_external' + AND job_id < $2 + ORDER BY job_id + "#, + queues = queue_and_stripes_sql("queue", "$1"), + )) + .bind(queue) + .bind(job_id_bound) + .fetch_all(pool) + .await + .map_err(map_sqlx_error)?; + drained += self.cancel_jobs_by_ids(pool, &waiting).await?.len() as u64; + + Ok(drained) + } + + /// Cancel up to [`DRAIN_BATCH_SIZE`] scheduled and retryable jobs of + /// `queue` with ids in `(after_job_id, job_id_bound)`, returning how many + /// were cancelled and the largest cancelled id. + async fn cancel_deferred_chunk_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + queue: &str, + after_job_id: i64, + job_id_bound: i64, + ) -> Result<(usize, i64), AwaError> { + let schema = self.schema(); + let deferred: Vec = sqlx::query_as(&format!( + r#" + DELETE FROM {schema}.deferred_jobs + WHERE job_id IN ( + SELECT job_id + FROM {schema}.deferred_jobs + WHERE {queues} + AND state IN ('scheduled', 'retryable') + AND job_id > $2 + AND job_id < $3 + ORDER BY job_id + LIMIT $4 + FOR UPDATE SKIP LOCKED + ) + RETURNING + job_id, + kind, + queue, + args, + state, + priority, + attempt, + run_lease, + max_attempts, + run_at, + attempted_at, + finalized_at, + created_at, + unique_key, + unique_states, + COALESCE(payload, '{{}}'::jsonb) AS payload + "#, + queues = queue_and_stripes_sql("queue", "$1"), + )) + .bind(queue) + .bind(after_job_id) + .bind(job_id_bound) + .bind(DRAIN_BATCH_SIZE) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + + let Some(last_job_id) = deferred.iter().map(|row| row.job_id).max() else { + return Ok((0, after_job_id)); + }; + + let cancelled = deferred.len(); + let (ready_slot, ready_generation) = self.current_queue_ring(tx).await?; + let lanes: BTreeSet<(String, i16)> = deferred + .iter() + .map(|row| (row.queue.clone(), row.priority)) + .collect(); + for (queue, priority) in &lanes { + self.ensure_lane(tx, queue, *priority, 0).await?; + } + + let finalized_at = Utc::now(); + let mut by_state: HashMap> = HashMap::new(); + for deferred in deferred { + // Same shape as a single deferred cancel: shard 0 and a negative + // lane_seq keep the done_entries key unique. + let done = DoneJobRow { + ready_slot, + ready_generation, + job_id: deferred.job_id, + kind: deferred.kind, + queue: deferred.queue, + args: deferred.args, + state: JobState::Cancelled, + priority: deferred.priority, + attempt: deferred.attempt, + run_lease: deferred.run_lease, + max_attempts: deferred.max_attempts, + lane_seq: -deferred.job_id, + enqueue_shard: 0, + run_at: deferred.run_at, + attempted_at: deferred.attempted_at, + finalized_at, + created_at: deferred.created_at, + unique_key: deferred.unique_key, + unique_states: deferred.unique_states, + payload: deferred.payload, + }; + by_state.entry(deferred.state).or_default().push(done); + } + for (old_state, rows) in by_state { + self.insert_done_rows_tx(tx, &rows, Some(old_state)).await?; + } + + Ok((cancelled, last_job_id)) + } + + /// Cancel up to [`DRAIN_BATCH_SIZE`] available jobs from `lanes`, + /// returning how many were cancelled and the claim-cursor advances that + /// skip them. + async fn cancel_available_chunk_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + lanes: &mut [DrainLane], + ) -> Result { + let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); + // Unclaimed rows are found the way a claim finds them: from each + // lane's claim cursor through its segments, then along the ready key + // within each segment's ring slot. The ready key leads with the ring + // slot, so a lane range without it scans every retained ready row. + // Rows are locked before the limit applies, so a row another + // transaction holds does not shorten the chunk. The tombstone check + // shares one NOT with the claim check: on its own, NOT EXISTS is + // planned as an anti join that reads the lane's tombstones again for + // every segment, and a lane enqueued one job at a time has a segment + // per row. Only rows whose + // tombstone this statement inserts are cancelled, so a concurrent + // cancel of the same row cannot produce a second done row. + let ready: Vec = sqlx::query_as(&format!( + r#" + WITH heads AS ( + SELECT + lanes.queue, + lanes.priority, + lanes.enqueue_shard, + GREATEST(lanes.from_seq, {schema}.sequence_next_value(claims.seq_name)) AS claim_seq, + lanes.to_seq AS next_seq + FROM unnest($1::text[], $2::smallint[], $3::smallint[], $4::bigint[], $5::bigint[]) + AS lanes(queue, priority, enqueue_shard, from_seq, to_seq) + JOIN {schema}.queue_claim_heads AS claims + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard + ), + target AS ( + SELECT ready.* + FROM heads + CROSS JOIN LATERAL ( + SELECT + segment.ready_slot, + segment.ready_generation, + GREATEST(segment.first_lane_seq, heads.claim_seq) AS from_seq, + LEAST(segment.next_lane_seq, heads.next_seq) AS to_seq + FROM {schema}.ready_segments AS segment + WHERE segment.queue = heads.queue + AND segment.priority = heads.priority + AND segment.enqueue_shard = heads.enqueue_shard + AND segment.next_lane_seq > heads.claim_seq + AND segment.first_lane_seq < heads.next_seq + ORDER BY segment.next_lane_seq + ) AS segment + CROSS JOIN LATERAL ( + SELECT ready.* + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = heads.queue + AND ready.priority = heads.priority + AND ready.enqueue_shard = heads.enqueue_shard + AND ready.lane_seq >= segment.from_seq + AND ready.lane_seq < segment.to_seq + AND NOT ( + EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + OR {claimed} + ) + ORDER BY ready.lane_seq + FOR UPDATE OF ready SKIP LOCKED + ) AS ready + LIMIT $6 + ), + tombstone AS ( + INSERT INTO {schema}.ready_tombstones ( + ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq, job_id + ) + SELECT ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq, job_id + FROM target + ON CONFLICT DO NOTHING + RETURNING ready_slot, ready_generation, queue, priority, enqueue_shard, lane_seq + ) + SELECT + target.ready_slot, + target.ready_generation, + target.job_id, + target.kind, + target.queue, + target.args, + target.priority, + target.attempt, + target.run_lease, + target.max_attempts, + target.lane_seq, + target.enqueue_shard, + target.run_at, + target.attempted_at, + target.created_at, + target.unique_key, + target.unique_states, + COALESCE(target.payload, '{{}}'::jsonb) AS payload + FROM target + JOIN tombstone + ON tombstone.ready_slot = target.ready_slot + AND tombstone.ready_generation = target.ready_generation + AND tombstone.queue = target.queue + AND tombstone.priority = target.priority + AND tombstone.enqueue_shard = target.enqueue_shard + AND tombstone.lane_seq = target.lane_seq + "# + )) + .bind(lanes.iter().map(|lane| lane.queue.as_str()).collect::>()) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind(lanes.iter().map(|lane| lane.enqueue_shard).collect::>()) + .bind(lanes.iter().map(|lane| lane.from_seq).collect::>()) + .bind(lanes.iter().map(|lane| lane.to_seq).collect::>()) + .bind(DRAIN_BATCH_SIZE) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + + if ready.is_empty() { + return Ok(0); + } + + // The next chunk resumes after the rows this one cancelled, so each + // lane is read once even where a claimed or locked row stays behind. + for row in &ready { + if let Some(lane) = lanes.iter_mut().find(|lane| { + lane.queue == row.queue + && lane.priority == row.priority + && lane.enqueue_shard == row.enqueue_shard + }) { + lane.from_seq = lane.from_seq.max(row.lane_seq + 1); + } + } + + let finalized_at = Utc::now(); + let done_rows: Vec = ready + .into_iter() + .map(|row| { + let payload = row.payload.clone(); + row.into_done_row(JobState::Cancelled, finalized_at, payload) + }) + .collect(); + self.insert_done_rows_tx(tx, &done_rows, Some(JobState::Available)) + .await?; + + Ok(done_rows.len()) + } + + /// Lock the claim heads of the lanes holding ready rows of `job_ids`, so + /// the transaction's next statement sees any claim of those rows that was + /// in progress. See [`Self::lock_claim_heads_tx`]. + async fn lock_claim_heads_of_jobs_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + job_ids: &[i64], + ) -> Result<(), AwaError> { + let schema = self.schema(); + sqlx::query(&format!( + r#" + SELECT 1 + FROM {schema}.queue_claim_heads AS claims + WHERE (claims.queue, claims.priority, claims.enqueue_shard) IN ( + SELECT ready.queue, ready.priority, ready.enqueue_shard + FROM {schema}.ready_entries AS ready + WHERE ready.job_id = ANY($1) + ) + ORDER BY claims.queue COLLATE "C", claims.priority, claims.enqueue_shard + FOR UPDATE OF claims + "# + )) + .bind(job_ids) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(()) + } + + /// Lock the claim heads of `lanes`. A claim holds its lane's head until + /// it commits, so once this returns the transaction's next statement sees + /// every claim on these lanes. Heads are locked in the byte order that + /// cursor advances use. + async fn lock_claim_heads_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + lanes: &[DrainLane], + ) -> Result<(), AwaError> { + let schema = self.schema(); + sqlx::query(&format!( + r#" + SELECT 1 + FROM {schema}.queue_claim_heads AS claims + JOIN unnest($1::text[], $2::smallint[], $3::smallint[]) + AS lanes(queue, priority, enqueue_shard) + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard + ORDER BY claims.queue COLLATE "C", claims.priority, claims.enqueue_shard + FOR UPDATE OF claims + "# + )) + .bind( + lanes + .iter() + .map(|lane| lane.queue.as_str()) + .collect::>(), + ) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind( + lanes + .iter() + .map(|lane| lane.enqueue_shard) + .collect::>(), + ) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(()) + } + + /// Move each lane's claim cursor over cancelled and claimed rows, up to + /// the lane's `to_seq`. A claim crosses cancelled rows only one claim + /// batch per poll, so without this the jobs behind a drained range wait + /// for many polls before a worker reaches them. + async fn skip_spent_claim_prefixes( + &self, + pool: &PgPool, + lanes: &[DrainLane], + ) -> Result<(), AwaError> { + let schema = self.schema(); + let claimed = self.ready_row_claimed_sql("ready"); + let mut lanes = lanes.to_vec(); + while !lanes.is_empty() { + let mut tx = pool + .begin_with(DRAIN_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; + self.lock_claim_heads_tx(&mut tx, &lanes).await?; + // Each window is a range of positions, so a live row inside it is + // always seen. Untombstoned rows are checked for a claim only from + // the first, a few at a time: claims past the cursor are the few + // whose claimer has not yet moved it. The tombstone check is a + // column behind OFFSET 0, so it is not pushed down as a NOT EXISTS, + // for the reason given in `cancel_available_chunk_tx`. A gap before a visible row + // is an enqueue that rolled back: enqueues on a lane commit in + // position order. + let unfinished: Vec<(String, i16, i16)> = sqlx::query_as(&format!( + r#" + WITH windows AS ( + SELECT + lanes.queue, + lanes.priority, + lanes.enqueue_shard, + claims.seq_name, + {schema}.sequence_next_value(claims.seq_name) AS claim_seq, + LEAST({schema}.sequence_next_value(claims.seq_name) + $5, lanes.to_seq) AS window_end, + lanes.to_seq + FROM unnest($1::text[], $2::smallint[], $3::smallint[], $4::bigint[]) + AS lanes(queue, priority, enqueue_shard, to_seq) + JOIN {schema}.queue_claim_heads AS claims + ON claims.queue = lanes.queue + AND claims.priority = lanes.priority + AND claims.enqueue_shard = lanes.enqueue_shard + ), + scanned AS ( + SELECT + windows.*, + unclaimed.first_live, + COALESCE( + unclaimed.first_live, + CASE + WHEN unclaimed.checked = $6 THEN unclaimed.last_checked + 1 + ELSE visible.last_seen + 1 + END + ) AS next_seq + FROM windows + CROSS JOIN LATERAL ( + SELECT max(ready.lane_seq) AS last_seen + FROM {schema}.ready_segments AS segment + CROSS JOIN LATERAL ( + SELECT ready.lane_seq + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = windows.queue + AND ready.priority = windows.priority + AND ready.enqueue_shard = windows.enqueue_shard + AND ready.lane_seq >= GREATEST(segment.first_lane_seq, windows.claim_seq) + AND ready.lane_seq < LEAST(segment.next_lane_seq, windows.window_end) + ORDER BY ready.lane_seq + ) AS ready + WHERE segment.queue = windows.queue + AND segment.priority = windows.priority + AND segment.enqueue_shard = windows.enqueue_shard + AND segment.next_lane_seq > windows.claim_seq + AND segment.first_lane_seq < windows.window_end + ) AS visible + CROSS JOIN LATERAL ( + SELECT + min(ready.lane_seq) FILTER (WHERE NOT ({claimed})) AS first_live, + max(ready.lane_seq) AS last_checked, + count(*) AS checked + FROM ( + SELECT ready.* + FROM {schema}.ready_segments AS segment + CROSS JOIN LATERAL ( + SELECT + ready.*, + EXISTS ( + SELECT 1 FROM {schema}.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) AS tombstoned + FROM {schema}.ready_entries AS ready + WHERE ready.ready_slot = segment.ready_slot + AND ready.ready_generation = segment.ready_generation + AND ready.queue = windows.queue + AND ready.priority = windows.priority + AND ready.enqueue_shard = windows.enqueue_shard + AND ready.lane_seq >= GREATEST(segment.first_lane_seq, windows.claim_seq) + AND ready.lane_seq < LEAST(segment.next_lane_seq, windows.window_end) + ORDER BY ready.lane_seq + OFFSET 0 + ) AS ready + WHERE segment.queue = windows.queue + AND segment.priority = windows.priority + AND segment.enqueue_shard = windows.enqueue_shard + AND segment.next_lane_seq > windows.claim_seq + AND segment.first_lane_seq < windows.window_end + AND NOT ready.tombstoned + ORDER BY ready.lane_seq + LIMIT $6 + ) AS ready + ) AS unclaimed + WHERE windows.claim_seq < windows.to_seq + ), + advanced AS MATERIALIZED ( + SELECT + scanned.*, + {schema}.set_sequence_next(scanned.seq_name, scanned.next_seq) AS moved + FROM scanned + WHERE scanned.next_seq > scanned.claim_seq + ) + SELECT queue, priority, enqueue_shard + FROM advanced + WHERE first_live IS NULL + AND next_seq < to_seq + "# + )) + .bind(lanes.iter().map(|lane| lane.queue.as_str()).collect::>()) + .bind(lanes.iter().map(|lane| lane.priority).collect::>()) + .bind(lanes.iter().map(|lane| lane.enqueue_shard).collect::>()) + .bind(lanes.iter().map(|lane| lane.to_seq).collect::>()) + .bind(DRAIN_BATCH_SIZE) + .bind(DRAIN_CLAIM_CHECK_ROWS) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + tx.commit().await.map_err(map_sqlx_error)?; + lanes.retain(|lane| { + unfinished.iter().any(|(queue, priority, enqueue_shard)| { + &lane.queue == queue + && lane.priority == *priority + && lane.enqueue_shard == *enqueue_shard + }) + }); + } + Ok(()) + } + pub async fn set_priority( &self, pool: &PgPool, @@ -8609,6 +9265,8 @@ impl QueueStorage { priority: Option, ) -> Result { let schema = self.schema(); + self.lock_claim_heads_of_jobs_tx(tx, &[job_id]).await?; + let claimed = self.ready_row_claimed_sql("ready"); let ready: Option = sqlx::query_as(&format!( r#" WITH target AS ( @@ -8629,6 +9287,7 @@ impl QueueStorage { AND tomb.ready_slot = ready.ready_slot AND tomb.ready_generation = ready.ready_generation ) + AND NOT ({claimed}) ORDER BY ready.lane_seq DESC LIMIT 1 FOR UPDATE OF ready SKIP LOCKED diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index c4235d75e..183a77c3c 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -13854,3 +13854,469 @@ async fn test_queue_storage_cancel_by_unique_key_reads_live_work_only() { {scoped_finished_claims} with retained claims" ); } + +async fn queue_deferred_count(pool: &sqlx::PgPool, store: &QueueStorage, queue: &str) -> i64 { + sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.deferred_jobs WHERE queue = $1", + store.schema() + )) + .bind(queue) + .fetch_one(pool) + .await + .expect("count deferred jobs") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_cancels_a_backlog_larger_than_one_chunk() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_backlog"; + let other_queue = "qs_drain_other"; + let store = create_store(&pool, "awa_qs_drain_backlog").await; + + store + .enqueue_batch(&pool, queue, 2, 3_000) + .await + .expect("enqueue history"); + loop { + let claimed = store + .claim_batch(&pool, queue, 1_000) + .await + .expect("claim history"); + if claimed.is_empty() { + break; + } + store + .complete_batch(&pool, &claimed) + .await + .expect("complete history"); + } + + store + .enqueue_batch(&pool, queue, 2, 3) + .await + .expect("enqueue retry jobs"); + let to_retry = store + .claim_runtime_batch(&pool, queue, 3, Duration::from_secs(30)) + .await + .expect("claim retry jobs"); + assert_eq!(to_retry.len(), 3); + for claimed in &to_retry { + store + .retry_after( + &pool, + claimed.job.id, + claimed.job.run_lease, + Duration::from_secs(3_600), + None, + ) + .await + .expect("retry job") + .expect("running job should become retryable"); + } + + store + .enqueue_batch(&pool, queue, 2, 1) + .await + .expect("enqueue running job"); + let running = store + .claim_runtime_batch(&pool, queue, 1, Duration::from_secs(30)) + .await + .expect("claim running job"); + assert_eq!(running.len(), 1); + let running_id = running[0].job.id; + + store + .enqueue_batch(&pool, queue, 2, 2_500) + .await + .expect("enqueue available backlog"); + let scheduled: Vec<_> = (0..2_500) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue scheduled backlog"); + let unique_opts = InsertOpts { + queue: queue.to_string(), + unique: Some(UniqueOpts::default()), + ..Default::default() + }; + for id in 0..3 { + enqueue_job( + &pool, + &store, + &CompleteJob { id: 900_000 + id }, + unique_opts.clone(), + ) + .await; + } + + store + .enqueue_batch(&pool, other_queue, 2, 10) + .await + .expect("enqueue other available"); + let other_scheduled: Vec<_> = (0..10) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: other_queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("other scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &other_scheduled) + .await + .expect("enqueue other scheduled"); + + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 3 + 2_500 + 2_500 + 3); + + let counts = store + .queue_counts(&pool, queue) + .await + .expect("queue counts after drain"); + assert_eq!(counts.available, 0); + assert_eq!(counts.running, 1); + assert_eq!(queue_deferred_count(&pool, &store, queue).await, 0); + let dispatcher_counts = store + .queue_counts_fast(&pool, queue) + .await + .expect("dispatcher counts after drain"); + assert_eq!( + dispatcher_counts.available, 0, + "the claim cursor should move past the drained rows" + ); + let cancelled: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.done_entries WHERE queue = $1 AND state = 'cancelled'", + store.schema() + )) + .bind(queue) + .fetch_one(&pool) + .await + .expect("count cancelled jobs"); + assert_eq!(cancelled, drained as i64); + let running_job = store + .load_job(&pool, running_id) + .await + .expect("load running job") + .expect("running job should exist"); + assert_eq!(running_job.state, JobState::Running); + + let other_counts = store + .queue_counts(&pool, other_queue) + .await + .expect("other queue counts"); + assert_eq!(other_counts.available, 10); + assert_eq!(queue_deferred_count(&pool, &store, other_queue).await, 10); + + // The drained unique jobs released their keys, and the lane's claim + // cursor moved past the cancelled rows. + let requeued = enqueue_job(&pool, &store, &CompleteJob { id: 900_000 }, unique_opts).await; + let claimed = store + .claim_runtime_batch(&pool, queue, 10, Duration::from_secs(30)) + .await + .expect("claim after drain"); + let claimed_ids: Vec = claimed.iter().map(|claimed| claimed.job.id).collect(); + assert_eq!(claimed_ids, vec![requeued]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_covers_every_stripe() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_striped"; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: "awa_qs_drain_striped".to_string(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 2, + lease_claim_receipts: false, + ..Default::default() + }, + ) + .await; + + store + .enqueue_batch(&pool, queue, 2, 20) + .await + .expect("enqueue striped available"); + let scheduled: Vec<_> = (0..20) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue striped scheduled"); + + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain striped queue"); + assert_eq!(drained, 40); + for stripe in 0..2 { + let physical = format!("{queue}#{stripe}"); + let counts = store + .queue_counts(&pool, &physical) + .await + .expect("stripe counts"); + assert_eq!(counts.available, 0, "{physical} available"); + assert_eq!( + queue_deferred_count(&pool, &store, &physical).await, + 0, + "{physical} deferred" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_covers_stripes_holding_only_scheduled_jobs() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_scheduled_stripes"; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: "awa_qs_drain_scheduled_stripes".to_string(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 4, + ..Default::default() + }, + ) + .await; + + let scheduled: Vec<_> = (0..40) + .map(|id| { + insert::params_with( + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + run_at: Some(Utc::now() + chrono::Duration::hours(1)), + ..Default::default() + }, + ) + .expect("scheduled params") + }) + .collect(); + store + .enqueue_params_batch(&pool, &scheduled) + .await + .expect("enqueue striped scheduled"); + let stripes: i64 = sqlx::query_scalar(&format!( + "SELECT count(DISTINCT queue)::bigint FROM {}.deferred_jobs", + store.schema() + )) + .fetch_one(&pool) + .await + .expect("count scheduled stripes"); + assert!(stripes > 1, "scheduled jobs should spread across stripes"); + + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain striped queue"); + assert_eq!(drained, 40); + let remaining: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {}.deferred_jobs", + store.schema() + )) + .fetch_one(&pool) + .await + .expect("count remaining scheduled"); + assert_eq!(remaining, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_and_cancel_leave_claims_the_cursor_has_not_passed() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_claim_window"; + let store = create_store(&pool, "awa_qs_drain_claim_window").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 5) + .await + .expect("enqueue available"); + let (seq_name, cursor_before): (String, i64) = sqlx::query_as(&format!( + "SELECT seq_name, {schema}.sequence_next_value(seq_name) \ + FROM {schema}.queue_claim_heads WHERE queue = $1" + )) + .bind(queue) + .fetch_one(&pool) + .await + .expect("read claim cursor"); + let claimed = store + .claim_runtime_batch(&pool, queue, 2, Duration::from_secs(30)) + .await + .expect("claim two jobs"); + assert_eq!(claimed.len(), 2); + // The claim committed but its post-commit cursor advance has not run. + sqlx::query("SELECT setval(format('%I.%I', $1::text, $2::text)::regclass, $3, false)") + .bind(&schema) + .bind(&seq_name) + .bind(cursor_before) + .execute(&pool) + .await + .expect("rewind claim cursor"); + + let cancelled = store + .cancel_job(&pool, claimed[0].job.id) + .await + .expect("cancel a claimed job"); + assert!( + cancelled.is_none_or(|row| row.run_lease == claimed[0].job.run_lease), + "cancel must act on the running attempt, not the claimed ready row" + ); + let moved = store + .move_queue( + &pool, + claimed[1].job.id, + "qs_drain_claim_window_moved", + None, + ) + .await + .expect("move a claimed job"); + assert!(!moved, "a claimed job must not be re-enqueued elsewhere"); + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 3); + + store + .complete_runtime_batch(&pool, &claimed[1..]) + .await + .expect("complete the other claimed job"); + let done_states: Vec = sqlx::query_scalar(&format!( + "SELECT state::text FROM {schema}.done_entries WHERE job_id = $1" + )) + .bind(claimed[1].job.id) + .fetch_all(&pool) + .await + .expect("load done rows"); + assert_eq!(done_states, vec!["completed".to_string()]); + let duplicated: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM ( + SELECT job_id FROM {schema}.done_entries GROUP BY job_id HAVING count(*) > 1 + ) AS duplicated" + )) + .fetch_one(&pool) + .await + .expect("count duplicated done rows"); + assert_eq!(duplicated, 0); + + store + .enqueue_batch(&pool, queue, 2, 1) + .await + .expect("enqueue after the drain"); + let next = store + .claim_runtime_batch(&pool, queue, 1, Duration::from_secs(30)) + .await + .expect("claim after the drain"); + assert_eq!( + next.len(), + 1, + "a job enqueued after a drain must be claimable on the next poll" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_waits_for_a_claim_in_flight() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_claim_in_flight"; + let store = create_store(&pool, "awa_qs_drain_claim_in_flight").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 5) + .await + .expect("enqueue available"); + let mut claim_tx = pool.begin().await.expect("begin claim"); + let claimed: Vec = sqlx::query_scalar(&format!( + "SELECT job_id FROM {schema}.claim_ready_runtime($1, 2, 0::float8, 0::float8)" + )) + .bind(queue) + .fetch_all(claim_tx.as_mut()) + .await + .expect("claim two jobs"); + assert_eq!(claimed.len(), 2); + + let drain = tokio::spawn({ + let pool = pool.clone(); + async move { admin::drain_queue(&pool, queue).await } + }); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + !drain.is_finished(), + "the drain must wait for the claim on its lane" + ); + claim_tx.commit().await.expect("commit claim"); + let drained = drain.await.expect("join drain").expect("drain queue"); + assert_eq!(drained, 3); + + let cancelled_claims: i64 = sqlx::query_scalar(&format!( + "SELECT count(*)::bigint FROM {schema}.done_entries WHERE job_id = ANY($1)" + )) + .bind(&claimed) + .fetch_one(&pool) + .await + .expect("count done rows of claimed jobs"); + assert_eq!(cancelled_claims, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_queue_storage_drain_queue_skips_a_locked_row_without_stopping() { + let (_db_guard, pool) = setup_pool(6).await; + let queue = "qs_drain_locked_row"; + let store = create_store(&pool, "awa_qs_drain_locked_row").await; + let schema = store.schema().to_string(); + + store + .enqueue_batch(&pool, queue, 2, 2_500) + .await + .expect("enqueue available backlog"); + let mut holder = pool.begin().await.expect("begin lock holder"); + let locked_id: i64 = sqlx::query_scalar(&format!( + "SELECT job_id FROM {schema}.ready_entries WHERE queue = $1 \ + ORDER BY lane_seq LIMIT 1 FOR UPDATE" + )) + .bind(queue) + .fetch_one(holder.as_mut()) + .await + .expect("lock the first ready row"); + + let drained = admin::drain_queue(&pool, queue).await.expect("drain queue"); + assert_eq!(drained, 2_499); + holder.commit().await.expect("release lock"); + + let locked_job = store + .load_job(&pool, locked_id) + .await + .expect("load locked job") + .expect("locked job should exist"); + assert_eq!(locked_job.state, JobState::Available); + let drained = admin::drain_queue(&pool, queue) + .await + .expect("drain the released row"); + assert_eq!(drained, 1); +}