From bc168e1df8db059b18433b182a6d7943d397a00e Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 18:22:41 +1300 Subject: [PATCH 01/22] fix(queue-storage): drain a large abandoned-claim backlog in bounded time (0.6) Backport of the main change. Rescue passes keep taking 500-attempt batches, each in its own transaction, within a time budget and a row cap, and run again a second later while a backlog remains. Stale-heartbeat passes take more than one batch only once runtime snapshots show the claims' owners gone, and then only claims heartbeated before the most recent gone runtime stopped reporting. Batches visit claim slots oldest first with a periodic others-first batch, walk only the claims near each slot's cursor, and settle rescued attempts in one statement per outcome. Refs: #627 --- CHANGELOG.md | 33 ++ awa-metrics/src/lib.rs | 52 +- awa-model/src/lib.rs | 3 +- awa-model/src/queue_storage.rs | 945 +++++++++++++++++++++++--------- awa-python/python/awa/_awa.pyi | 2 + awa-python/python/awa/client.py | 9 + awa-python/src/client.rs | 10 +- awa-worker/src/client.rs | 32 ++ awa-worker/src/maintenance.rs | 567 +++++++++++++++++-- awa/tests/bulk_rescue_test.rs | 672 +++++++++++++++++++++++ awa/tests/ci_timing.rs | 186 +++++++ docs/configuration.md | 20 + docs/grafana/README.md | 2 + docs/troubleshooting.md | 25 + 14 files changed, 2256 insertions(+), 302 deletions(-) create mode 100644 awa/tests/bulk_rescue_test.rs create mode 100644 awa/tests/ci_timing.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 879a17deb..5508fd06c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,39 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- **Queue-storage rescue drains a large backlog of abandoned claims in + minutes, not hours (#627).** Stale-heartbeat, deadline and callback-timeout + rescue each closed at most 500 attempts per pass, one pass per rescue + interval (30 s by default). A runtime killed while holding tens of thousands + of claims left a backlog that took hours to clear, and the claim ring could + not rotate back into a slot that still held open claims for all that time. + A rescue pass now keeps taking 500-attempt batches, each in its own short + transaction, while the previous batch came back full, until it has spent + `rescue_pass_budget` (default 1 s, at most half the rescue interval) or + rescued `rescue_pass_max_rows` (default 5,000). While a backlog remains the + next pass runs a second later. Batches visit claim slots oldest first, with + every fourth batch visiting the other slots first so they are not starved, + and look only at the claims near each slot's rescue cursor rather than the + whole slot. Rescued attempts are settled in one statement per outcome + instead of several per attempt. + + A stale heartbeat cannot tell a dead runtime from a stalled one, so a + stale-heartbeat pass takes more than one batch only once the runtime + snapshots show every runtime either fresh with its heartbeat loop alive or + silent for at least `heartbeat_staleness`, continuously for a whole + `heartbeat_staleness` window, and its extra batches take only claims last + heartbeated before the most recently gone runtime stopped reporting, so a + live runtime whose heartbeats lag is never drained in bulk. This needs + `heartbeat_rescue_interval` below half of `heartbeat_staleness`, as the + defaults are. Deadline and + callback-timeout rescues drain without that evidence. The per-attempt + rescue rules, attempt counting and DLQ moves are unchanged; rescued retries + become due no faster than `rescue_pass_max_rows` per second in total. New + metrics `awa.maintenance.rescue.pass_size` and + `awa.maintenance.rescue.oldest_attempt_age` show a drain's progress. Set + `rescue_pass_budget` to zero to keep one batch per pass. Python takes the + same settings as `rescue_pass_budget_ms` and `rescue_pass_max_rows`. + - **`admin::job_kind_overviews` on queue storage no longer depends on the planner's row estimates.** Behind the admin UI's kinds page, it counted every job through the `current_jobs` view. Two nodes of that view are diff --git a/awa-metrics/src/lib.rs b/awa-metrics/src/lib.rs index 3f81abce3..1c027fbf1 100644 --- a/awa-metrics/src/lib.rs +++ b/awa-metrics/src/lib.rs @@ -61,6 +61,9 @@ pub mod names { pub const COMPLETION_FLUSH_DURATION: &str = "awa.completion.flush_duration"; pub const HEARTBEAT_BATCHES: &str = "awa.heartbeat.batches"; pub const MAINTENANCE_RESCUES: &str = "awa.maintenance.rescues"; + pub const MAINTENANCE_RESCUE_PASS_SIZE: &str = "awa.maintenance.rescue.pass_size"; + pub const MAINTENANCE_RESCUE_OLDEST_ATTEMPT_AGE: &str = + "awa.maintenance.rescue.oldest_attempt_age"; pub const MAINTENANCE_PROMOTE_BATCHES: &str = "awa.maintenance.promote_batches"; pub const MAINTENANCE_PROMOTE_BATCH_SIZE: &str = "awa.maintenance.promote_batch_size"; pub const MAINTENANCE_PROMOTE_DURATION: &str = "awa.maintenance.promote_duration"; @@ -168,6 +171,14 @@ pub struct AwaMetrics { pub heartbeat_batches: Counter, /// Total maintenance rescue operations. pub maintenance_rescues: Counter, + /// Attempts rescued per queue-storage rescue pass, attributed by + /// `awa.rescue.kind` and why the pass stopped (`awa.rescue.pass_end`). + pub maintenance_rescue_pass_size: Histogram, + /// Age of the oldest attempt the latest queue-storage rescue pass + /// rescued, per `awa.rescue.kind`; 0 when it rescued nothing. Rescue + /// works oldest claim first, so a value well above the heartbeat + /// staleness or deadline means a backlog of abandoned claims. + pub maintenance_rescue_oldest_attempt_age_seconds: Gauge, /// Total jobs parked for external callback. pub jobs_waiting_external: Counter, /// Current queue depth per state — how many jobs are in each state per queue. @@ -419,6 +430,19 @@ impl AwaMetrics { .with_description("Number of jobs rescued by maintenance") .with_unit("{job}") .build(), + maintenance_rescue_pass_size: meter + .u64_histogram(names::MAINTENANCE_RESCUE_PASS_SIZE) + .with_description("Attempts rescued per queue-storage rescue pass") + .with_unit("{job}") + .with_boundaries(COUNT_BUCKETS.to_vec()) + .build(), + maintenance_rescue_oldest_attempt_age_seconds: meter + .f64_gauge(names::MAINTENANCE_RESCUE_OLDEST_ATTEMPT_AGE) + .with_description( + "Age of the oldest attempt rescued by the latest queue-storage rescue pass", + ) + .with_unit("s") + .build(), jobs_waiting_external: meter .u64_counter(names::JOB_WAITING_EXTERNAL) .with_description("Number of jobs parked for external callback") @@ -737,6 +761,28 @@ impl AwaMetrics { .record(duration.as_secs_f64(), &attrs); } + /// Record one queue-storage rescue pass: how many attempts it rescued, + /// why it stopped, and the age of the oldest attempt it rescued. + pub fn record_rescue_pass( + &self, + kind: &'static str, + pass_end: &'static str, + rescued: u64, + oldest_attempt_age: Duration, + ) { + self.maintenance_rescue_pass_size.record( + rescued, + &[ + opentelemetry::KeyValue::new("awa.rescue.kind", kind), + opentelemetry::KeyValue::new("awa.rescue.pass_end", pass_end), + ], + ); + self.maintenance_rescue_oldest_attempt_age_seconds.record( + oldest_attempt_age.as_secs_f64(), + &[opentelemetry::KeyValue::new("awa.rescue.kind", kind)], + ); + } + /// Record a scheduled/retryable promotion batch. pub fn record_promotion_batch(&self, state: &str, batch_size: u64, duration: Duration) { let attrs = [opentelemetry::KeyValue::new( @@ -1218,7 +1264,7 @@ mod tests { &JOB_DURATION_BUCKETS_SECONDS, ), ]; - let counts: [(&str, &Histogram); 7] = [ + let counts: [(&str, &Histogram); 8] = [ (names::ENQUEUE_BATCH_SIZE, &metrics.enqueue_batch_size), ( names::DISPATCH_CAPACITY_AVAILABLE, @@ -1233,6 +1279,10 @@ mod tests { names::MAINTENANCE_PROMOTE_BATCH_SIZE, &metrics.promotion_batch_size, ), + ( + names::MAINTENANCE_RESCUE_PASS_SIZE, + &metrics.maintenance_rescue_pass_size, + ), ( names::MAINTENANCE_ROTATE_SKIPPED_ROWS, &metrics.maintenance_rotate_skipped_rows, diff --git a/awa-model/src/lib.rs b/awa-model/src/lib.rs index 267495146..acff1469e 100644 --- a/awa-model/src/lib.rs +++ b/awa-model/src/lib.rs @@ -46,7 +46,8 @@ pub use partitioned_queue::{ }; pub use queue_storage::{ ClaimedEntry, ClaimedRuntimeJob, PruneOutcome, QueueCounts, QueueStorage, QueueStorageConfig, - RotateOutcome, SkipReason, TerminalDeltaRollupOutcome, + RescueBatch, RescueBatchOptions, RescueSlotOrder, RotateOutcome, SkipReason, + TerminalDeltaRollupOutcome, }; pub use storage::StorageStatus; diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 930aab715..6a8084048 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -25,6 +25,7 @@ const COPY_NULL_SENTINEL: &str = "__AWA_NULL__"; const COPY_CHUNK_TARGET_BYTES: usize = 256 * 1024; const TERMINAL_COUNTER_BUCKETS: i16 = 256; const RECEIPT_RESCUE_BATCH_LIMIT: i64 = 500; +const RESCUED_DUPLICATE_ERROR: &str = "rescued as duplicate: unique claim held by a newer job"; const DRAIN_BATCH_SIZE: i64 = 1_000; // The planner overestimates the drain statements enough to JIT-compile them, // which takes seconds against milliseconds of execution. @@ -343,6 +344,82 @@ pub enum PruneOutcome { }, } +/// Which claim slot a rescue batch visits first. A batch stops at its row +/// limit, so whichever slot it visits first gets most of a busy batch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum RescueSlotOrder { + /// The oldest initialized claim slot, then the rest. The claim ring + /// cannot rotate back into that slot until its claims are closed. + #[default] + OldestFirst, + /// Every other slot, starting at this offset (modulo their count), then + /// the oldest, so a long drain of the oldest slot does not starve the rest. + OthersFirst(usize), +} + +/// Options for one rescue batch ([`QueueStorage::rescue_stale_heartbeats_batch`] +/// and its deadline and callback counterparts). +#[derive(Debug, Clone, Default)] +pub struct RescueBatchOptions { + pub slot_order: RescueSlotOrder, + /// Claim slots to leave out, typically ones an earlier batch of the same + /// pass found nothing in ([`RescueBatch::idle_slots`]). + pub skip_slots: Vec, + /// Earliest `run_at` for a rescued attempt that goes back to `retryable`. + /// Its backoff still applies when that is later. + pub retry_not_before: Option>, + /// How many claims past its cursor each slot's stale-heartbeat or + /// row-local deadline walk examines (default 10,000). Every claim + /// examined costs its closure checks, so a batch that follows one that + /// filled up on a dense backlog can examine far fewer. + pub cursor_scan_limit: Option, +} + +/// The outcome of one rescue batch, which runs in one transaction. +#[derive(Debug, Default)] +pub struct RescueBatch { + pub rescued: Vec, + /// A scan stopped at its row limit, so more attempts may already be + /// eligible. False means the batch rescued everything it found. + pub saturated: bool, + /// Claim slots this batch searched and found nothing to rescue in. + pub idle_slots: Vec, +} + +/// Receipt claims one rescue batch took, and what it found on the way. +#[derive(Default)] +struct ReceiptRescueBatch { + rescued: Vec, + saturated: bool, + idle_slots: Vec, +} + +/// Where a rescue sends an exhausted attempt on a queue whose DLQ is enabled. +struct RescueDlq<'a> { + reason: &'static str, + enabled: &'a (dyn Fn(&str) -> bool + Sync), +} + +/// Split rescued attempts by the state they left, keeping first-seen order, +/// because each terminal or deferred insert releases unique claims for one +/// previous state. +fn group_by_previous_state( + rows: Vec<(LeaseTransitionRow, T)>, +) -> Vec<(JobState, Vec<(LeaseTransitionRow, T)>)> { + let mut groups: Vec<(JobState, Vec<(LeaseTransitionRow, T)>)> = Vec::new(); + for (row, settled) in rows { + let state = row.state; + match groups + .iter_mut() + .find(|(group_state, _)| *group_state == state) + { + Some((_, group)) => group.push((row, settled)), + None => groups.push((state, vec![(row, settled)])), + } + } + groups +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct TerminalDeltaRollupOutcome { pub rolled_slots: usize, @@ -10831,40 +10908,65 @@ impl QueueStorage { &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, cutoff: DateTime, - ) -> Result, AwaError> { - let mut rescued = Vec::new(); + options: &RescueBatchOptions, + batch: &mut ReceiptRescueBatch, + ) -> Result<(), AwaError> { let mut remaining = RECEIPT_RESCUE_BATCH_LIMIT; - let preferred_slot = self.oldest_initialized_claim_slot_tx(tx).await?; - - if let Some(slot) = preferred_slot { - let mut slot_rescued = self - .rescue_stale_receipt_claims_for_slot_tx(tx, slot, cutoff, remaining) + for slot in self.rescue_slot_order_tx(tx, options).await? { + let slot_rescued = self + .rescue_stale_receipt_claims_for_slot_tx( + tx, + slot, + cutoff, + options + .cursor_scan_limit + .map_or(RECEIPT_RESCUE_CURSOR_SCAN_LIMIT, |limit| { + limit.max(1) as i64 + }), + remaining, + ) .await?; + if slot_rescued.is_empty() { + batch.idle_slots.push(slot); + } remaining = remaining.saturating_sub(slot_rescued.len() as i64); - rescued.append(&mut slot_rescued); + batch.rescued.extend(slot_rescued); if remaining == 0 { - return Ok(rescued); + batch.saturated = true; + break; } } + Ok(()) + } - for slot in 0..self.claim_slot_count() { - let slot = slot as i32; - if Some(slot) == preferred_slot { - continue; + /// The claim slots one rescue batch visits, in order. Slots go from the + /// oldest initialized one to the current one, so older claims come first. + async fn rescue_slot_order_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + options: &RescueBatchOptions, + ) -> Result, AwaError> { + let (oldest, by_age) = self.claim_slots_by_age_tx(tx).await?; + let others: Vec = by_age + .into_iter() + .filter(|slot| Some(*slot) != oldest) + .collect(); + let mut slots = Vec::with_capacity(self.claim_slot_count()); + match options.slot_order { + RescueSlotOrder::OldestFirst => { + slots.extend(oldest); + slots.extend(others); } - - let mut slot_rescued = self - .rescue_stale_receipt_claims_for_slot_tx(tx, slot, cutoff, remaining) - .await?; - remaining = remaining.saturating_sub(slot_rescued.len() as i64); - rescued.append(&mut slot_rescued); - - if remaining == 0 { - break; + RescueSlotOrder::OthersFirst(offset) => { + if !others.is_empty() { + let start = offset % others.len(); + slots.extend(others[start..].iter().chain(&others[..start])); + } + slots.extend(oldest); } } - - Ok(rescued) + slots.retain(|slot| !options.skip_slots.contains(slot)); + Ok(slots) } async fn rescue_stale_receipt_claims_for_slot_tx<'a>( @@ -10872,6 +10974,7 @@ impl QueueStorage { tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, slot: i32, cutoff: DateTime, + scan_limit: i64, rescue_limit: i64, ) -> Result, AwaError> { let schema = self.schema(); @@ -10890,8 +10993,33 @@ impl QueueStorage { WHERE slot = $1 FOR UPDATE ), - claim_source AS MATERIALIZED ( - SELECT + -- The walk takes the first $3 claims after the cursor in + -- (claimed_at, job_id, run_lease) order, then wraps to the + -- first claims at or before it. Each side reads only the claims + -- it can return: row-local claims through the cursor index, and + -- compact batches (whose members share claimed_at) only up to + -- the batch at which $3 members are certainly covered. + after_batch_bound AS ( + SELECT min(running.claimed_at) AS claimed_at + FROM ( + SELECT + claim_batches.claimed_at, + sum(claim_batches.claimed_count) FILTER ( + WHERE claim_batches.claimed_at > cursor_row.rescue_cursor_claimed_at + ) OVER ( + ORDER BY claim_batches.claimed_at, claim_batches.batch_id + ) AS covered + FROM {claim_batch_child} AS claim_batches + CROSS JOIN cursor_row + WHERE claim_batches.claim_slot = $1 + AND claim_batches.claimed_at < $2 + AND claim_batches.claimed_at >= cursor_row.rescue_cursor_claimed_at + ) AS running + WHERE running.covered >= $3 + ), + after_source AS ( + ( + SELECT claims.claim_slot, NULL::bigint AS batch_id, claims.ready_slot, @@ -10908,8 +11036,19 @@ impl QueueStorage { claims.claimed_at, claims.closed_at, false AS compact_batch - FROM {claim_child} AS claims - WHERE claims.claim_slot = $1 + FROM {claim_child} AS claims + CROSS JOIN cursor_row + WHERE claims.claim_slot = $1 + AND claims.claimed_at < $2 + AND (claims.claimed_at, claims.job_id, claims.run_lease) + > ( + cursor_row.rescue_cursor_claimed_at, + cursor_row.rescue_cursor_job_id, + cursor_row.rescue_cursor_run_lease + ) + ORDER BY claims.claimed_at, claims.job_id, claims.run_lease + LIMIT $3 + ) UNION ALL SELECT claim_batches.claim_slot, @@ -10929,6 +11068,8 @@ impl QueueStorage { NULL::timestamptz AS closed_at, true AS compact_batch FROM {claim_batch_child} AS claim_batches + CROSS JOIN cursor_row + CROSS JOIN after_batch_bound CROSS JOIN LATERAL unnest( claim_batches.job_ids, claim_batches.run_leases, @@ -10938,119 +11079,197 @@ impl QueueStorage { claim_batches.max_attempts ) AS items(job_id, run_lease, receipt_id, lane_seq, attempt, max_attempts) WHERE claim_batches.claim_slot = $1 - ), - after_cursor AS MATERIALIZED ( - SELECT - claim_source.claim_slot, - claim_source.job_id, - claim_source.run_lease, - row_number() OVER ( - ORDER BY claim_source.claimed_at, claim_source.job_id, claim_source.run_lease - ) AS rn - FROM claim_source - CROSS JOIN cursor_row - WHERE claim_source.claimed_at < $2 - AND (claim_source.claimed_at, claim_source.job_id, claim_source.run_lease) + AND claim_batches.claimed_at < $2 + AND claim_batches.claimed_at >= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at + <= COALESCE(after_batch_bound.claimed_at, 'infinity'::timestamptz) + AND (claim_batches.claimed_at, items.job_id, items.run_lease) > ( cursor_row.rescue_cursor_claimed_at, cursor_row.rescue_cursor_job_id, cursor_row.rescue_cursor_run_lease ) - ORDER BY claim_source.claimed_at, claim_source.job_id, claim_source.run_lease + ), + after_cursor AS MATERIALIZED ( + SELECT + after_source.*, + row_number() OVER ( + ORDER BY after_source.claimed_at, after_source.job_id, after_source.run_lease + ) AS rn + FROM after_source + ORDER BY after_source.claimed_at, after_source.job_id, after_source.run_lease LIMIT $3 ), - after_count AS ( + after_count AS MATERIALIZED ( SELECT count(*)::bigint AS count FROM after_cursor ), - before_cursor AS MATERIALIZED ( + before_batch_bound AS ( + SELECT min(running.claimed_at) AS claimed_at + FROM ( + SELECT + claim_batches.claimed_at, + sum(claim_batches.claimed_count) FILTER ( + WHERE claim_batches.claimed_at < cursor_row.rescue_cursor_claimed_at + ) OVER ( + ORDER BY claim_batches.claimed_at, claim_batches.batch_id + ) AS covered + FROM {claim_batch_child} AS claim_batches + CROSS JOIN cursor_row + WHERE claim_batches.claim_slot = $1 + AND claim_batches.claimed_at < $2 + AND claim_batches.claimed_at <= cursor_row.rescue_cursor_claimed_at + ) AS running + WHERE running.covered >= $3 + ), + before_source AS ( + ( + SELECT + claims.claim_slot, + NULL::bigint AS batch_id, + claims.ready_slot, + claims.ready_generation, + claims.job_id, + claims.queue, + claims.priority, + claims.attempt, + claims.run_lease, + claims.max_attempts, + claims.lane_seq, + claims.enqueue_shard, + claims.receipt_id, + claims.claimed_at, + claims.closed_at, + false AS compact_batch + FROM {claim_child} AS claims + CROSS JOIN cursor_row + WHERE (SELECT count FROM after_count) < $3 + AND claims.claim_slot = $1 + AND claims.claimed_at < $2 + AND (claims.claimed_at, claims.job_id, claims.run_lease) + <= ( + cursor_row.rescue_cursor_claimed_at, + cursor_row.rescue_cursor_job_id, + cursor_row.rescue_cursor_run_lease + ) + ORDER BY claims.claimed_at, claims.job_id, claims.run_lease + LIMIT $3 + ) + UNION ALL SELECT - claim_source.claim_slot, - claim_source.job_id, - claim_source.run_lease, - after_count.count + row_number() OVER ( - ORDER BY claim_source.claimed_at, claim_source.job_id, claim_source.run_lease - ) AS rn - FROM claim_source + claim_batches.claim_slot, + claim_batches.batch_id, + claim_batches.ready_slot, + claim_batches.ready_generation, + items.job_id, + claim_batches.queue, + claim_batches.priority, + items.attempt, + items.run_lease, + items.max_attempts, + items.lane_seq, + claim_batches.enqueue_shard, + items.receipt_id, + claim_batches.claimed_at, + NULL::timestamptz AS closed_at, + true AS compact_batch + FROM {claim_batch_child} AS claim_batches CROSS JOIN cursor_row - CROSS JOIN after_count - WHERE after_count.count < $3 - AND claim_source.claimed_at < $2 - AND (claim_source.claimed_at, claim_source.job_id, claim_source.run_lease) + CROSS JOIN before_batch_bound + CROSS JOIN LATERAL unnest( + claim_batches.job_ids, + claim_batches.run_leases, + claim_batches.receipt_ids, + claim_batches.lane_seqs, + claim_batches.attempts, + claim_batches.max_attempts + ) AS items(job_id, run_lease, receipt_id, lane_seq, attempt, max_attempts) + WHERE (SELECT count FROM after_count) < $3 + AND claim_batches.claim_slot = $1 + AND claim_batches.claimed_at < $2 + AND claim_batches.claimed_at <= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at + <= COALESCE(before_batch_bound.claimed_at, 'infinity'::timestamptz) + AND (claim_batches.claimed_at, items.job_id, items.run_lease) <= ( cursor_row.rescue_cursor_claimed_at, cursor_row.rescue_cursor_job_id, cursor_row.rescue_cursor_run_lease ) - ORDER BY claim_source.claimed_at, claim_source.job_id, claim_source.run_lease + ), + before_cursor AS MATERIALIZED ( + SELECT + before_source.*, + after_count.count + row_number() OVER ( + ORDER BY before_source.claimed_at, before_source.job_id, before_source.run_lease + ) AS rn + FROM before_source + CROSS JOIN after_count + ORDER BY before_source.claimed_at, before_source.job_id, before_source.run_lease LIMIT (SELECT GREATEST($3 - count, 0) FROM after_count) ), - candidate_keys AS MATERIALIZED ( - SELECT claim_slot, job_id, run_lease, rn FROM after_cursor + candidate_rows AS MATERIALIZED ( + SELECT * FROM after_cursor UNION ALL - SELECT claim_slot, job_id, run_lease, rn FROM before_cursor + SELECT * FROM before_cursor ), candidates AS MATERIALIZED ( SELECT - claim_source.claim_slot, - claim_source.batch_id, - claim_source.ready_slot, - claim_source.ready_generation, - claim_source.job_id, - claim_source.queue, - claim_source.priority, - claim_source.attempt, - claim_source.run_lease, - claim_source.max_attempts, - claim_source.lane_seq, - claim_source.enqueue_shard, - claim_source.receipt_id, - claim_source.claimed_at, - claim_source.closed_at, - claim_source.compact_batch, - COALESCE(attempt.heartbeat_at, claim_source.claimed_at) < $2 AS is_stale, + candidate_rows.claim_slot, + candidate_rows.batch_id, + candidate_rows.ready_slot, + candidate_rows.ready_generation, + candidate_rows.job_id, + candidate_rows.queue, + candidate_rows.priority, + candidate_rows.attempt, + candidate_rows.run_lease, + candidate_rows.max_attempts, + candidate_rows.lane_seq, + candidate_rows.enqueue_shard, + candidate_rows.receipt_id, + candidate_rows.claimed_at, + candidate_rows.closed_at, + candidate_rows.compact_batch, + COALESCE(attempt.heartbeat_at, candidate_rows.claimed_at) < $2 AS is_stale, ( - claim_source.closed_at IS NOT NULL + candidate_rows.closed_at IS NOT NULL OR EXISTS ( SELECT 1 FROM {closure_child} AS closures - WHERE closures.claim_slot = claim_source.claim_slot - AND closures.job_id = claim_source.job_id - AND closures.run_lease = claim_source.run_lease + WHERE closures.claim_slot = candidate_rows.claim_slot + AND closures.job_id = candidate_rows.job_id + AND closures.run_lease = candidate_rows.run_lease ) OR EXISTS ( SELECT 1 FROM {closure_batch_child} AS closure_batches - WHERE closure_batches.receipt_ranges @> claim_source.receipt_id + WHERE closure_batches.receipt_ranges @> candidate_rows.receipt_id ) OR EXISTS ( SELECT 1 FROM {schema}.done_entries AS done - WHERE done.job_id = claim_source.job_id - AND done.run_lease = claim_source.run_lease + WHERE done.job_id = candidate_rows.job_id + AND done.run_lease = candidate_rows.run_lease ) OR EXISTS ( SELECT 1 FROM {schema}.deferred_jobs AS deferred - WHERE deferred.job_id = claim_source.job_id - AND deferred.run_lease = claim_source.run_lease + WHERE deferred.job_id = candidate_rows.job_id + AND deferred.run_lease = candidate_rows.run_lease ) OR EXISTS ( SELECT 1 FROM {schema}.dlq_entries AS dlq - WHERE dlq.job_id = claim_source.job_id - AND dlq.run_lease = claim_source.run_lease + WHERE dlq.job_id = candidate_rows.job_id + AND dlq.run_lease = candidate_rows.run_lease ) ) AS is_closed, EXISTS ( SELECT 1 FROM {schema}.leases AS lease - WHERE lease.job_id = claim_source.job_id - AND lease.run_lease = claim_source.run_lease + WHERE lease.job_id = candidate_rows.job_id + AND lease.run_lease = candidate_rows.run_lease ) AS is_lease_managed, - candidate_keys.rn - FROM candidate_keys - JOIN claim_source - ON claim_source.claim_slot = candidate_keys.claim_slot - AND claim_source.job_id = candidate_keys.job_id - AND claim_source.run_lease = candidate_keys.run_lease + candidate_rows.rn + FROM candidate_rows LEFT JOIN {schema}.attempt_state AS attempt - ON attempt.job_id = claim_source.job_id - AND attempt.run_lease = claim_source.run_lease + ON attempt.job_id = candidate_rows.job_id + AND attempt.run_lease = candidate_rows.run_lease ), stale_candidates AS ( SELECT candidates.* @@ -11083,7 +11302,7 @@ impl QueueStorage { )) .bind(slot) .bind(cutoff) - .bind(RECEIPT_RESCUE_CURSOR_SCAN_LIMIT) + .bind(scan_limit) .bind(rescue_limit) .fetch_all(tx.as_mut()) .await @@ -11314,7 +11533,7 @@ impl QueueStorage { )) .bind(slot) .bind(cutoff) - .bind(RECEIPT_RESCUE_CURSOR_SCAN_LIMIT) + .bind(scan_limit) .bind(rescue_limit) .bind(&locked_job_ids) .bind(&locked_run_leases) @@ -11344,48 +11563,44 @@ impl QueueStorage { async fn rescue_expired_receipt_deadlines_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, - ) -> Result, AwaError> { - let mut rescued = Vec::new(); + options: &RescueBatchOptions, + batch: &mut ReceiptRescueBatch, + ) -> Result<(), AwaError> { let mut remaining = RECEIPT_RESCUE_BATCH_LIMIT; - let preferred_slot = self.oldest_initialized_claim_slot_tx(tx).await?; - - if let Some(slot) = preferred_slot { - let mut slot_rescued = self - .rescue_expired_receipt_deadlines_for_slot_tx(tx, slot, remaining) + for slot in self.rescue_slot_order_tx(tx, options).await? { + let slot_rescued = self + .rescue_expired_receipt_deadlines_for_slot_tx( + tx, + slot, + options + .cursor_scan_limit + .map_or(RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT, |limit| { + limit.max(1) as i64 + }), + remaining, + ) .await?; - remaining = remaining.saturating_sub(slot_rescued.len() as i64); - rescued.append(&mut slot_rescued); - if remaining == 0 { - return Ok(rescued); + if slot_rescued.is_empty() { + batch.idle_slots.push(slot); } - } - - for slot in 0..self.claim_slot_count() { - let slot = slot as i32; - if Some(slot) == preferred_slot { - continue; - } - - let mut slot_rescued = self - .rescue_expired_receipt_deadlines_for_slot_tx(tx, slot, remaining) - .await?; remaining = remaining.saturating_sub(slot_rescued.len() as i64); - rescued.append(&mut slot_rescued); - + batch.rescued.extend(slot_rescued); if remaining == 0 { + batch.saturated = true; break; } } - - Ok(rescued) + Ok(()) } - async fn oldest_initialized_claim_slot_tx<'a>( + /// The oldest initialized claim slot, and every slot ordered from it + /// around the ring to the current one, then any not yet initialized. + async fn claim_slots_by_age_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, - ) -> Result, AwaError> { + ) -> Result<(Option, Vec), AwaError> { let schema = self.schema(); - let preferred_slot = sqlx::query_as::<_, (i32, i64, i32)>(&format!( + let ring_cursor = sqlx::query_as::<_, (i32, i64, i32)>(&format!( r#" SELECT current_slot, generation, slot_count FROM {schema}.claim_ring_state @@ -11394,20 +11609,36 @@ impl QueueStorage { )) .fetch_optional(tx.as_mut()) .await - .map_err(map_sqlx_error)? - .and_then(|(current_slot, generation, slot_count)| { - oldest_initialized_ring_slot(current_slot, generation, slot_count) - .map(|(slot, _generation)| slot) - .filter(|slot| *slot >= 0 && (*slot as usize) < self.claim_slot_count()) - }); - - Ok(preferred_slot) + .map_err(map_sqlx_error)?; + let slot_count = self.claim_slot_count() as i32; + let Some((current_slot, generation, ring_slot_count)) = ring_cursor else { + return Ok((None, (0..slot_count).collect())); + }; + let oldest = oldest_initialized_ring_slot(current_slot, generation, ring_slot_count) + .map(|(slot, _generation)| slot) + .filter(|slot| (0..slot_count).contains(slot)); + let start = oldest.unwrap_or(current_slot).rem_euclid(slot_count); + let mut by_age: Vec = Vec::with_capacity(slot_count as usize); + let mut slot = start; + loop { + by_age.push(slot); + if slot == current_slot.rem_euclid(slot_count) || by_age.len() == slot_count as usize { + break; + } + slot = (slot + 1).rem_euclid(slot_count); + } + let uninitialized: Vec = (0..slot_count) + .filter(|slot| !by_age.contains(slot)) + .collect(); + by_age.extend(uninitialized); + Ok((oldest, by_age)) } async fn rescue_expired_receipt_deadlines_for_slot_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, slot: i32, + scan_limit: i64, rescue_limit: i64, ) -> Result, AwaError> { if rescue_limit <= 0 { @@ -11541,7 +11772,7 @@ impl QueueStorage { "# )) .bind(slot) - .bind(RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT) + .bind(scan_limit) .bind(rescue_limit) .fetch_all(tx.as_mut()) .await @@ -11653,7 +11884,7 @@ impl QueueStorage { "# )) .bind(slot) - .bind(RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT) + .bind(scan_limit) .bind(rescue_limit) .bind(&locked_job_ids) .bind(&locked_run_leases) @@ -13338,59 +13569,198 @@ impl QueueStorage { done.into_job_row() } - /// Settle one attempt that maintenance rescued (stale heartbeat, expired + /// Settle attempts that maintenance rescued (stale heartbeat, expired /// deadline or callback timeout): back to `retryable` with backoff while /// attempts remain, `failed` once the rescued attempt was the last. A job /// that kills or hangs its worker every time therefore stops after /// `max_attempts`, like one whose handler returns an error. /// - /// An exhausted attempt on a queue whose DLQ is enabled (`dlq_reason` - /// is `Some`) goes straight to `dlq_entries` in this transaction, as - /// `fail_to_dlq` does for a handler error, so there is no window in - /// which it sits `failed` outside the DLQ. - async fn settle_rescued_attempt_tx( + /// An exhausted attempt on a queue whose DLQ is enabled goes straight to + /// `dlq_entries` in this transaction, as `fail_to_dlq` does for a handler + /// error, so there is no window in which it sits `failed` outside the DLQ. + /// + /// Each outcome is written in one statement per previous state. If that + /// statement hits a unique-claim conflict, the group is retried row by + /// row so only the conflicting attempts become cancellations. + async fn settle_rescued_attempts_tx( &self, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - row: LeaseTransitionRow, + rows: Vec, error: &str, - dlq_reason: Option<&str>, - ) -> Result { - let mut payload = RuntimePayload::from_json(Self::payload_with_attempt_state( - row.payload.clone(), - row.progress.clone(), - )?)?; - let exhausted = row.attempt >= row.max_attempts; - payload.push_error(lifecycle_error(error, row.attempt, exhausted)); - let duplicate_error = "rescued as duplicate: unique claim held by a newer job"; - if exhausted { - let finalized_at = Utc::now(); - if let Some(dlq_reason) = dlq_reason { - let dlq = row.clone().into_dlq_row( + dlq: RescueDlq<'_>, + retry_not_before: Option>, + ) -> Result, AwaError> { + if rows.is_empty() { + return Ok(Vec::new()); + } + + let (retry_attempts, retry_max_attempts): (Vec, Vec) = rows + .iter() + .filter(|row| row.attempt < row.max_attempts) + .map(|row| (row.attempt, row.max_attempts)) + .unzip(); + let mut retry_run_ats = self + .backoffs_at_tx(tx, &retry_attempts, &retry_max_attempts) + .await? + .into_iter(); + + let finalized_at = Utc::now(); + let mut retries: Vec<(LeaseTransitionRow, DeferredJobRow)> = Vec::new(); + let mut failures: Vec<(LeaseTransitionRow, DoneJobRow)> = Vec::new(); + let mut dead_letters: Vec<(LeaseTransitionRow, DlqJobRow)> = Vec::new(); + for row in rows { + let mut payload = RuntimePayload::from_json(Self::payload_with_attempt_state( + row.payload.clone(), + row.progress.clone(), + )?)?; + let exhausted = row.attempt >= row.max_attempts; + payload.push_error(lifecycle_error(error, row.attempt, exhausted)); + if !exhausted { + let backoff_at = retry_run_ats.next().ok_or_else(|| { + AwaError::Validation("rescue backoff missing for a retryable attempt".into()) + })?; + let run_at = retry_not_before.map_or(backoff_at, |floor| floor.max(backoff_at)); + let deferred = row.clone().into_deferred_row( + JobState::Retryable, + run_at, + Some(finalized_at), + payload.into_json(), + ); + retries.push((row, deferred)); + } else if (dlq.enabled)(&row.queue) { + let dead_letter = row.clone().into_dlq_row( finalized_at, payload.into_json(), - dlq_reason.to_string(), + dlq.reason.to_string(), finalized_at, ); - return self - .insert_rescued_dlq_or_cancel_tx(tx, row, dlq, duplicate_error) - .await; + dead_letters.push((row, dead_letter)); + } else { + let done = + row.clone() + .into_done_row(JobState::Failed, finalized_at, payload.into_json()); + failures.push((row, done)); } - let done = - row.clone() - .into_done_row(JobState::Failed, finalized_at, payload.into_json()); - self.insert_rescued_done_or_cancel_tx(tx, row, done, duplicate_error) + } + + let mut settled = Vec::with_capacity(retries.len() + failures.len() + dead_letters.len()); + for (old_state, group) in group_by_previous_state(retries) { + let deferred: Vec = group.iter().map(|(_, row)| row.clone()).collect(); + let mut savepoint = tx.begin().await.map_err(map_sqlx_error)?; + match self + .insert_deferred_rows_tx(&mut savepoint, deferred, Some(old_state)) .await - } else { - let deferred = row.clone().into_deferred_row( - JobState::Retryable, - self.backoff_at_tx(tx, row.attempt, row.max_attempts) - .await?, - Some(Utc::now()), - payload.into_json(), - ); - self.insert_rescued_deferred_or_cancel_tx(tx, row, deferred, duplicate_error) + { + Ok(_) => { + savepoint.commit().await.map_err(map_sqlx_error)?; + for (_, deferred) in group { + settled.push(deferred.into_job_row()?); + } + } + Err(AwaError::UniqueConflict { .. }) => { + savepoint.rollback().await.map_err(map_sqlx_error)?; + for (row, deferred) in group { + settled.push( + self.insert_rescued_deferred_or_cancel_tx( + tx, + row, + deferred, + RESCUED_DUPLICATE_ERROR, + ) + .await?, + ); + } + } + Err(err) => return Err(err), + } + } + for (old_state, group) in group_by_previous_state(failures) { + let done: Vec = group.iter().map(|(_, row)| row.clone()).collect(); + let mut savepoint = tx.begin().await.map_err(map_sqlx_error)?; + match self + .insert_done_rows_tx(&mut savepoint, &done, Some(old_state)) .await + { + Ok(_) => { + savepoint.commit().await.map_err(map_sqlx_error)?; + for done in done { + settled.push(done.into_job_row()?); + } + } + Err(AwaError::UniqueConflict { .. }) => { + savepoint.rollback().await.map_err(map_sqlx_error)?; + for (row, done) in group { + settled.push( + self.insert_rescued_done_or_cancel_tx( + tx, + row, + done, + RESCUED_DUPLICATE_ERROR, + ) + .await?, + ); + } + } + Err(err) => return Err(err), + } } + for (old_state, group) in group_by_previous_state(dead_letters) { + let dead_letters: Vec = group.iter().map(|(_, row)| row.clone()).collect(); + let mut savepoint = tx.begin().await.map_err(map_sqlx_error)?; + match self + .insert_dlq_rows_tx(&mut savepoint, &dead_letters, Some(old_state)) + .await + { + Ok(_) => { + savepoint.commit().await.map_err(map_sqlx_error)?; + for dead_letter in dead_letters { + settled.push(dead_letter.into_job_row()?); + } + } + Err(AwaError::UniqueConflict { .. }) => { + savepoint.rollback().await.map_err(map_sqlx_error)?; + for (row, dead_letter) in group { + settled.push( + self.insert_rescued_dlq_or_cancel_tx( + tx, + row, + dead_letter, + RESCUED_DUPLICATE_ERROR, + ) + .await?, + ); + } + } + Err(err) => return Err(err), + } + } + Ok(settled) + } + + /// `clock_timestamp() + awa.backoff_duration(..)` for each attempt, in + /// one round trip, in input order. + async fn backoffs_at_tx( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + attempts: &[i16], + max_attempts: &[i16], + ) -> Result>, AwaError> { + if attempts.is_empty() { + return Ok(Vec::new()); + } + sqlx::query_scalar( + r#" + SELECT clock_timestamp() + awa.backoff_duration(attempts.attempt, attempts.max_attempts) + FROM unnest($1::smallint[], $2::smallint[]) + WITH ORDINALITY AS attempts(attempt, max_attempts, ord) + ORDER BY attempts.ord + "#, + ) + .bind(attempts) + .bind(max_attempts) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error) } /// Rescue-path fallback for unique-claim conflicts: insert the rescued @@ -14140,16 +14510,36 @@ impl QueueStorage { /// [`Self::rescue_stale_heartbeats`], moving an exhausted attempt into /// `dlq_entries` (reason `heartbeat_stale`) in the rescue transaction /// when `dlq_enabled` returns true for its queue. + pub async fn rescue_stale_heartbeats_with_dlq( + &self, + pool: &PgPool, + staleness: Duration, + dlq_enabled: &(dyn Fn(&str) -> bool + Sync), + ) -> Result, AwaError> { + Ok(self + .rescue_stale_heartbeats_batch( + pool, + staleness, + dlq_enabled, + &RescueBatchOptions::default(), + ) + .await? + .rescued) + } + + /// One stale-heartbeat rescue batch in its own transaction: up to + /// 500 lease rows and 500 receipt claims. #[tracing::instrument( - skip(self, pool, dlq_enabled), + skip(self, pool, dlq_enabled, options), name = "queue_storage.rescue_stale_heartbeats" )] - pub async fn rescue_stale_heartbeats_with_dlq( + pub async fn rescue_stale_heartbeats_batch( &self, pool: &PgPool, staleness: Duration, dlq_enabled: &(dyn Fn(&str) -> bool + Sync), - ) -> Result, AwaError> { + options: &RescueBatchOptions, + ) -> Result { let schema = self.schema(); let mut tx = pool.begin().await.map_err(map_sqlx_error)?; let cutoff = Utc::now() @@ -14192,7 +14582,7 @@ impl QueueStorage { WHERE lease.state = 'running' AND COALESCE(attempt.heartbeat_at, lease.heartbeat_at) < $1 ORDER BY COALESCE(attempt.heartbeat_at, lease.heartbeat_at) ASC - LIMIT 500 + LIMIT $2 FOR UPDATE OF lease SKIP LOCKED ) RETURNING @@ -14215,19 +14605,27 @@ impl QueueStorage { "# )) .bind(cutoff) + .bind(RECEIPT_RESCUE_BATCH_LIMIT) .fetch_all(tx.as_mut()) .await .map_err(map_sqlx_error)?; + let mut saturated = deleted.len() as i64 >= RECEIPT_RESCUE_BATCH_LIMIT; - let rescued_receipts = if self.lease_claim_receipts() { - self.rescue_stale_receipt_claims_tx(&mut tx, cutoff).await? - } else { - Vec::new() - }; + let mut receipts = ReceiptRescueBatch::default(); + if self.lease_claim_receipts() { + self.rescue_stale_receipt_claims_tx(&mut tx, cutoff, options, &mut receipts) + .await?; + } + saturated |= receipts.saturated; + let idle_slots = receipts.idle_slots; + let rescued_receipts = receipts.rescued; if deleted.is_empty() && rescued_receipts.is_empty() { tx.commit().await.map_err(map_sqlx_error)?; - return Ok(Vec::new()); + return Ok(RescueBatch { + idle_slots, + ..RescueBatch::default() + }); } let moved_leases = self.hydrate_deleted_leases_tx(&mut tx, deleted).await?; @@ -14235,33 +14633,37 @@ impl QueueStorage { .hydrate_deleted_leases_tx(&mut tx, rescued_receipts) .await?; - let mut rescued = Vec::with_capacity(moved_leases.len() + moved_receipts.len()); - for row in moved_leases { - let dlq_reason = dlq_enabled(&row.queue).then_some("heartbeat_stale"); - rescued.push( - self.settle_rescued_attempt_tx( - &mut tx, - row, - "heartbeat stale: worker presumed dead", - dlq_reason, - ) - .await?, - ); - } - for row in moved_receipts { - let dlq_reason = dlq_enabled(&row.queue).then_some("heartbeat_stale"); - rescued.push( - self.settle_rescued_attempt_tx( - &mut tx, - row, - "receipt claim stale: worker presumed dead", - dlq_reason, - ) - .await?, - ); - } + let mut rescued = self + .settle_rescued_attempts_tx( + &mut tx, + moved_leases, + "heartbeat stale: worker presumed dead", + RescueDlq { + reason: "heartbeat_stale", + enabled: dlq_enabled, + }, + options.retry_not_before, + ) + .await?; + rescued.extend( + self.settle_rescued_attempts_tx( + &mut tx, + moved_receipts, + "receipt claim stale: worker presumed dead", + RescueDlq { + reason: "heartbeat_stale", + enabled: dlq_enabled, + }, + options.retry_not_before, + ) + .await?, + ); tx.commit().await.map_err(map_sqlx_error)?; - Ok(rescued) + Ok(RescueBatch { + rescued, + saturated, + idle_slots, + }) } /// Rescue attempts past their hard deadline. Exhausted attempts fail @@ -14275,15 +14677,29 @@ impl QueueStorage { /// [`Self::rescue_expired_deadlines`], moving an exhausted attempt into /// `dlq_entries` (reason `deadline_exceeded`) in the rescue transaction /// when `dlq_enabled` returns true for its queue. + pub async fn rescue_expired_deadlines_with_dlq( + &self, + pool: &PgPool, + dlq_enabled: &(dyn Fn(&str) -> bool + Sync), + ) -> Result, AwaError> { + Ok(self + .rescue_expired_deadlines_batch(pool, dlq_enabled, &RescueBatchOptions::default()) + .await? + .rescued) + } + + /// One deadline rescue batch in its own transaction: up to + /// 500 lease rows and 500 receipt claims. #[tracing::instrument( - skip(self, pool, dlq_enabled), + skip(self, pool, dlq_enabled, options), name = "queue_storage.rescue_expired_deadlines" )] - pub async fn rescue_expired_deadlines_with_dlq( + pub async fn rescue_expired_deadlines_batch( &self, pool: &PgPool, dlq_enabled: &(dyn Fn(&str) -> bool + Sync), - ) -> Result, AwaError> { + options: &RescueBatchOptions, + ) -> Result { let schema = self.schema(); let mut tx = pool.begin().await.map_err(map_sqlx_error)?; let deleted: Vec = sqlx::query_as(&format!( @@ -14296,7 +14712,7 @@ impl QueueStorage { AND deadline_at IS NOT NULL AND deadline_at < clock_timestamp() ORDER BY deadline_at ASC - LIMIT 500 + LIMIT $1 FOR UPDATE SKIP LOCKED ) RETURNING @@ -14318,24 +14734,32 @@ impl QueueStorage { callback_timeout_at "# )) + .bind(RECEIPT_RESCUE_BATCH_LIMIT) .fetch_all(tx.as_mut()) .await .map_err(map_sqlx_error)?; + let mut saturated = deleted.len() as i64 >= RECEIPT_RESCUE_BATCH_LIMIT; // Receipts-mode short-path claims hold their deadline on // `lease_claims.deadline_at` rather than on a `leases` row, so // the receipt-plane needs its own scan; merge both populations // into one `moved` set so the maintenance caller observes a // single rescue batch per tick. - let receipt_deleted = if self.lease_claim_receipts() { - self.rescue_expired_receipt_deadlines_tx(&mut tx).await? - } else { - Vec::new() - }; + let mut receipts = ReceiptRescueBatch::default(); + if self.lease_claim_receipts() { + self.rescue_expired_receipt_deadlines_tx(&mut tx, options, &mut receipts) + .await?; + } + saturated |= receipts.saturated; + let idle_slots = receipts.idle_slots; + let receipt_deleted = receipts.rescued; if deleted.is_empty() && receipt_deleted.is_empty() { tx.commit().await.map_err(map_sqlx_error)?; - return Ok(Vec::new()); + return Ok(RescueBatch { + idle_slots, + ..RescueBatch::default() + }); } let mut moved = self.hydrate_deleted_leases_tx(&mut tx, deleted).await?; @@ -14344,16 +14768,24 @@ impl QueueStorage { .await?, ); - let mut rescued = Vec::with_capacity(moved.len()); - for row in moved { - let dlq_reason = dlq_enabled(&row.queue).then_some("deadline_exceeded"); - rescued.push( - self.settle_rescued_attempt_tx(&mut tx, row, "hard deadline exceeded", dlq_reason) - .await?, - ); - } + let rescued = self + .settle_rescued_attempts_tx( + &mut tx, + moved, + "hard deadline exceeded", + RescueDlq { + reason: "deadline_exceeded", + enabled: dlq_enabled, + }, + options.retry_not_before, + ) + .await?; tx.commit().await.map_err(map_sqlx_error)?; - Ok(rescued) + Ok(RescueBatch { + rescued, + saturated, + idle_slots, + }) } /// Rescue attempts whose callback timed out. Exhausted attempts fail @@ -14367,15 +14799,30 @@ impl QueueStorage { /// [`Self::rescue_expired_callbacks`], moving an exhausted attempt into /// `dlq_entries` (reason `callback_timeout`) in the rescue transaction /// when `dlq_enabled` returns true for its queue. + pub async fn rescue_expired_callbacks_with_dlq( + &self, + pool: &PgPool, + dlq_enabled: &(dyn Fn(&str) -> bool + Sync), + ) -> Result, AwaError> { + Ok(self + .rescue_expired_callbacks_batch(pool, dlq_enabled, &RescueBatchOptions::default()) + .await? + .rescued) + } + + /// One callback-timeout rescue batch in its own transaction: up to + /// 500 attempts. `slot_order` does not apply; + /// callback waits live in `leases`, not in the claim ring. #[tracing::instrument( - skip(self, pool, dlq_enabled), + skip(self, pool, dlq_enabled, options), name = "queue_storage.rescue_expired_callbacks" )] - pub async fn rescue_expired_callbacks_with_dlq( + pub async fn rescue_expired_callbacks_batch( &self, pool: &PgPool, dlq_enabled: &(dyn Fn(&str) -> bool + Sync), - ) -> Result, AwaError> { + options: &RescueBatchOptions, + ) -> Result { let schema = self.schema(); let mut tx = pool.begin().await.map_err(map_sqlx_error)?; let deleted: Vec = sqlx::query_as(&format!( @@ -14388,7 +14835,7 @@ impl QueueStorage { AND callback_timeout_at IS NOT NULL AND callback_timeout_at < clock_timestamp() ORDER BY callback_timeout_at ASC - LIMIT 500 + LIMIT $1 FOR UPDATE SKIP LOCKED ) RETURNING @@ -14410,27 +14857,37 @@ impl QueueStorage { callback_timeout_at "# )) + .bind(RECEIPT_RESCUE_BATCH_LIMIT) .fetch_all(tx.as_mut()) .await .map_err(map_sqlx_error)?; + let saturated = deleted.len() as i64 >= RECEIPT_RESCUE_BATCH_LIMIT; if deleted.is_empty() { tx.commit().await.map_err(map_sqlx_error)?; - return Ok(Vec::new()); + return Ok(RescueBatch::default()); } let moved = self.hydrate_deleted_leases_tx(&mut tx, deleted).await?; - let mut rescued = Vec::with_capacity(moved.len()); - for row in moved { - let dlq_reason = dlq_enabled(&row.queue).then_some("callback_timeout"); - rescued.push( - self.settle_rescued_attempt_tx(&mut tx, row, "callback timed out", dlq_reason) - .await?, - ); - } + let rescued = self + .settle_rescued_attempts_tx( + &mut tx, + moved, + "callback timed out", + RescueDlq { + reason: "callback_timeout", + enabled: dlq_enabled, + }, + options.retry_not_before, + ) + .await?; tx.commit().await.map_err(map_sqlx_error)?; - Ok(rescued) + Ok(RescueBatch { + rescued, + saturated, + idle_slots: Vec::new(), + }) } pub async fn promote_due( diff --git a/awa-python/python/awa/_awa.pyi b/awa-python/python/awa/_awa.pyi index fcd040853..74b98d6ac 100644 --- a/awa-python/python/awa/_awa.pyi +++ b/awa-python/python/awa/_awa.pyi @@ -574,6 +574,8 @@ class Client: heartbeat_staleness_ms: int | None = None, deadline_rescue_interval_ms: int | None = None, callback_rescue_interval_ms: int | None = None, + rescue_pass_budget_ms: int | None = None, + rescue_pass_max_rows: int | None = None, terminal_count_rollup_interval_ms: int | None = None, queue_storage_schema: str | None = None, queue_storage_queue_slot_count: int = 16, diff --git a/awa-python/python/awa/client.py b/awa-python/python/awa/client.py index a237cc838..f0db57e7c 100644 --- a/awa-python/python/awa/client.py +++ b/awa-python/python/awa/client.py @@ -858,6 +858,8 @@ async def start( heartbeat_staleness_ms: int | None = None, deadline_rescue_interval_ms: int | None = None, callback_rescue_interval_ms: int | None = None, + rescue_pass_budget_ms: int | None = None, + rescue_pass_max_rows: int | None = None, terminal_count_rollup_interval_ms: int | None = None, queue_storage_schema: str | None = None, queue_storage_queue_slot_count: int = DEFAULT_QUEUE_STORAGE_QUEUE_SLOT_COUNT, @@ -900,6 +902,11 @@ async def start( counters. Exact reads include pending deltas, so the default favors fewer hot counter updates over immediate folding. + ``rescue_pass_budget_ms`` (default 1000) and ``rescue_pass_max_rows`` + (default 5000) bound how long one queue-storage rescue pass keeps + taking 500-attempt batches and how many attempts it rescues; see the + configuration guide's "Draining a backlog of abandoned claims". + Queue dicts also accept ``claimers`` and ``claim_batch_size`` for hot queue-storage queues. ``claimers`` adds dispatcher/claimer loops that share the queue's worker permits; benchmark ``2`` or ``4`` before @@ -920,6 +927,8 @@ async def start( heartbeat_staleness_ms=heartbeat_staleness_ms, deadline_rescue_interval_ms=deadline_rescue_interval_ms, callback_rescue_interval_ms=callback_rescue_interval_ms, + rescue_pass_budget_ms=rescue_pass_budget_ms, + rescue_pass_max_rows=rescue_pass_max_rows, terminal_count_rollup_interval_ms=terminal_count_rollup_interval_ms, queue_storage_schema=queue_storage_schema, queue_storage_queue_slot_count=queue_storage_queue_slot_count, diff --git a/awa-python/src/client.rs b/awa-python/src/client.rs index 7de483c14..c6329e734 100644 --- a/awa-python/src/client.rs +++ b/awa-python/src/client.rs @@ -1666,7 +1666,7 @@ impl PyClient { }) } - #[pyo3(signature = (queues=None, *, poll_interval_ms=200, global_max_workers=None, completed_retention_hours=None, failed_retention_hours=None, descriptor_retention_days=None, cleanup_batch_size=None, leader_election_interval_ms=None, heartbeat_interval_ms=None, promote_interval_ms=None, heartbeat_rescue_interval_ms=None, heartbeat_staleness_ms=None, deadline_rescue_interval_ms=None, callback_rescue_interval_ms=None, terminal_count_rollup_interval_ms=None, queue_storage_schema=None, queue_storage_queue_slot_count=16, queue_storage_lease_slot_count=8, queue_storage_claim_slot_count=8, queue_storage_queue_stripe_count=1, queue_storage_queue_rotate_interval_ms=1000, queue_storage_lease_rotate_interval_ms=1000, queue_storage_claim_rotate_interval_ms=None, storage_transition_role=None))] + #[pyo3(signature = (queues=None, *, poll_interval_ms=200, global_max_workers=None, completed_retention_hours=None, failed_retention_hours=None, descriptor_retention_days=None, cleanup_batch_size=None, leader_election_interval_ms=None, heartbeat_interval_ms=None, promote_interval_ms=None, heartbeat_rescue_interval_ms=None, heartbeat_staleness_ms=None, deadline_rescue_interval_ms=None, callback_rescue_interval_ms=None, rescue_pass_budget_ms=None, rescue_pass_max_rows=None, terminal_count_rollup_interval_ms=None, queue_storage_schema=None, queue_storage_queue_slot_count=16, queue_storage_lease_slot_count=8, queue_storage_claim_slot_count=8, queue_storage_queue_stripe_count=1, queue_storage_queue_rotate_interval_ms=1000, queue_storage_lease_rotate_interval_ms=1000, queue_storage_claim_rotate_interval_ms=None, storage_transition_role=None))] #[allow(clippy::too_many_arguments)] fn start<'py>( &self, @@ -1685,6 +1685,8 @@ impl PyClient { heartbeat_staleness_ms: Option, deadline_rescue_interval_ms: Option, callback_rescue_interval_ms: Option, + rescue_pass_budget_ms: Option, + rescue_pass_max_rows: Option, terminal_count_rollup_interval_ms: Option, queue_storage_schema: Option, queue_storage_queue_slot_count: u32, @@ -1803,6 +1805,12 @@ impl PyClient { if let Some(ms) = callback_rescue_interval_ms { builder = builder.callback_rescue_interval(Duration::from_millis(ms)); } + if let Some(ms) = rescue_pass_budget_ms { + builder = builder.rescue_pass_budget(Duration::from_millis(ms)); + } + if let Some(max_rows) = rescue_pass_max_rows { + builder = builder.rescue_pass_max_rows(max_rows); + } if let Some(ms) = terminal_count_rollup_interval_ms { if ms == 0 { return Err(pyo3::exceptions::PyValueError::new_err( diff --git a/awa-worker/src/client.rs b/awa-worker/src/client.rs index e5a498dbb..30b28b674 100644 --- a/awa-worker/src/client.rs +++ b/awa-worker/src/client.rs @@ -142,6 +142,8 @@ pub struct ClientBuilder { heartbeat_staleness: Option, deadline_rescue_interval: Option, callback_rescue_interval: Option, + rescue_pass_budget: Option, + rescue_pass_max_rows: Option, periodic_jobs: Vec, global_max_workers: Option, leader_election_interval: Option, @@ -200,6 +202,8 @@ impl ClientBuilder { heartbeat_staleness: None, deadline_rescue_interval: None, callback_rescue_interval: None, + rescue_pass_budget: None, + rescue_pass_max_rows: None, periodic_jobs: Vec::new(), global_max_workers: None, leader_election_interval: None, @@ -758,6 +762,24 @@ impl ClientBuilder { self } + /// Set how long one queue-storage rescue pass may keep starting batches of + /// 500 attempts, each in its own transaction (default: 1s, capped at half + /// the rescue interval). While a backlog remains the next pass runs + /// a second later. `Duration::ZERO` keeps every pass to one batch. + pub fn rescue_pass_budget(mut self, budget: Duration) -> Self { + self.rescue_pass_budget = Some(budget); + self + } + + /// Set how many attempts one queue-storage rescue pass rescues before it + /// stops (default: 5000). With a pass a second while a backlog remains, + /// this caps a drain at about this many attempts per second per rescue + /// kind, and rescued retries become due no faster than this per second. + pub fn rescue_pass_max_rows(mut self, max_rows: usize) -> Self { + self.rescue_pass_max_rows = Some(max_rows); + self + } + /// Set the leader election retry interval (default: 10s). /// /// Controls how often a non-leader instance retries acquiring the maintenance @@ -1079,6 +1101,8 @@ impl ClientBuilder { heartbeat_staleness: self.heartbeat_staleness, deadline_rescue_interval: self.deadline_rescue_interval, callback_rescue_interval: self.callback_rescue_interval, + rescue_pass_budget: self.rescue_pass_budget, + rescue_pass_max_rows: self.rescue_pass_max_rows, periodic_jobs: Arc::new(self.periodic_jobs), dispatch_cancel: CancellationToken::new(), service_cancel: CancellationToken::new(), @@ -1174,6 +1198,8 @@ pub struct Client { heartbeat_staleness: Option, deadline_rescue_interval: Option, callback_rescue_interval: Option, + rescue_pass_budget: Option, + rescue_pass_max_rows: Option, periodic_jobs: Arc>, /// Cancellation token for dispatchers only — stops claiming new jobs. dispatch_cancel: CancellationToken, @@ -1638,6 +1664,12 @@ impl Client { if let Some(interval) = self.callback_rescue_interval { maintenance = maintenance.callback_rescue_interval(interval); } + if let Some(budget) = self.rescue_pass_budget { + maintenance = maintenance.rescue_pass_budget(budget); + } + if let Some(max_rows) = self.rescue_pass_max_rows { + maintenance = maintenance.rescue_pass_max_rows(max_rows); + } if let Some(interval) = self.leader_election_interval { maintenance = maintenance.leader_election_interval(interval); } diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index b7c134177..893a13001 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -7,14 +7,15 @@ use awa_model::cron::{ #[cfg(test)] use awa_model::SkipReason; use awa_model::{ - JobRow, JobState, PeriodicJob, PruneOutcome, RotateOutcome, TerminalDeltaRollupOutcome, + JobRow, JobState, PeriodicJob, PruneOutcome, RescueBatch, RescueBatchOptions, RescueSlotOrder, + RotateOutcome, TerminalDeltaRollupOutcome, }; -use chrono::Utc; +use chrono::{DateTime, TimeDelta, Utc}; use croner::Cron; use sqlx::pool::PoolConnection; use sqlx::{PgPool, Postgres}; use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::task::JoinHandle; @@ -224,6 +225,144 @@ const CALLBACK_RESCUE_PER_ROW_SQL: &str = r#" /// How often the leader scans for attempts past their deadline by default. pub(crate) const DEFAULT_DEADLINE_RESCUE_INTERVAL: Duration = Duration::from_secs(30); +/// Default wall-clock budget for one queue-storage rescue pass. +pub(crate) const DEFAULT_RESCUE_PASS_BUDGET: Duration = Duration::from_secs(1); + +/// Default cap on attempts one queue-storage rescue pass rescues. +pub(crate) const DEFAULT_RESCUE_PASS_MAX_ROWS: usize = 5_000; + +/// Attempts one rescue batch rescues at most per source (lease rows, receipt +/// claims), each batch in its own transaction. +const RESCUE_BATCH_ROWS: usize = 500; + +/// While a rescue kind still has a backlog, its next pass runs after this +/// rather than after its full interval. With the per-pass row cap this bounds +/// a drain to about `rescue_pass_max_rows` attempts per second per kind. +const RESCUE_BACKLOG_INTERVAL: Duration = Duration::from_secs(1); + +/// Claims each slot's cursor walk examines in a batch that follows a full +/// one in the same pass. The first batch keeps the store's wider default so +/// a pass can still walk past long runs of closed claims. +const RESCUE_FOLLOW_UP_SCAN_LIMIT: usize = 2_000; + +/// One batch in this many visits the other claim slots before the oldest, so +/// a long drain of the oldest slot leaves the rest at least that share. +const RESCUE_OTHERS_FIRST_EVERY: usize = 4; + +/// Runtime snapshots the leader reads before letting a stale-heartbeat pass +/// take more than one batch. A runtime is `fresh` while its snapshot is +/// within three snapshot intervals; one that is neither fresh nor silent for +/// the whole heartbeat staleness ($1, milliseconds) could be a live runtime +/// that is stalled rather than gone. `last_gone_heartbeat` bounds the last +/// heartbeat any gone runtime can have written: it wrote a snapshot at least +/// every interval while it lived. +const RESCUE_RUNTIME_EVIDENCE_SQL: &str = r#" + SELECT + count(*) FILTER (WHERE fresh AND NOT heartbeat_alive)::bigint AS fresh_without_heartbeat, + count(*) FILTER (WHERE NOT fresh AND NOT gone)::bigint AS unconfirmed, + max(last_seen_at + snapshot_interval_ms * 2 * interval '1 millisecond') + FILTER (WHERE gone) AS last_gone_heartbeat + FROM ( + SELECT + heartbeat_alive, + last_seen_at, + snapshot_interval_ms, + last_seen_at >= now() - GREATEST(snapshot_interval_ms, 1000) * 3 * interval '1 millisecond' + AS fresh, + last_seen_at < now() - GREATEST(snapshot_interval_ms, 1000) * 3 * interval '1 millisecond' + AND last_seen_at < now() - $1 * interval '1 millisecond' + AS gone + FROM awa.runtime_instances + ) AS runtimes +"#; + +/// The queue-storage rescue kinds a leader drains in passes of batches. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RescueKind { + Heartbeat, + Deadline, + Callback, +} + +impl RescueKind { + /// The `awa.rescue.kind` attribute value, as on `awa.maintenance.rescues`. + fn as_str(self) -> &'static str { + match self { + Self::Heartbeat => "heartbeat", + Self::Deadline => "deadline", + Self::Callback => "callback_timeout", + } + } +} + +/// Why a queue-storage rescue pass stopped taking batches. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RescuePassEnd { + /// The last batch found fewer eligible attempts than its limit. + Drained, + /// The pass budget is spent. + TimeBudget, + /// The pass reached `rescue_pass_max_rows`. + RowLimit, + /// Stale-heartbeat only: the runtime snapshots do not yet show that the + /// owners of stale claims are gone, so the pass keeps to one batch. + OwnersUnconfirmed, + Error, +} + +impl RescuePassEnd { + fn as_str(self) -> &'static str { + match self { + Self::Drained => "drained", + Self::TimeBudget => "time_budget", + Self::RowLimit => "row_limit", + Self::OwnersUnconfirmed => "owners_unconfirmed", + Self::Error => "error", + } + } + + /// The pass stopped with eligible attempts left that it was allowed to + /// take, so the next pass should come sooner than the interval. + fn backlog_remains(self) -> bool { + matches!(self, Self::TimeBudget | Self::RowLimit) + } +} + +/// The leader's record of whether every runtime that could own a stale +/// claim is gone. A claim does not record its runtime, so the evidence is +/// fleet-wide: every runtime snapshot is either fresh with its heartbeat +/// loop alive, or silent for at least the heartbeat staleness. That must +/// hold at every sample for a full staleness window before a stale-heartbeat +/// pass may take more than one batch, so a live runtime coming out of a stall +/// has had the whole window to refresh its heartbeats first. A gap between +/// samples longer than half the staleness restarts the window, because a +/// stall the leader did not sample could have hidden behind it. +#[derive(Debug, Default)] +struct RescueOwnerEvidence { + steady_since: Option, + last_sample: Option, +} + +impl RescueOwnerEvidence { + /// Record one sample and return whether the evidence now allows a + /// stale-heartbeat pass to take more than one batch. + fn observe(&mut self, now: Instant, settled: bool, staleness: Duration) -> bool { + let sampled_recently = self + .last_sample + .is_some_and(|last| now.saturating_duration_since(last) <= staleness / 2); + self.last_sample = Some(now); + if !settled { + self.steady_since = None; + return false; + } + if !sampled_recently { + self.steady_since = Some(now); + } + let steady_since = *self.steady_since.get_or_insert(now); + now.saturating_duration_since(steady_since) >= staleness + } +} + /// Per-queue or global retention policy for completed and failed/cancelled jobs. #[derive(Debug, Clone)] pub struct RetentionPolicy { @@ -685,6 +824,13 @@ pub struct MaintenanceService { heartbeat_rescue_interval: Duration, deadline_rescue_interval: Duration, callback_rescue_interval: Duration, + rescue_pass_budget: Duration, + rescue_pass_max_rows: usize, + rescue_owner_evidence: std::sync::Mutex, + /// Earliest `run_at` for the next attempt a queue-storage rescue sends + /// back to `retryable`; see [`Self::rescue_retry_floor`]. + rescue_retry_horizon: std::sync::Mutex>>, + rescue_others_first_offset: AtomicUsize, promote_interval: Duration, cleanup_interval: Duration, cron_sync_interval: Duration, @@ -753,6 +899,11 @@ impl MaintenanceService { heartbeat_rescue_interval: Duration::from_secs(30), deadline_rescue_interval: DEFAULT_DEADLINE_RESCUE_INTERVAL, callback_rescue_interval: Duration::from_secs(30), + rescue_pass_budget: DEFAULT_RESCUE_PASS_BUDGET, + rescue_pass_max_rows: DEFAULT_RESCUE_PASS_MAX_ROWS, + rescue_owner_evidence: std::sync::Mutex::new(RescueOwnerEvidence::default()), + rescue_retry_horizon: std::sync::Mutex::new(None), + rescue_others_first_offset: AtomicUsize::new(0), promote_interval: Duration::from_millis(250), cleanup_interval: Duration::from_secs(60), cron_sync_interval: Duration::from_secs(60), @@ -859,6 +1010,21 @@ impl MaintenanceService { self } + /// Set how long one queue-storage rescue pass may keep starting batches + /// (default: 1s, and never more than half the rescue interval). Zero + /// keeps every pass to one batch. + pub fn rescue_pass_budget(mut self, budget: Duration) -> Self { + self.rescue_pass_budget = budget; + self + } + + /// Set how many attempts one queue-storage rescue pass rescues before it + /// stops (default: 5000). A pass always finishes the batch it started. + pub fn rescue_pass_max_rows(mut self, max_rows: usize) -> Self { + self.rescue_pass_max_rows = max_rows; + self + } + /// Set how long a heartbeat must be stale before the job is rescued (default: 90s). /// /// Should be at least 3× the heartbeat interval to avoid false rescues @@ -1066,20 +1232,29 @@ impl MaintenanceService { } _ = heartbeat_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_stale_heartbeats", self.heartbeat_rescue_interval, &self.metrics) { - self.rescue_stale_heartbeats().await; + let backlog = self.rescue_stale_heartbeats().await; timer.finish(); + if backlog { + heartbeat_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.heartbeat_rescue_interval)); + } } } _ = deadline_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_expired_deadlines", self.deadline_rescue_interval, &self.metrics) { - self.rescue_expired_deadlines().await; + let backlog = self.rescue_expired_deadlines().await; timer.finish(); + if backlog { + deadline_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.deadline_rescue_interval)); + } } } _ = callback_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_expired_callbacks", self.callback_rescue_interval, &self.metrics) { - self.rescue_expired_callbacks().await; + let backlog = self.rescue_expired_callbacks().await; timer.finish(); + if backlog { + callback_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.callback_rescue_interval)); + } } } _ = promote_timer.tick() => { @@ -1409,42 +1584,287 @@ impl MaintenanceService { /// `canonical_live_backlog` above zero and wedge `finalize --wait`, /// and stuck queue-storage rows would wait out a drain-only leader. /// Both cross-plane sweeps are cheap no-ops once their tables are empty. + /// + /// Returns whether the queue-storage pass left a backlog it may keep + /// draining, so the next pass should run before the full interval. #[tracing::instrument(skip(self), name = "maintenance.rescue_stale")] - async fn rescue_stale_heartbeats(&self) { + async fn rescue_stale_heartbeats(&self) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_stale_heartbeats().await; self.process_heartbeat_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { - let outcome = runtime - .store - .rescue_stale_heartbeats_with_dlq( - &self.pool, - self.heartbeat_staleness, - &|queue| self.dlq_policy.enabled_for(queue), - ) - .await; - self.process_heartbeat_rescues(outcome, false, Some(runtime)) + return self + .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, false) .await; } + false } RuntimeStorage::QueueStorage(runtime) => { - let outcome = runtime - .store - .rescue_stale_heartbeats_with_dlq( - &self.pool, - self.heartbeat_staleness, - &|queue| self.dlq_policy.enabled_for(queue), - ) - .await; - self.process_heartbeat_rescues(outcome, true, Some(runtime)) + let backlog = self + .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, true) .await; let outcome = self.rescue_canonical_stale_heartbeats().await; self.process_heartbeat_rescues(outcome, false, None).await; + backlog } } } + /// One queue-storage rescue pass: batches of up to 500 attempts per + /// source, each in its own transaction, until a batch comes back short, + /// the pass budget is spent, or the pass reaches `rescue_pass_max_rows`. Most batches visit the oldest claim slot + /// first, since the ring cannot rotate back into it while it holds open + /// claims; every [`RESCUE_OTHERS_FIRST_EVERY`]th visits the others first. + /// A slot a batch found nothing in is left out for the rest of the pass. + /// + /// A stale heartbeat cannot tell a dead runtime from a stalled one, so a + /// stale-heartbeat pass takes more than one batch only when + /// [`RescueOwnerEvidence`] holds, and its later batches take only claims + /// last heartbeated before the most recently gone runtime stopped + /// reporting. A live runtime whose heartbeats merely lag has refreshed its + /// claims since then. Deadline and callback rescues act on a deadline that + /// has passed whoever owns the attempt. + /// + /// Returns whether the pass stopped with a backlog it may keep draining, + /// so the next pass should run before the full interval. + async fn rescue_queue_storage_pass( + &self, + kind: RescueKind, + runtime: &QueueStorageRuntime, + signal_local: bool, + ) -> bool { + let started = Instant::now(); + let interval = match kind { + RescueKind::Heartbeat => self.heartbeat_rescue_interval, + RescueKind::Deadline => self.deadline_rescue_interval, + RescueKind::Callback => self.callback_rescue_interval, + }; + let budget = self.rescue_pass_budget.min(interval / 2); + let dlq_enabled = |queue: &str| self.dlq_policy.enabled_for(queue); + let mut bulk_heartbeat_cutoff: Option>> = None; + let mut rescued_total = 0usize; + let mut oldest_attempt: Option> = None; + let mut batches = 0usize; + let mut idle_slots: Vec = Vec::new(); + let pass_end = loop { + let slot_order = if batches % RESCUE_OTHERS_FIRST_EVERY == RESCUE_OTHERS_FIRST_EVERY - 1 + { + RescueSlotOrder::OthersFirst( + self.rescue_others_first_offset + .fetch_add(1, Ordering::Relaxed), + ) + } else { + RescueSlotOrder::OldestFirst + }; + let options = RescueBatchOptions { + slot_order, + skip_slots: idle_slots.clone(), + retry_not_before: self.rescue_retry_floor(), + cursor_scan_limit: (batches > 0).then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), + }; + let outcome = match kind { + RescueKind::Heartbeat => { + // Follow-up batches take only claims last heartbeated + // before a runtime known to be gone stopped reporting. + let staleness = bulk_heartbeat_cutoff + .flatten() + .and_then(|cutoff| (Utc::now() - cutoff).to_std().ok()) + .map_or(self.heartbeat_staleness, |since_cutoff| { + since_cutoff.max(self.heartbeat_staleness) + }); + runtime + .store + .rescue_stale_heartbeats_batch( + &self.pool, + staleness, + &dlq_enabled, + &options, + ) + .await + } + RescueKind::Deadline => { + runtime + .store + .rescue_expired_deadlines_batch(&self.pool, &dlq_enabled, &options) + .await + } + RescueKind::Callback => { + runtime + .store + .rescue_expired_callbacks_batch(&self.pool, &dlq_enabled, &options) + .await + } + }; + batches += 1; + let RescueBatch { + rescued, + saturated, + idle_slots: batch_idle_slots, + } = match outcome { + Ok(batch) => batch, + Err(err) => { + self.process_queue_storage_rescues(kind, Err(err), signal_local, runtime) + .await; + break RescuePassEnd::Error; + } + }; + idle_slots.extend(batch_idle_slots); + rescued_total += rescued.len(); + self.advance_rescue_retry_horizon( + rescued + .iter() + .filter(|job| job.state == JobState::Retryable) + .count(), + ); + if let Some(attempted_at) = rescued.iter().filter_map(|job| job.attempted_at).min() { + oldest_attempt = + Some(oldest_attempt.map_or(attempted_at, |oldest| oldest.min(attempted_at))); + } + self.process_queue_storage_rescues(kind, Ok(rescued), signal_local, runtime) + .await; + + if !saturated { + break RescuePassEnd::Drained; + } + if budget.is_zero() { + break RescuePassEnd::TimeBudget; + } + if kind == RescueKind::Heartbeat { + let cutoff = match bulk_heartbeat_cutoff { + Some(cutoff) => cutoff, + None => *bulk_heartbeat_cutoff.insert(self.stale_claim_owners_gone().await), + }; + if cutoff.is_none() { + break RescuePassEnd::OwnersUnconfirmed; + } + } + if rescued_total >= self.rescue_pass_max_rows { + break RescuePassEnd::RowLimit; + } + if started.elapsed() >= budget { + break RescuePassEnd::TimeBudget; + } + }; + + let oldest_attempt_age = oldest_attempt + .and_then(|attempted_at| (Utc::now() - attempted_at).to_std().ok()) + .unwrap_or_default(); + self.metrics.record_rescue_pass( + kind.as_str(), + pass_end.as_str(), + rescued_total as u64, + oldest_attempt_age, + ); + if batches > 1 { + info!( + rescue_kind = kind.as_str(), + rescued = rescued_total, + batches, + pass_end = pass_end.as_str(), + elapsed_ms = started.elapsed().as_millis() as u64, + oldest_attempt_age_s = oldest_attempt_age.as_secs(), + "Queue-storage rescue pass drained a backlog" + ); + } + pass_end.backlog_remains() && !budget.is_zero() + } + + async fn process_queue_storage_rescues( + &self, + kind: RescueKind, + outcome: Result, awa_model::AwaError>, + signal_local: bool, + runtime: &QueueStorageRuntime, + ) { + match kind { + RescueKind::Heartbeat => { + self.process_heartbeat_rescues(outcome, signal_local, Some(runtime)) + .await + } + RescueKind::Deadline => { + self.process_deadline_rescues(outcome, signal_local, Some(runtime)) + .await + } + RescueKind::Callback => { + self.process_callback_rescues(outcome, signal_local, Some(runtime)) + .await + } + } + } + + /// Sample the runtime snapshots and update [`RescueOwnerEvidence`]. + /// Returns the heartbeat cutoff for follow-up stale-heartbeat batches + /// when the evidence holds and some runtime is known to be gone; `None` + /// keeps the pass to one batch. A failed read counts as no evidence. + async fn stale_claim_owners_gone(&self) -> Option> { + let staleness_ms = i64::try_from(self.heartbeat_staleness.as_millis()).unwrap_or(i64::MAX); + let (settled, last_gone_heartbeat) = match sqlx::query_as::< + _, + (i64, i64, Option>), + >(RESCUE_RUNTIME_EVIDENCE_SQL) + .bind(staleness_ms) + .fetch_one(&self.pool) + .await + { + Ok((fresh_without_heartbeat, unconfirmed, last_gone_heartbeat)) => { + if fresh_without_heartbeat > 0 || unconfirmed > 0 { + debug!( + fresh_without_heartbeat, + unconfirmed, + "Stale-heartbeat rescue kept to one batch: runtime snapshots do not show the claim owners gone" + ); + } + ( + fresh_without_heartbeat == 0 && unconfirmed == 0, + last_gone_heartbeat, + ) + } + Err(err) => { + warn!(error = %err, "Failed to read runtime snapshots for stale-heartbeat rescue"); + (false, None) + } + }; + let steady = self + .rescue_owner_evidence + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .observe(Instant::now(), settled, self.heartbeat_staleness); + if steady { + last_gone_heartbeat + } else { + None + } + } + + /// The earliest `run_at` for the next attempt a queue-storage rescue + /// sends back to `retryable`. Rescued attempts become due no faster than + /// `rescue_pass_max_rows` per second across all rescue kinds, so a large + /// drain reaches promotion and the ready lanes spread over time rather + /// than all at once. `None` while rescue runs below that rate. + fn rescue_retry_floor(&self) -> Option> { + let horizon = *self + .rescue_retry_horizon + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + horizon.filter(|horizon| *horizon > Utc::now()) + } + + fn advance_rescue_retry_horizon(&self, retried: usize) { + if retried == 0 { + return; + } + let per_second = self.rescue_pass_max_rows.max(RESCUE_BATCH_ROWS) as i64; + let spread = TimeDelta::microseconds(retried as i64 * 1_000_000 / per_second); + let now = Utc::now(); + let mut horizon = self + .rescue_retry_horizon + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let start = horizon.map_or(now, |horizon| horizon.max(now)); + *horizon = Some(start + spread); + } + /// Batched canonical stale-heartbeat rescue with the #388 row-at-a-time /// fallback on unique-claim conflict. Exhausted attempts fail in place, /// as callback timeouts do, so a job that keeps killing or hanging its @@ -1608,33 +2028,25 @@ impl MaintenanceService { /// #456 mirror: sweeps both storage planes; see /// [`Self::rescue_stale_heartbeats`]. #[tracing::instrument(skip(self), name = "maintenance.rescue_deadline")] - async fn rescue_expired_deadlines(&self) { + async fn rescue_expired_deadlines(&self) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_expired_deadlines().await; self.process_deadline_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { - let outcome = runtime - .store - .rescue_expired_deadlines_with_dlq(&self.pool, &|queue| { - self.dlq_policy.enabled_for(queue) - }) - .await; - self.process_deadline_rescues(outcome, false, Some(runtime)) + return self + .rescue_queue_storage_pass(RescueKind::Deadline, runtime, false) .await; } + false } RuntimeStorage::QueueStorage(runtime) => { - let outcome = runtime - .store - .rescue_expired_deadlines_with_dlq(&self.pool, &|queue| { - self.dlq_policy.enabled_for(queue) - }) - .await; - self.process_deadline_rescues(outcome, true, Some(runtime)) + let backlog = self + .rescue_queue_storage_pass(RescueKind::Deadline, runtime, true) .await; let outcome = self.rescue_canonical_expired_deadlines().await; self.process_deadline_rescues(outcome, false, None).await; + backlog } } } @@ -1781,33 +2193,25 @@ impl MaintenanceService { /// doubly here because callback-carrying jobs never migrate off the /// canonical plane during a transition. #[tracing::instrument(skip(self), name = "maintenance.rescue_callback_timeout")] - async fn rescue_expired_callbacks(&self) { + async fn rescue_expired_callbacks(&self) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_expired_callbacks().await; self.process_callback_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { - let outcome = runtime - .store - .rescue_expired_callbacks_with_dlq(&self.pool, &|queue| { - self.dlq_policy.enabled_for(queue) - }) - .await; - self.process_callback_rescues(outcome, false, Some(runtime)) + return self + .rescue_queue_storage_pass(RescueKind::Callback, runtime, false) .await; } + false } RuntimeStorage::QueueStorage(runtime) => { - let outcome = runtime - .store - .rescue_expired_callbacks_with_dlq(&self.pool, &|queue| { - self.dlq_policy.enabled_for(queue) - }) - .await; - self.process_callback_rescues(outcome, true, Some(runtime)) + let backlog = self + .rescue_queue_storage_pass(RescueKind::Callback, runtime, true) .await; let outcome = self.rescue_canonical_expired_callbacks().await; self.process_callback_rescues(outcome, false, None).await; + backlog } } } @@ -4067,6 +4471,59 @@ mod tests { ); } + #[test] + fn rescue_owner_evidence_needs_a_full_settled_staleness_window() { + let staleness = Duration::from_secs(90); + let sample_every = Duration::from_secs(30); + let mut evidence = RescueOwnerEvidence::default(); + let mut clock = Instant::now(); + + for _ in 0..3 { + assert!(!evidence.observe(clock, true, staleness)); + clock += sample_every; + } + assert!( + evidence.observe(clock, true, staleness), + "settled at every sample across the staleness window" + ); + + clock += sample_every; + assert!( + !evidence.observe(clock, false, staleness), + "an unsettled sample closes the gate at once" + ); + for _ in 0..3 { + clock += sample_every; + assert!( + !evidence.observe(clock, true, staleness), + "the window restarts after an unsettled sample" + ); + } + clock += sample_every; + assert!(evidence.observe(clock, true, staleness)); + } + + #[test] + fn rescue_owner_evidence_restarts_after_an_unsampled_gap() { + let staleness = Duration::from_secs(90); + let mut evidence = RescueOwnerEvidence::default(); + let mut clock = Instant::now(); + for _ in 0..4 { + evidence.observe(clock, true, staleness); + clock += Duration::from_secs(30); + } + assert!(evidence.observe(clock, true, staleness)); + + // The leader did not sample for longer than half the staleness: a + // stall in that gap would have been invisible, so start over. + clock += Duration::from_secs(46); + assert!(!evidence.observe(clock, true, staleness)); + clock += Duration::from_secs(45); + assert!(!evidence.observe(clock, true, staleness)); + clock += Duration::from_secs(45); + assert!(evidence.observe(clock, true, staleness)); + } + #[test] fn branch_tracker_without_cooldown_tracks_overrun_but_keeps_running() { let tracker = MaintenanceBranchTracker::new(); diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs new file mode 100644 index 000000000..fb474e7c5 --- /dev/null +++ b/awa/tests/bulk_rescue_test.rs @@ -0,0 +1,672 @@ +//! A runtime holding tens of thousands of receipt claims dies without a +//! graceful shutdown. Maintenance must rescue the whole backlog in bounded +//! time, the claim ring must resume rotating, a runtime that stays alive +//! must never have a claim rescued, and every rescued attempt must count +//! (#627). + +#[path = "ci_timing.rs"] +mod ci_timing; + +use awa::model::{migrations, QueueStorage, QueueStorageConfig}; +use awa::{Client, InsertOpts, JobArgs, JobEvent, JobResult, QueueConfig}; +use ci_timing::{scaled_staleness, scaled_timeout}; +use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; +use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; +use serde::{Deserialize, Serialize}; +use sqlx::postgres::{PgConnection, PgPoolOptions}; +use sqlx::{Connection, PgPool}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const ABANDONED: i64 = 50_000; +/// Every tenth abandoned job has one attempt, so its rescue exhausts it. +const EXHAUSTED_EVERY: i64 = 10; +const ABANDONED_QUEUE: &str = "bulk_rescue_abandoned"; +const LIVE_QUEUE: &str = "bulk_rescue_live"; + +/// What each run of a job saw, by job id. +type RunsByJob = HashMap>; + +#[derive(Debug, Serialize, Deserialize, JobArgs)] +struct AbandonedJob { + n: i64, +} + +#[derive(Debug, Serialize, Deserialize, JobArgs)] +struct LiveJob { + n: i64, + hold_ms: u64, +} + +fn queue_storage_builder( + pool: &PgPool, + staleness: Duration, + heartbeat_interval: Duration, +) -> awa::ClientBuilder { + Client::builder(pool.clone()) + .queue_storage( + QueueStorageConfig::default(), + Duration::from_secs(1), + Duration::from_secs(1), + ) + // Rotate the claim ring often so the abandoned claims spread across + // slots and the ring wraps onto the oldest of them. + .claim_rotate_interval(Duration::from_millis(200)) + .heartbeat_interval(heartbeat_interval) + .heartbeat_staleness(staleness) + .runtime_snapshot_interval(Duration::from_millis(250)) + .leader_check_interval(Duration::from_millis(200)) + .promote_interval(Duration::from_millis(100)) + .dlq_enabled_by_default(true) +} + +async fn count(pool: &PgPool, sql: String) -> i64 { + sqlx::query_scalar(&sql) + .fetch_one(pool) + .await + .unwrap_or_else(|err| panic!("{sql}: {err}")) +} + +async fn running(pool: &PgPool, store: &QueueStorage, queue: &str) -> i64 { + store + .queue_counts(pool, queue) + .await + .expect("queue counts") + .running +} + +async fn claim_ring_generation(pool: &PgPool) -> i64 { + count( + pool, + "SELECT generation FROM awa.claim_ring_state WHERE singleton".into(), + ) + .await +} + +/// Rescued and dead-lettered rows for a queue: what a false rescue would leave. +async fn rescued_rows(pool: &PgPool, queue: &str) -> i64 { + sqlx::query_scalar( + "SELECT (SELECT count(*) FROM awa.deferred_jobs WHERE queue = $1 AND state = 'retryable') + + (SELECT count(*) FROM awa.dlq_entries WHERE queue = $1)", + ) + .bind(queue) + .fetch_one(pool) + .await + .expect("count rescued rows") +} + +async fn wait_until(timeout: Duration, what: &str, mut done: F) -> Duration +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let started = Instant::now(); + loop { + if done().await { + return started.elapsed(); + } + assert!( + started.elapsed() < timeout, + "timed out after {timeout:?} waiting for {what}" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +fn largest_heartbeat_rescue_pass(exporter: &InMemoryMetricExporter) -> u64 { + let mut largest = 0; + for resource in exporter.get_finished_metrics().expect("read metrics") { + for scope in resource.scope_metrics() { + for metric in scope.metrics() { + if metric.name() != "awa.maintenance.rescue.pass_size" { + continue; + } + if let AggregatedMetrics::U64(MetricData::Histogram(histogram)) = metric.data() { + for point in histogram.data_points() { + let heartbeat = point.attributes().any(|attribute| { + attribute.key.as_str() == "awa.rescue.kind" + && attribute.value.as_str() == "heartbeat" + }); + if heartbeat { + largest = largest.max(point.max().unwrap_or(0)); + } + } + } + } + } + } + largest +} + +fn database_url() -> String { + std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://postgres:test@localhost:15432/awa_test".to_string()) +} + +fn with_database(url: &str, database: &str) -> String { + let (base, _) = url + .rsplit_once('/') + .expect("database URL has a database name"); + format!("{base}/{database}") +} + +/// A database of its own with queue storage active, dropped afterwards. +struct BulkRescueDatabase { + name: String, + pool: PgPool, +} + +impl BulkRescueDatabase { + async fn create() -> Self { + let name = format!("awa_bulk_rescue_{}", std::process::id()); + let mut admin = PgConnection::connect(&with_database(&database_url(), "postgres")) + .await + .expect("admin connection"); + sqlx::raw_sql(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .execute(&mut admin) + .await + .expect("drop leftover database"); + sqlx::raw_sql(&format!("CREATE DATABASE {name}")) + .execute(&mut admin) + .await + .expect("create database"); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&with_database(&database_url(), &name)) + .await + .expect("test pool"); + migrations::run(&pool).await.expect("migrate"); + QueueStorage::new(QueueStorageConfig::default()) + .expect("queue storage") + .install(&pool) + .await + .expect("install queue storage"); + awa_testing::setup::activate_queue_storage(&pool).await; + Self { name, pool } + } + + fn pool(&self) -> &PgPool { + &self.pool + } + + fn url(&self) -> String { + with_database(&database_url(), &self.name) + } +} + +impl Drop for BulkRescueDatabase { + fn drop(&mut self) { + let name = self.name.clone(); + let admin_url = with_database(&database_url(), "postgres"); + let cleanup = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("cleanup runtime"); + runtime.block_on(async { + if let Ok(mut admin) = PgConnection::connect(&admin_url).await { + let _ = sqlx::raw_sql(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .execute(&mut admin) + .await; + } + }); + }); + let _ = cleanup.join(); + } +} + +/// A client on its own Tokio runtime, so it can be killed the way SIGKILL +/// would: every task is dropped mid-flight, no shutdown runs, and its +/// connections close with their transactions rolled back. +struct DoomedClient { + kill: Option>, + thread: Option>, +} + +impl DoomedClient { + fn start(url: String, staleness: Duration, heartbeat_interval: Duration) -> Self { + let (kill, killed) = tokio::sync::oneshot::channel::<()>(); + let (started_tx, started_rx) = std::sync::mpsc::channel::<()>(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .expect("doomed client runtime"); + let client = runtime.block_on(async { + let pool = PgPoolOptions::new() + .max_connections(20) + .connect(&url) + .await + .expect("doomed client pool"); + let client = queue_storage_builder(&pool, staleness, heartbeat_interval) + .queue( + ABANDONED_QUEUE, + QueueConfig { + max_workers: ABANDONED as u32 + 1_000, + poll_interval: Duration::from_millis(10), + deadline_duration: Duration::ZERO, + claimers: 4, + claim_batch_size: 1_000, + ..QueueConfig::default() + }, + ) + .register::(|_, _| async { + std::future::pending::>().await + }) + .build() + .expect("doomed client"); + client.start().await.expect("start doomed client"); + client + }); + started_tx.send(()).expect("report doomed client start"); + runtime.block_on(async { + let _ = killed.await; + }); + runtime.shutdown_background(); + drop(client); + }); + started_rx.recv().expect("doomed client started"); + Self { + kill: Some(kill), + thread: Some(thread), + } + } + + fn kill(mut self) { + if let Some(kill) = self.kill.take() { + let _ = kill.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().expect("doomed client thread"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_test_writer() + .try_init(); + let exporter = InMemoryMetricExporter::default(); + let meter_provider = SdkMeterProvider::builder() + .with_periodic_exporter(exporter.clone()) + .build(); + opentelemetry::global::set_meter_provider(meter_provider.clone()); + + let db = BulkRescueDatabase::create().await; + let pool = db.pool().clone(); + let store = QueueStorage::new(QueueStorageConfig::default()).expect("queue storage"); + let claim_slots = store.claim_slot_count() as i64; + + // Once the doomed runtime has claimed its backlog, live jobs run for + // three staleness windows, so a rescue that ignored their heartbeats + // would take them. Until then they finish at once, so their claims do + // not hold up the claim ring while the doomed runtime fills it. + let staleness = scaled_staleness(Duration::from_secs(6)); + let heartbeat_interval = staleness / 8; + let live_job_duration = staleness * 3; + + let live_pool = PgPoolOptions::new() + .max_connections(30) + .connect(&db.url()) + .await + .expect("live pool"); + let live_runs: Arc>> = Arc::default(); + // The live runtime leads maintenance, so it sees every rescue. + let rescues: Arc>> = Arc::default(); + let live = { + let live_runs = live_runs.clone(); + let live_rescues = rescues.clone(); + let abandoned_rescues = rescues.clone(); + queue_storage_builder(&live_pool, staleness, heartbeat_interval) + // Under half the staleness, so the leader samples runtime + // snapshots often enough to confirm the dead owner. + .heartbeat_rescue_interval(Duration::from_secs(2)) + .deadline_rescue_interval(Duration::from_secs(2)) + .leader_election_interval(Duration::from_millis(100)) + .queue( + LIVE_QUEUE, + QueueConfig { + max_workers: 1_000, + poll_interval: Duration::from_millis(25), + deadline_duration: Duration::ZERO, + ..QueueConfig::default() + }, + ) + .on_event::(move |event| { + if let JobEvent::Rescued { job, .. } = event { + live_rescues + .lock() + .expect("rescues") + .entry(job.id) + .or_default() + .push((LIVE_QUEUE, job.attempt)); + } + async {} + }) + .on_event::(move |event| { + if let JobEvent::Rescued { job, .. } = event { + abandoned_rescues + .lock() + .expect("rescues") + .entry(job.id) + .or_default() + .push((ABANDONED_QUEUE, job.attempt)); + } + async {} + }) + .register::(move |job, ctx| { + live_runs + .lock() + .expect("live runs") + .entry(ctx.job.id) + .or_default() + .push(ctx.job.attempt); + async move { + tokio::time::sleep(Duration::from_millis(job.hold_ms)).await; + Ok(JobResult::Completed) + } + }) + .build() + .expect("live client") + }; + live.start().await.expect("start live client"); + wait_until( + scaled_timeout(Duration::from_secs(10)), + "live leader", + || async { live.health_check().await.leader }, + ) + .await; + + // The live runtime keeps claiming new work for the whole test. + let producing = Arc::new(AtomicBool::new(true)); + let live_hold_ms = Arc::new(AtomicU64::new(0)); + let producer = { + let pool = pool.clone(); + let store = QueueStorage::new(QueueStorageConfig::default()).expect("queue storage"); + let producing = producing.clone(); + let live_hold_ms = live_hold_ms.clone(); + tokio::spawn(async move { + let mut n = 0; + let mut produced = 0usize; + while producing.load(Ordering::SeqCst) { + let params: Vec<_> = (0..5) + .map(|i| { + awa::model::insert::params_with( + &LiveJob { + n: n + i, + hold_ms: live_hold_ms.load(Ordering::SeqCst), + }, + InsertOpts { + queue: LIVE_QUEUE.into(), + ..Default::default() + }, + ) + .expect("live params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue live jobs"); + n += 5; + produced += params.len(); + tokio::time::sleep(Duration::from_millis(500)).await; + } + produced + }) + }; + + for chunk_start in (0..ABANDONED).step_by(5_000) { + let params: Vec<_> = (chunk_start..(chunk_start + 5_000).min(ABANDONED)) + .map(|n| { + awa::model::insert::params_with( + &AbandonedJob { n }, + InsertOpts { + queue: ABANDONED_QUEUE.into(), + max_attempts: if n % EXHAUSTED_EVERY == 0 { 1 } else { 3 }, + ..Default::default() + }, + ) + .expect("abandoned params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue abandoned jobs"); + } + + let generation_before = claim_ring_generation(&pool).await; + let doomed = DoomedClient::start(db.url(), staleness, heartbeat_interval); + let claimed_in = wait_until( + scaled_timeout(Duration::from_secs(120)), + "the doomed runtime to claim every abandoned job", + || async { running(&pool, &store, ABANDONED_QUEUE).await >= ABANDONED }, + ) + .await; + assert_eq!( + rescued_rows(&pool, ABANDONED_QUEUE).await, + 0, + "rescue took a claim while its runtime was alive and heartbeating" + ); + let generation_at_kill = claim_ring_generation(&pool).await; + live_hold_ms.store(live_job_duration.as_millis() as u64, Ordering::SeqCst); + doomed.kill(); + assert!( + generation_at_kill - generation_before >= 2, + "the abandoned claims should span several claim slots \ + (ring generation {generation_before} -> {generation_at_kill})" + ); + + // Before the drain: the claims become stale one staleness after the + // kill, and the owner evidence needs one more staleness window. After + // that the backlog must drain at the multi-batch rate, not 500 claims + // per heartbeat-rescue interval. + let drained_in = wait_until( + staleness * 2 + scaled_timeout(Duration::from_secs(90)), + "rescue to drain the abandoned claims", + || async { running(&pool, &store, ABANDONED_QUEUE).await == 0 }, + ) + .await; + let generation_drained = claim_ring_generation(&pool).await; + eprintln!( + "claimed {ABANDONED} in {claimed_in:?}; drained {drained_in:?} after the kill \ + (claims stale after {staleness:?})" + ); + + wait_until( + scaled_timeout(Duration::from_secs(10)), + "a rescue event for every abandoned job", + || async { rescues.lock().expect("rescues").len() as i64 >= ABANDONED }, + ) + .await; + type Rescued = (&'static str, i16); + let rescued_twice = |rescues: &RunsByJob| -> Vec<(i64, Vec)> { + rescues + .iter() + .filter(|(_, rescued)| rescued.as_slice() != [(ABANDONED_QUEUE, 1)]) + .take(5) + .map(|(id, rescued)| (*id, rescued.clone())) + .collect() + }; + { + let rescues = rescues.lock().expect("rescues"); + assert_eq!(rescues.len() as i64, ABANDONED); + let unexpected = rescued_twice(&rescues); + assert!( + unexpected.is_empty(), + "each abandoned job is rescued once and nothing else is: {unexpected:?}" + ); + } + + let exhausted = ABANDONED / EXHAUSTED_EVERY; + assert_eq!( + count( + &pool, + format!( + "SELECT count(*) FROM awa.dlq_entries + WHERE queue = '{ABANDONED_QUEUE}' AND dlq_reason = 'heartbeat_stale' AND attempt = 1" + ), + ) + .await, + exhausted, + "every one-attempt job is dead-lettered by its rescue" + ); + let retryable = count( + &pool, + format!( + "SELECT count(*) FROM awa.deferred_jobs + WHERE queue = '{ABANDONED_QUEUE}' AND state = 'retryable'" + ), + ) + .await; + let promoted = store + .queue_counts(&pool, ABANDONED_QUEUE) + .await + .expect("abandoned queue counts") + .available; + assert_eq!( + retryable + promoted, + ABANDONED - exhausted, + "every other abandoned job is waiting to retry" + ); + meter_provider.force_flush().expect("flush metrics"); + let largest_pass = largest_heartbeat_rescue_pass(&exporter); + assert!( + largest_pass > 500, + "a stale-heartbeat pass should take more than one 500-claim batch once the \ + owner is confirmed gone; the largest took {largest_pass}" + ); + + // Live claims can hold a slot for a whole live job. + let rotated_in = wait_until( + live_job_duration * 2 + scaled_timeout(Duration::from_secs(30)), + "the claim ring to rotate through every slot after the drain", + || async { claim_ring_generation(&pool).await >= generation_drained + claim_slots }, + ) + .await; + eprintln!("claim ring rotated a full revolution {rotated_in:?} after the drain"); + + // A fresh runtime runs every retried job exactly once, on its second attempt. + let recovered: Arc>> = Arc::default(); + let recovery_pool = PgPoolOptions::new() + .max_connections(20) + .connect(&db.url()) + .await + .expect("recovery pool"); + let recovery = { + let recovered = recovered.clone(); + queue_storage_builder(&recovery_pool, staleness, heartbeat_interval) + .queue( + ABANDONED_QUEUE, + // Small claims, so finished attempts never wait on a long + // completion backlog while the recovery runtime is under test. + QueueConfig { + max_workers: 100, + poll_interval: Duration::from_millis(10), + deadline_duration: Duration::ZERO, + claim_batch_size: 100, + ..QueueConfig::default() + }, + ) + .register::(move |_, ctx| { + let rescue_recorded = ctx + .job + .errors + .as_ref() + .and_then(|errors| errors.last()) + .and_then(|error| error.get("error")) + .and_then(|error| error.as_str()) + .is_some_and(|error| error.contains("presumed dead")); + recovered + .lock() + .expect("recovered") + .entry(ctx.job.id) + .or_default() + .push((ctx.job.attempt, rescue_recorded)); + async { Ok(JobResult::Completed) } + }) + .build() + .expect("recovery client") + }; + recovery.start().await.expect("start recovery client"); + wait_until( + scaled_timeout(Duration::from_secs(120)), + "the retried jobs to complete", + || async { recovered.lock().expect("recovered").len() as i64 >= ABANDONED - exhausted }, + ) + .await; + recovery.shutdown(Duration::from_secs(10)).await; + { + let recovered = recovered.lock().expect("recovered"); + assert_eq!(recovered.len() as i64, ABANDONED - exhausted); + let unexpected: Vec<_> = recovered + .iter() + .filter(|(_, runs)| runs.first() != Some(&(2, true))) + .take(5) + .collect(); + assert!( + unexpected.is_empty(), + "each retried job first runs on attempt 2, with the rescue recorded: {unexpected:?}" + ); + } + + producing.store(false, Ordering::SeqCst); + let produced = producer.await.expect("producer"); + wait_until( + live_job_duration + scaled_timeout(Duration::from_secs(30)), + "the live jobs to finish", + || async { + running(&pool, &store, LIVE_QUEUE).await == 0 + && store + .queue_counts(&pool, LIVE_QUEUE) + .await + .expect("live counts") + .available + == 0 + }, + ) + .await; + live.shutdown(Duration::from_secs(10)).await; + + assert_eq!( + rescued_rows(&pool, LIVE_QUEUE).await, + 0, + "rescue took a claim from the live runtime" + ); + // On this line a retried job's finished attempt can still be rescued + // from the recovery runtime, as it was before bounded rescue, so only + // the live runtime's claims are checked here. + { + let rescues = rescues.lock().expect("rescues"); + let live_rescued: Vec<_> = rescues + .iter() + .filter(|(_, rescued)| rescued.iter().any(|(queue, _)| *queue == LIVE_QUEUE)) + .take(5) + .collect(); + assert!( + live_rescued.is_empty(), + "no claim of the live runtime was rescued: {live_rescued:?}" + ); + } + let live_runs = live_runs.lock().expect("live runs"); + assert_eq!(live_runs.len(), produced, "every live job ran"); + let rerun: Vec<_> = live_runs + .iter() + .filter(|(_, attempts)| attempts.as_slice() != [1]) + .take(5) + .collect(); + assert!( + rerun.is_empty(), + "a live job ran more than once or on a later attempt: {rerun:?}" + ); + eprintln!( + "live runtime ran {} jobs of {live_job_duration:?} each with no rescue", + live_runs.len() + ); +} diff --git a/awa/tests/ci_timing.rs b/awa/tests/ci_timing.rs new file mode 100644 index 000000000..3704a0ba7 --- /dev/null +++ b/awa/tests/ci_timing.rs @@ -0,0 +1,186 @@ +//! Shared CI contention scaling for the nightly chaos and benchmark suites. +//! +//! The nightly suites run on shared GitHub runners whose CPU allocation +//! varies run to run. Three assertion shapes are sensitive to that and +//! flaked repeatedly through 2026-07 (#399, #434): +//! +//! 1. **Wall-clock waits.** A test waits N seconds for a state the +//! runtime reaches in milliseconds when it has a core to itself. +//! Scale with [`scaled_timeout`]. +//! +//! 2. **Aggressive rescue cadences.** A chaos client sets +//! `heartbeat_staleness` a few multiples above `heartbeat_interval` +//! so its own backdating triggers rescue promptly. Under contention +//! a *live* worker's heartbeat can miss that window, so the runtime +//! correctly rescues a healthy attempt — and the test sees a genuine +//! duplicate completion, which reads as a correctness failure. Scale +//! with [`scaled_staleness`]: the interval stays fast so rescue is +//! still exercised, but the staleness window gains the same margin +//! as the waits around it. +//! +//! 3. **Minimum-progress floors.** A gate asserts "at least N of these +//! happened" to catch a stalled subsystem. The floor must sit far +//! below the *observed* operating point, not just below the nominal +//! one. Scale with [`contention_floor`]. +//! +//! Scaling only ever loosens a bound, and only on CI. A local run keeps +//! the strict values, so a real regression still fails fast on a +//! developer machine. Override with `AWA_CHAOS_TIMEOUT_MULTIPLIER`. +#![allow(dead_code)] + +use std::time::Duration; + +/// Upper bound on the override. `Duration::mul_f64` panics on a non-finite +/// or overflowing result, so an unbounded override would turn a typo into a +/// failed nightly rather than a loose bound. +const MAX_MULTIPLIER: f64 = 100.0; + +/// How much slack to give contention-sensitive bounds. `1.0` locally, +/// `3.0` on CI, or the `AWA_CHAOS_TIMEOUT_MULTIPLIER` override (clamped to +/// `1.0 ..= MAX_MULTIPLIER` so the override can only ever loosen, and can +/// never produce a duration `mul_f64` refuses). +pub fn chaos_timeout_multiplier() -> f64 { + let override_var = std::env::var("AWA_CHAOS_TIMEOUT_MULTIPLIER").ok(); + multiplier_from(override_var.as_deref(), std::env::var_os("CI").is_some()) +} + +/// The multiplier decision, with the environment passed in so it can be +/// tested without mutating process-global state. +fn multiplier_from(override_var: Option<&str>, is_ci: bool) -> f64 { + // `"inf"` and `"nan"` parse successfully as f64, and both reach + // `Duration::mul_f64`, which panics rather than saturating. Treat a + // non-finite override as no override at all. + if let Some(parsed) = override_var + .and_then(|raw| raw.parse::().ok()) + .filter(|parsed| parsed.is_finite()) + { + return parsed.clamp(1.0, MAX_MULTIPLIER); + } + + if is_ci { + 3.0 + } else { + 1.0 + } +} + +/// Grow a wait deadline by the contention multiplier. +pub fn scaled_timeout(timeout: Duration) -> Duration { + timeout.mul_f64(chaos_timeout_multiplier()) +} + +/// Grow a heartbeat-staleness window by the contention multiplier. +/// +/// Distinct from [`scaled_timeout`] only in intent: this one is passed to +/// `ClientBuilder::heartbeat_staleness`, where the cost of being too tight +/// is a spurious rescue of a live attempt rather than a timeout. Paired +/// heartbeat/rescue *intervals* are deliberately left unscaled so the +/// rescue path still runs at chaos cadence. +pub fn scaled_staleness(staleness: Duration) -> Duration { + staleness.mul_f64(chaos_timeout_multiplier()) +} + +/// Shrink a minimum-progress floor by the contention multiplier. +/// +/// Use for "this subsystem must have advanced at least N times" gates. +/// The returned floor is at least 1: the regression these gates exist to +/// catch is a fully stalled subsystem, so zero progress must still fail. +pub fn contention_floor(nominal: i64) -> i64 { + let scaled = (nominal as f64 / chaos_timeout_multiplier()).floor() as i64; + scaled.max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_runs_keep_strict_bounds() { + assert_eq!(multiplier_from(None, false), 1.0); + } + + #[test] + fn ci_runs_get_margin() { + assert_eq!(multiplier_from(None, true), 3.0); + } + + #[test] + fn override_can_only_loosen() { + assert_eq!(multiplier_from(Some("0.1"), true), 1.0); + assert_eq!(multiplier_from(Some("-5"), true), 1.0); + assert_eq!(multiplier_from(Some("6"), false), 6.0); + } + + #[test] + fn unparseable_override_falls_back_to_the_environment() { + assert_eq!(multiplier_from(Some("banana"), true), 3.0); + assert_eq!(multiplier_from(Some(""), false), 1.0); + } + + #[test] + fn non_finite_override_falls_back_to_the_environment() { + // These all parse as f64 and would otherwise reach `mul_f64`, which + // panics on a non-finite result instead of saturating. + for raw in ["inf", "-inf", "infinity", "NaN", "nan"] { + assert_eq!(multiplier_from(Some(raw), true), 3.0, "raw={raw}"); + assert_eq!(multiplier_from(Some(raw), false), 1.0, "raw={raw}"); + } + } + + #[test] + fn oversized_override_is_clamped() { + assert_eq!(multiplier_from(Some("1e308"), true), MAX_MULTIPLIER); + assert_eq!(multiplier_from(Some("10000"), true), MAX_MULTIPLIER); + } + + #[test] + fn every_multiplier_produces_a_usable_duration() { + // `Duration::mul_f64` panics rather than saturating, so the clamp is + // what keeps a bad override from failing the nightly it was set to + // rescue. + for raw in [ + "inf", "-inf", "NaN", "1e308", "10000", "0", "-1", "banana", "", + ] { + for is_ci in [true, false] { + let m = multiplier_from(Some(raw), is_ci); + assert!(m.is_finite() && (1.0..=MAX_MULTIPLIER).contains(&m)); + // Would panic if the multiplier were unbounded. + let _ = Duration::from_secs(60).mul_f64(m); + } + } + } + + #[test] + fn scaling_a_timeout_grows_it() { + // 250ms staleness against a 50ms heartbeat interval is a 5x margin + // locally; on CI it becomes 15x, which is what stops a contended + // runner's live worker from being rescued as if it had died. + assert_eq!( + Duration::from_millis(250).mul_f64(multiplier_from(None, true)), + Duration::from_millis(750) + ); + } + + #[test] + fn progress_floor_keeps_margin_under_the_observed_rate() { + // The receipt gate's nominal floor is duration_secs / 4 = 45 for a + // 180s run. #399 saw a healthy run produce 41, so the CI floor has + // to sit well under that. + let ci_floor = (45.0_f64 / multiplier_from(None, true)).floor() as i64; + assert_eq!(ci_floor, 15); + assert!( + ci_floor < 41, + "floor must clear the observed operating point" + ); + } + + #[test] + fn progress_floor_never_reaches_zero() { + // Whatever the multiplier, a fully pinned ring (zero rotations) has + // to keep failing — that is the regression the gate exists for. + assert_eq!(contention_floor(45), contention_floor(45).max(1)); + let absurd = (45.0_f64 / multiplier_from(Some("10000"), true)).floor() as i64; + assert_eq!(absurd.max(1), 1); + assert_eq!(contention_floor(0), 1); + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 868dfb3cf..c7d7950da 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -473,6 +473,26 @@ If you suspend a handler with `wait_for_callback()` and the external system neve | --- | --- | --- | | `ClientBuilder::callback_rescue_interval` | `30s` | How often the maintenance leader scans for `callback_timeout_at < now()`. The per-callback timeout itself is set when registering the callback in the handler. | +### Draining a backlog of abandoned claims { #rescue-backlog } + +A runtime that is killed (OOM, `SIGKILL`, a lost node) leaves every attempt it had claimed open. On queue storage those open claims also stop the claim ring from rotating back into their slot, so a large backlog has to be rescued quickly, not 500 attempts per rescue interval. + +Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pass of batches. A batch rescues up to 500 attempts per source (lease rows and receipt claims) in its own short transaction. The pass keeps starting batches while the previous one came back full, until its time budget is spent or it has rescued `rescue_pass_max_rows`. While a backlog remains, the next pass of that kind runs one second later instead of after its interval, so a drain proceeds at up to about `rescue_pass_max_rows` attempts per second per rescue kind. + +| Knob | Default | What it does | +| --- | --- | --- | +| `ClientBuilder::rescue_pass_budget` (Rust) / `rescue_pass_budget_ms` (Python kwarg) | `1s` | How long one pass keeps starting batches. Capped at half the rescue interval. `Duration::ZERO` / `0` keeps every pass to one batch. | +| `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic) per pass and so per second. | + +How a pass chooses and paces its work: + +- **Oldest claims first.** Batches visit claim slots from the oldest to the current one, because the ring cannot rotate back into the oldest slot until its claims are closed. Every fourth batch visits the other slots first, so a long drain of one slot does not starve the rest. A slot a batch found nothing in is skipped for the rest of the pass. +- **Stale heartbeats need evidence that the owner is gone.** A stale heartbeat cannot tell a dead runtime from a stalled one (a database stall, a long GC pause, a network partition). A stale-heartbeat pass takes more than one batch only when the runtime snapshots in `awa.runtime_instances` show every runtime either fresh with its heartbeat loop alive, or silent for at least `heartbeat_staleness`, and that has held at every pass for a whole `heartbeat_staleness` window. A live runtime coming out of a stall has had that window to refresh its heartbeats, so its claims are not stale by the time a drain can reach them. The extra batches also take only claims last heartbeated before the most recent of those gone runtimes stopped reporting, so a live runtime whose heartbeat pass merely lags behind `heartbeat_staleness` (one holding many claims under load, say) is not drained in bulk; its claims stay at one batch per pass, as before. The leader samples the snapshots once per pass, so this evidence needs `heartbeat_rescue_interval` below half of `heartbeat_staleness` (the defaults, `30s` and `90s`, qualify); otherwise stale-heartbeat rescue stays at one batch per pass. The per-attempt staleness rule itself is unchanged. Deadline and callback-timeout rescues act on a deadline that has passed, so they drain without this evidence. +- **Retries are spread out.** A rescued attempt that goes back to `retryable` keeps its backoff, and rescued retries across all kinds become due no faster than `rescue_pass_max_rows` per second, so promotion moves a large drain into the ready lanes over time instead of all at once. +- **Exhausted attempts are dead-lettered in the same batches.** On a DLQ-enabled queue an attempt rescued on its last try goes to the DLQ in its batch's transaction (reason `heartbeat_stale`, `deadline_exceeded` or `callback_timeout`), so a drain can move many jobs to the DLQ at once. `awa.job.dlq_moved` counts them by reason. + +A pass holds the maintenance leader's loop for up to its budget plus one batch, so promotion and the other maintenance branches can wait about that long while a backlog drains. Watch a drain with `awa.maintenance.rescue.pass_size` (attempts per pass, by `awa.rescue.kind` and `awa.rescue.pass_end`: `drained`, `time_budget`, `row_limit`, `owners_unconfirmed` or `error`) and `awa.maintenance.rescue.oldest_attempt_age` (the age of the oldest attempt the latest pass rescued, `0` when it rescued nothing). An `oldest_attempt_age` far above `heartbeat_staleness` or the queue's deadline means rescue is behind. + ### Retention and cleanup Retention depends on the storage path. diff --git a/docs/grafana/README.md b/docs/grafana/README.md index 823102462..5a7d1dce0 100644 --- a/docs/grafana/README.md +++ b/docs/grafana/README.md @@ -157,6 +157,8 @@ All metrics use the `awa` OTel meter name and are exported via OTLP to your conf | `awa.maintenance.promote_batch_size` | Histogram | state | Jobs promoted | | `awa.maintenance.promote_duration` | Histogram (s) | state | Promotion time | | `awa.maintenance.rescues` | Counter | rescue_kind | Jobs rescued | +| `awa.maintenance.rescue.pass_size` | Histogram | rescue_kind, rescue_pass_end | Attempts rescued per queue-storage rescue pass, and why the pass stopped (`drained`, `time_budget`, `row_limit`, `owners_unconfirmed`, `error`) | +| `awa.maintenance.rescue.oldest_attempt_age` | Gauge (s) | rescue_kind | Age of the oldest attempt the latest queue-storage rescue pass rescued; `0` when it rescued none | ### Heartbeat diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index bfa095cd5..7883706f0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -285,6 +285,31 @@ If long-running jobs are expected, remember: If a job really needs `20m`, set a longer deadline for that queue. +## Claim Ring Stops Rotating After A Runtime Is Killed + +Symptoms on queue storage, usually after a high-throughput runtime was OOM-killed or lost without a graceful shutdown: + +- `awa.maintenance.rotate.attempts` for the claim ring reports `skipped_busy` with blocker `claim.rows`, and `awa.ring.generation` for the claim ring stops advancing +- the current claim slot keeps growing +- jobs the dead runtime held wait well past `heartbeat_staleness` before they run again + +The dead runtime's claims stay open until rescue closes them, and the ring cannot rotate back into a slot that still holds open claims. Rescue drains such a backlog in passes of 500-attempt batches ([Draining a backlog of abandoned claims](configuration.md#rescue-backlog)). To see where it is: + +- `awa.maintenance.rescue.oldest_attempt_age` by `awa.rescue.kind` shows how old the attempts rescue is closing are. It should fall towards `heartbeat_staleness` (or the queue's deadline) as the backlog drains. +- `awa.maintenance.rescue.pass_size` by `awa.rescue.pass_end` shows why passes stop. `time_budget` or `row_limit` means a drain is running at its configured rate. `owners_unconfirmed` means stale-heartbeat rescue is waiting for evidence that the claims' runtime is gone and is taking one batch per pass. + +If stale-heartbeat passes stay at `owners_unconfirmed`, look at the runtime snapshots: + +```sql +SELECT instance_id, hostname, last_seen_at, snapshot_interval_ms, heartbeat_alive +FROM awa.runtime_instances +ORDER BY last_seen_at DESC; +``` + +A runtime that has stopped reporting but has not yet been silent for `heartbeat_staleness`, or a reporting runtime with `heartbeat_alive = false`, holds the drain back until it is either gone or healthy. The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. + +To drain faster, raise `rescue_pass_max_rows` and `rescue_pass_budget`, keeping in mind that a pass holds the maintenance leader for its budget and writes up to `rescue_pass_max_rows` closures and retry rows per pass. + ## Dead Tuples Growing In Queue Storage ### What It Usually Means From 303b7ad9873ee9f685bec69b89b1956e8ea935e4 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 18:57:44 +1300 Subject: [PATCH 02/22] fix(worker): bound finished attempts waiting for finalization (0.6) The executor removed an attempt from the heartbeat set and released its worker slot as soon as the handler returned, then queued its finalize on the completion batcher. When finalization was slower than claiming, the dispatcher kept claiming and the queue of finished, unheartbeated attempts grew without bound. Those that waited past heartbeat_staleness were rescued and run again. Admit at most the completion pipeline's depth (2 x batch size x shards, and at least one per worker) of attempts to finalization. Past that, a finished attempt keeps its worker slot and heartbeat until there is room, which stops the dispatcher claiming. This is the bound main already has. --- CHANGELOG.md | 12 ++ awa-worker/src/client.rs | 2 + awa-worker/src/completion.rs | 19 ++- awa-worker/src/dispatcher.rs | 2 + awa-worker/src/executor.rs | 35 ++++- awa/tests/ci_timing.rs | 186 ++++++++++++++++++++++++ awa/tests/queue_storage_runtime_test.rs | 147 +++++++++++++++++++ 7 files changed, 397 insertions(+), 6 deletions(-) create mode 100644 awa/tests/ci_timing.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 879a17deb..9527d89a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- **A completion backlog no longer gets finished jobs rescued and run + again.** An attempt stops heartbeating when its handler returns, and its + finalize then waits for the completion batcher. The runtime released the + worker slot at the same moment and kept claiming, so when finalization was + slower than claiming (for example, retried jobs, whose completions take + the slower per-row path) the queue of finished, unheartbeated attempts grew + without bound. Attempts that waited longer than `heartbeat_staleness` were + rescued and executed again, some of them twice. At most `2 × batch size × + completion shards` attempts (1,024 by default), and at least one per + worker, now wait for finalization. Past that, a finished attempt keeps its + worker slot and its heartbeat until there is room, so the dispatcher stops + claiming until finalization catches up. - **`admin::job_kind_overviews` on queue storage no longer depends on the planner's row estimates.** Behind the admin UI's kinds page, it counted every job through the `current_jobs` view. Two nodes of that view are diff --git a/awa-worker/src/client.rs b/awa-worker/src/client.rs index e5a498dbb..42ff2aee2 100644 --- a/awa-worker/src/client.rs +++ b/awa-worker/src/client.rs @@ -1566,6 +1566,8 @@ impl Client { completion_handle, effective_storage.clone(), self.dlq_policy.clone(), + self.service_cancel.clone(), + runtime_worker_capacity, )); // Admin cancellation listener: fires the in-flight cancel flag diff --git a/awa-worker/src/completion.rs b/awa-worker/src/completion.rs index 87d3646ff..ac54468ca 100644 --- a/awa-worker/src/completion.rs +++ b/awa-worker/src/completion.rs @@ -105,9 +105,19 @@ fn completion_sort_key(request: &CompletionRequest) -> (i32, i64, RunLease) { #[derive(Clone)] pub(crate) struct CompletionBatcherHandle { shards: Vec>, + pipeline_capacity: usize, } impl CompletionBatcherHandle { + /// How many finished attempts may wait for finalization at once: enough + /// to keep every shard's in-flight and next batch full, and at least one + /// per worker. + pub fn finalizer_capacity(&self, runtime_worker_capacity: u32) -> usize { + self.pipeline_capacity + .max(runtime_worker_capacity.max(1) as usize) + .min(tokio::sync::Semaphore::MAX_PERMITS) + } + pub async fn complete(&self, job_id: i64, run_lease: RunLease) -> Result { self.complete_inner(job_id, run_lease, None, None).await } @@ -185,7 +195,14 @@ impl CompletionBatcher { }); } - (Self { workers }, CompletionBatcherHandle { shards }) + ( + Self { workers }, + CompletionBatcherHandle { + shards, + // Each shard needs an in-flight batch and a queued batch. + pipeline_capacity: batch_size.saturating_mul(shard_count).saturating_mul(2), + }, + ) } pub fn spawn(self) -> Vec> { diff --git a/awa-worker/src/dispatcher.rs b/awa-worker/src/dispatcher.rs index 11e3e9dbd..08d55c02b 100644 --- a/awa-worker/src/dispatcher.rs +++ b/awa-worker/src/dispatcher.rs @@ -851,6 +851,8 @@ mod tests { completion, storage.clone(), crate::executor::DlqPolicy::default(), + CancellationToken::new(), + 1, )); let config = QueueConfig { max_workers: 1, diff --git a/awa-worker/src/executor.rs b/awa-worker/src/executor.rs index 3d99ec8df..99b8c9c18 100644 --- a/awa-worker/src/executor.rs +++ b/awa-worker/src/executor.rs @@ -11,6 +11,8 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; use std::time::Duration; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, warn, Instrument}; /// Result of executing a job handler. @@ -267,6 +269,8 @@ pub struct JobExecutor { completion_batcher: CompletionBatcherHandle, storage: RuntimeStorage, dlq_policy: DlqPolicy, + finalize_cancel: CancellationToken, + finalize_capacity: Arc, } impl JobExecutor { @@ -288,7 +292,12 @@ impl JobExecutor { completion_batcher: CompletionBatcherHandle, storage: RuntimeStorage, dlq_policy: DlqPolicy, + finalize_cancel: CancellationToken, + runtime_worker_capacity: u32, ) -> Self { + let finalize_capacity = Arc::new(Semaphore::new( + completion_batcher.finalizer_capacity(runtime_worker_capacity), + )); Self { pool, workers, @@ -301,6 +310,8 @@ impl JobExecutor { completion_batcher, storage, dlq_policy, + finalize_cancel, + finalize_capacity, } } @@ -327,6 +338,8 @@ impl JobExecutor { let completion_batcher = self.completion_batcher.clone(); let storage = self.storage.clone(); let dlq_policy = self.dlq_policy.clone(); + let finalize_cancel = self.finalize_cancel.clone(); + let finalize_capacity = self.finalize_capacity.clone(); let job_id = job.id; let job_run_lease = job.run_lease; let job_kind = job.kind.clone(); @@ -430,17 +443,28 @@ impl JobExecutor { guard.clone_latest() }; - // Remove from in-flight immediately after the handler returns and - // the progress snapshot is captured. This keeps local worker - // capacity tied to active handler execution, not to the tail - // latency of durable completion bookkeeping. + // Leaving `in_flight` ends the attempt's heartbeat, so only a + // bounded number of finished attempts may wait for finalization. + // While that queue is full the attempt keeps its worker slot and + // its heartbeat, and the dispatcher stops claiming: a completion + // backlog cannot outgrow `heartbeat_staleness` and get finished + // attempts rescued and run again. + let finalize_permit = match result { + Some(_) => tokio::select! { + permit = finalize_capacity.acquire_owned() => { + Some(permit.expect("finalizer capacity is never closed")) + } + _ = finalize_cancel.cancelled() => None, + }, + None => None, + }; in_flight.remove((job_id, job_run_lease)); if let Some(counter) = queue_in_flight.get(&job_queue) { counter.fetch_sub(1, Ordering::SeqCst); } metrics.record_in_flight_change(&job_queue, -1); - let Some(result) = result else { + let (Some(result), Some(finalize_permit)) = (result, finalize_permit) else { return; }; let dlq_enabled = dlq_policy.enabled_for(&job_queue); @@ -487,6 +511,7 @@ impl JobExecutor { outcome => break outcome, } }; + drop(finalize_permit); match &outcome { Ok(CompletionOutcome::Applied { terminal, .. }) => { diff --git a/awa/tests/ci_timing.rs b/awa/tests/ci_timing.rs new file mode 100644 index 000000000..3704a0ba7 --- /dev/null +++ b/awa/tests/ci_timing.rs @@ -0,0 +1,186 @@ +//! Shared CI contention scaling for the nightly chaos and benchmark suites. +//! +//! The nightly suites run on shared GitHub runners whose CPU allocation +//! varies run to run. Three assertion shapes are sensitive to that and +//! flaked repeatedly through 2026-07 (#399, #434): +//! +//! 1. **Wall-clock waits.** A test waits N seconds for a state the +//! runtime reaches in milliseconds when it has a core to itself. +//! Scale with [`scaled_timeout`]. +//! +//! 2. **Aggressive rescue cadences.** A chaos client sets +//! `heartbeat_staleness` a few multiples above `heartbeat_interval` +//! so its own backdating triggers rescue promptly. Under contention +//! a *live* worker's heartbeat can miss that window, so the runtime +//! correctly rescues a healthy attempt — and the test sees a genuine +//! duplicate completion, which reads as a correctness failure. Scale +//! with [`scaled_staleness`]: the interval stays fast so rescue is +//! still exercised, but the staleness window gains the same margin +//! as the waits around it. +//! +//! 3. **Minimum-progress floors.** A gate asserts "at least N of these +//! happened" to catch a stalled subsystem. The floor must sit far +//! below the *observed* operating point, not just below the nominal +//! one. Scale with [`contention_floor`]. +//! +//! Scaling only ever loosens a bound, and only on CI. A local run keeps +//! the strict values, so a real regression still fails fast on a +//! developer machine. Override with `AWA_CHAOS_TIMEOUT_MULTIPLIER`. +#![allow(dead_code)] + +use std::time::Duration; + +/// Upper bound on the override. `Duration::mul_f64` panics on a non-finite +/// or overflowing result, so an unbounded override would turn a typo into a +/// failed nightly rather than a loose bound. +const MAX_MULTIPLIER: f64 = 100.0; + +/// How much slack to give contention-sensitive bounds. `1.0` locally, +/// `3.0` on CI, or the `AWA_CHAOS_TIMEOUT_MULTIPLIER` override (clamped to +/// `1.0 ..= MAX_MULTIPLIER` so the override can only ever loosen, and can +/// never produce a duration `mul_f64` refuses). +pub fn chaos_timeout_multiplier() -> f64 { + let override_var = std::env::var("AWA_CHAOS_TIMEOUT_MULTIPLIER").ok(); + multiplier_from(override_var.as_deref(), std::env::var_os("CI").is_some()) +} + +/// The multiplier decision, with the environment passed in so it can be +/// tested without mutating process-global state. +fn multiplier_from(override_var: Option<&str>, is_ci: bool) -> f64 { + // `"inf"` and `"nan"` parse successfully as f64, and both reach + // `Duration::mul_f64`, which panics rather than saturating. Treat a + // non-finite override as no override at all. + if let Some(parsed) = override_var + .and_then(|raw| raw.parse::().ok()) + .filter(|parsed| parsed.is_finite()) + { + return parsed.clamp(1.0, MAX_MULTIPLIER); + } + + if is_ci { + 3.0 + } else { + 1.0 + } +} + +/// Grow a wait deadline by the contention multiplier. +pub fn scaled_timeout(timeout: Duration) -> Duration { + timeout.mul_f64(chaos_timeout_multiplier()) +} + +/// Grow a heartbeat-staleness window by the contention multiplier. +/// +/// Distinct from [`scaled_timeout`] only in intent: this one is passed to +/// `ClientBuilder::heartbeat_staleness`, where the cost of being too tight +/// is a spurious rescue of a live attempt rather than a timeout. Paired +/// heartbeat/rescue *intervals* are deliberately left unscaled so the +/// rescue path still runs at chaos cadence. +pub fn scaled_staleness(staleness: Duration) -> Duration { + staleness.mul_f64(chaos_timeout_multiplier()) +} + +/// Shrink a minimum-progress floor by the contention multiplier. +/// +/// Use for "this subsystem must have advanced at least N times" gates. +/// The returned floor is at least 1: the regression these gates exist to +/// catch is a fully stalled subsystem, so zero progress must still fail. +pub fn contention_floor(nominal: i64) -> i64 { + let scaled = (nominal as f64 / chaos_timeout_multiplier()).floor() as i64; + scaled.max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_runs_keep_strict_bounds() { + assert_eq!(multiplier_from(None, false), 1.0); + } + + #[test] + fn ci_runs_get_margin() { + assert_eq!(multiplier_from(None, true), 3.0); + } + + #[test] + fn override_can_only_loosen() { + assert_eq!(multiplier_from(Some("0.1"), true), 1.0); + assert_eq!(multiplier_from(Some("-5"), true), 1.0); + assert_eq!(multiplier_from(Some("6"), false), 6.0); + } + + #[test] + fn unparseable_override_falls_back_to_the_environment() { + assert_eq!(multiplier_from(Some("banana"), true), 3.0); + assert_eq!(multiplier_from(Some(""), false), 1.0); + } + + #[test] + fn non_finite_override_falls_back_to_the_environment() { + // These all parse as f64 and would otherwise reach `mul_f64`, which + // panics on a non-finite result instead of saturating. + for raw in ["inf", "-inf", "infinity", "NaN", "nan"] { + assert_eq!(multiplier_from(Some(raw), true), 3.0, "raw={raw}"); + assert_eq!(multiplier_from(Some(raw), false), 1.0, "raw={raw}"); + } + } + + #[test] + fn oversized_override_is_clamped() { + assert_eq!(multiplier_from(Some("1e308"), true), MAX_MULTIPLIER); + assert_eq!(multiplier_from(Some("10000"), true), MAX_MULTIPLIER); + } + + #[test] + fn every_multiplier_produces_a_usable_duration() { + // `Duration::mul_f64` panics rather than saturating, so the clamp is + // what keeps a bad override from failing the nightly it was set to + // rescue. + for raw in [ + "inf", "-inf", "NaN", "1e308", "10000", "0", "-1", "banana", "", + ] { + for is_ci in [true, false] { + let m = multiplier_from(Some(raw), is_ci); + assert!(m.is_finite() && (1.0..=MAX_MULTIPLIER).contains(&m)); + // Would panic if the multiplier were unbounded. + let _ = Duration::from_secs(60).mul_f64(m); + } + } + } + + #[test] + fn scaling_a_timeout_grows_it() { + // 250ms staleness against a 50ms heartbeat interval is a 5x margin + // locally; on CI it becomes 15x, which is what stops a contended + // runner's live worker from being rescued as if it had died. + assert_eq!( + Duration::from_millis(250).mul_f64(multiplier_from(None, true)), + Duration::from_millis(750) + ); + } + + #[test] + fn progress_floor_keeps_margin_under_the_observed_rate() { + // The receipt gate's nominal floor is duration_secs / 4 = 45 for a + // 180s run. #399 saw a healthy run produce 41, so the CI floor has + // to sit well under that. + let ci_floor = (45.0_f64 / multiplier_from(None, true)).floor() as i64; + assert_eq!(ci_floor, 15); + assert!( + ci_floor < 41, + "floor must clear the observed operating point" + ); + } + + #[test] + fn progress_floor_never_reaches_zero() { + // Whatever the multiplier, a fully pinned ring (zero rotations) has + // to keep failing — that is the regression the gate exists for. + assert_eq!(contention_floor(45), contention_floor(45).max(1)); + let absurd = (45.0_f64 / multiplier_from(Some("10000"), true)).floor() as i64; + assert_eq!(absurd.max(1), 1); + assert_eq!(contention_floor(0), 1); + } +} diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index a060369f8..8174c7d66 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -3,6 +3,8 @@ //! These tests exercise the full dispatcher/worker/maintenance wiring with the //! queue_storage backend enabled. +mod ci_timing; + use awa::model::{ admin, batch_operations, insert, migrations, storage, AwaError, BatchOperationFilter, BatchOperationSpec, PruneOutcome, QueueStorage, QueueStorageConfig, RotateOutcome, SkipReason, @@ -2009,6 +2011,151 @@ impl Worker for MultiClientTrackingWorker { } } +#[derive(Debug, Serialize, Deserialize, JobArgs)] +struct FinalizeBacklogJob { + n: i64, +} + +/// A runtime's finished attempts stop heartbeating once they are handed to +/// finalization, so it must stop claiming when finalization falls behind. +/// Otherwise the backlog of finished, unheartbeated attempts grows with the +/// claim rate, outlives `heartbeat_staleness`, and rescue runs those jobs +/// again. Holding the first jobs' receipt-completion locks stalls the +/// completion batcher, standing in for a batcher that cannot keep up. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_runtime_stops_claiming_while_finalization_is_stalled() { + // The default completion pipeline: one shard of 512-job batches, one + // batch in flight and one queued. + const FINALIZER_CAPACITY: usize = 1_024; + const WORKERS: u32 = 64; + const JOBS: i64 = 1_600; + let (_db_guard, pool) = setup_pool(10).await; + let queue = "qs_finalize_backlog"; + let store_config = QueueStorageConfig { + schema: "awa_qs_finalize_backlog".to_string(), + ..Default::default() + }; + let store = create_store_with_config(&pool, store_config.clone()).await; + for first in (0..JOBS).step_by(500) { + let params: Vec<_> = (first..(first + 500).min(JOBS)) + .map(|n| { + insert::params_with( + &FinalizeBacklogJob { n }, + InsertOpts { + queue: queue.to_string(), + ..Default::default() + }, + ) + .expect("params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue backlog"); + } + + let first_jobs: Vec = sqlx::query_scalar(&format!( + "SELECT job_id FROM {}.ready_entries WHERE queue = $1 ORDER BY lane_seq LIMIT $2", + store.schema() + )) + .bind(queue) + .bind(i64::from(WORKERS) * 2) + .fetch_all(&pool) + .await + .expect("first jobs"); + let mut lock_holder = pool.acquire().await.expect("lock connection"); + sqlx::query( + "SELECT pg_catalog.pg_advisory_lock(pg_catalog.hashtextextended( + format('awa.receipt.complete:%s:%s', job_id, 1), 0)) + FROM unnest($1::bigint[]) AS job_id", + ) + .bind(&first_jobs) + .execute(&mut *lock_holder) + .await + .expect("stall the first jobs' finalization"); + + let runs: Arc>> = Arc::default(); + let recorded = runs.clone(); + let client = Client::builder(pool.clone()) + .queue( + queue, + QueueConfig { + max_workers: WORKERS, + poll_interval: Duration::from_millis(10), + deadline_duration: Duration::ZERO, + claim_batch_size: WORKERS as usize, + ..QueueConfig::default() + }, + ) + .queue_storage( + store_config, + Duration::from_secs(60), + Duration::from_millis(50), + ) + .claim_rotate_interval(Duration::from_secs(60)) + .register::(move |_, ctx| { + *recorded + .lock() + .expect("runs") + .entry(ctx.job.id) + .or_insert(0) += 1; + async { Ok(JobResult::Completed) } + }) + .heartbeat_staleness(Duration::from_secs(300)) + .leader_election_interval(Duration::from_millis(100)) + .leader_check_interval(Duration::from_millis(50)) + .build() + .expect("build finalize backlog client"); + client.start().await.expect("start finalize backlog client"); + + // Wait for claiming to stop, or to run past the bound. A fixed runtime + // can never pass it, so a pause in claiming only weakens the check. + let limit = FINALIZER_CAPACITY + WORKERS as usize; + let deadline = Instant::now() + ci_timing::scaled_timeout(Duration::from_secs(30)); + let mut last = (0, Instant::now()); + let started_while_stalled = loop { + let started = runs.lock().expect("runs").len(); + if started > limit { + break started; + } + if started != last.0 { + last = (started, Instant::now()); + } else if started > 0 && last.1.elapsed() >= Duration::from_secs(5) { + break started; + } + assert!( + Instant::now() < deadline, + "claiming neither stopped nor passed {limit}" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + }; + + sqlx::query("SELECT pg_catalog.pg_advisory_unlock_all()") + .execute(&mut *lock_holder) + .await + .expect("release the stalled finalization"); + drop(lock_holder); + let deadline = Instant::now() + ci_timing::scaled_timeout(Duration::from_secs(60)); + while completed_terminal_count(&pool, &store, queue).await < JOBS { + assert!(Instant::now() < deadline, "the backlog did not drain"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + client.shutdown(Duration::from_secs(10)).await; + + assert!( + started_while_stalled <= limit, + "{started_while_stalled} jobs started while finalization was stalled; \ + at most {limit} may wait without a heartbeat or hold a worker" + ); + let runs = runs.lock().expect("runs"); + assert_eq!(runs.len() as i64, JOBS); + assert!( + runs.values().all(|count| *count == 1), + "a job ran more than once" + ); +} + /// ADR-023 claim-ring control-plane smoke test. Exercises /// `rotate_claims`, the busy-check, and `prune_oldest_claims` on an /// empty schema: rotate cycles through every slot, prune is a noop From b90f36fa27dbe19cd47015e5c23fef3965e67d1b Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 19:48:31 +1300 Subject: [PATCH 03/22] fix(metrics): keep the rescue backlog-age name constant on one line The OTel-to-Prometheus name check reads each name constant from a single line. --- awa-metrics/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/awa-metrics/src/lib.rs b/awa-metrics/src/lib.rs index 1c027fbf1..929aa3dc8 100644 --- a/awa-metrics/src/lib.rs +++ b/awa-metrics/src/lib.rs @@ -62,8 +62,7 @@ pub mod names { pub const HEARTBEAT_BATCHES: &str = "awa.heartbeat.batches"; pub const MAINTENANCE_RESCUES: &str = "awa.maintenance.rescues"; pub const MAINTENANCE_RESCUE_PASS_SIZE: &str = "awa.maintenance.rescue.pass_size"; - pub const MAINTENANCE_RESCUE_OLDEST_ATTEMPT_AGE: &str = - "awa.maintenance.rescue.oldest_attempt_age"; + pub const MAINTENANCE_RESCUE_OLDEST_AGE: &str = "awa.maintenance.rescue.oldest_attempt_age"; pub const MAINTENANCE_PROMOTE_BATCHES: &str = "awa.maintenance.promote_batches"; pub const MAINTENANCE_PROMOTE_BATCH_SIZE: &str = "awa.maintenance.promote_batch_size"; pub const MAINTENANCE_PROMOTE_DURATION: &str = "awa.maintenance.promote_duration"; @@ -437,7 +436,7 @@ impl AwaMetrics { .with_boundaries(COUNT_BUCKETS.to_vec()) .build(), maintenance_rescue_oldest_attempt_age_seconds: meter - .f64_gauge(names::MAINTENANCE_RESCUE_OLDEST_ATTEMPT_AGE) + .f64_gauge(names::MAINTENANCE_RESCUE_OLDEST_AGE) .with_description( "Age of the oldest attempt rescued by the latest queue-storage rescue pass", ) From dd44e05ba88d72351ec7dabe142f067f55308e6d Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 20:20:41 +1300 Subject: [PATCH 04/22] test(queue-storage): spread the abandoned backlog over claim slots deterministically The doomed runtime now claims the backlog in chunks, and the test checks the open claims span several slots directly rather than inferring it from ring rotations, which a fast claim phase could finish within one slot. --- awa/tests/bulk_rescue_test.rs | 63 +++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index fb474e7c5..7bcdd3afd 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -21,6 +21,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; const ABANDONED: i64 = 50_000; +const ABANDONED_CHUNK: i64 = 10_000; /// Every tenth abandoned job has one attempt, so its rescue exhausts it. const EXHAUSTED_EVERY: i64 = 10; const ABANDONED_QUEUE: &str = "bulk_rescue_abandoned"; @@ -85,6 +86,20 @@ async fn claim_ring_generation(pool: &PgPool) -> i64 { .await } +async fn abandoned_claim_slots(pool: &PgPool) -> i64 { + count( + pool, + format!( + "SELECT count(DISTINCT claim_slot) FROM ( + SELECT claim_slot FROM awa.lease_claim_batches WHERE queue = '{ABANDONED_QUEUE}' + UNION ALL + SELECT claim_slot FROM awa.lease_claims WHERE queue = '{ABANDONED_QUEUE}' + ) AS claims" + ), + ) + .await +} + /// Rescued and dead-lettered rows for a queue: what a false rescue would leave. async fn rescued_rows(pool: &PgPool, queue: &str) -> i64 { sqlx::query_scalar( @@ -421,8 +436,14 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { }) }; - for chunk_start in (0..ABANDONED).step_by(5_000) { - let params: Vec<_> = (chunk_start..(chunk_start + 5_000).min(ABANDONED)) + // The doomed runtime claims the backlog in chunks, each after the claim + // ring has rotated, so the abandoned claims span several slots. + let generation_before = claim_ring_generation(&pool).await; + let doomed = DoomedClient::start(db.url(), staleness, heartbeat_interval); + let claim_started = Instant::now(); + for chunk_start in (0..ABANDONED).step_by(ABANDONED_CHUNK as usize) { + let chunk_end = (chunk_start + ABANDONED_CHUNK).min(ABANDONED); + let params: Vec<_> = (chunk_start..chunk_end) .map(|n| { awa::model::insert::params_with( &AbandonedJob { n }, @@ -435,20 +456,27 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { .expect("abandoned params") }) .collect(); - store - .enqueue_params_batch(&pool, ¶ms) - .await - .expect("enqueue abandoned jobs"); + let generation = claim_ring_generation(&pool).await; + for batch in params.chunks(5_000) { + store + .enqueue_params_batch(&pool, batch) + .await + .expect("enqueue abandoned jobs"); + } + wait_until( + scaled_timeout(Duration::from_secs(60)), + "the doomed runtime to claim the chunk", + || async { running(&pool, &store, ABANDONED_QUEUE).await >= chunk_end }, + ) + .await; + // Give the ring a chance to move on before the next chunk; once it + // wraps onto a slot of abandoned claims it stays there. + let rotate_deadline = Instant::now() + scaled_timeout(Duration::from_secs(3)); + while claim_ring_generation(&pool).await <= generation && Instant::now() < rotate_deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } } - - let generation_before = claim_ring_generation(&pool).await; - let doomed = DoomedClient::start(db.url(), staleness, heartbeat_interval); - let claimed_in = wait_until( - scaled_timeout(Duration::from_secs(120)), - "the doomed runtime to claim every abandoned job", - || async { running(&pool, &store, ABANDONED_QUEUE).await >= ABANDONED }, - ) - .await; + let claimed_in = claim_started.elapsed(); assert_eq!( rescued_rows(&pool, ABANDONED_QUEUE).await, 0, @@ -457,9 +485,10 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { let generation_at_kill = claim_ring_generation(&pool).await; live_hold_ms.store(live_job_duration.as_millis() as u64, Ordering::SeqCst); doomed.kill(); + let slots = abandoned_claim_slots(&pool).await; assert!( - generation_at_kill - generation_before >= 2, - "the abandoned claims should span several claim slots \ + slots >= 2, + "the abandoned claims should span several claim slots: {slots} \ (ring generation {generation_before} -> {generation_at_kill})" ); From 717877c81710b0d9236329657963408725517195 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 21:23:58 +1300 Subject: [PATCH 05/22] fix(rescue): preserve bounded scan progress and gate early passes Closed or freshly heartbeated history can exhaust a scan without rescuing anything. Report scan progress independently of rescued counts, preserve complete sweep boundaries across early passes, and revisit finished slots at the ordinary cadence. Keep first-batch heartbeat rescue on its original cadence unless settled owner evidence allows acceleration. Remove the retry pacing horizon because normal backoffs can cluster due times; document the actual rescue write bound. Validation: cargo fmt --all; SQLX_OFFLINE=true cargo clippy --all-targets --all-features -- -D warnings; SQLX_OFFLINE=true cargo build --workspace; seven maintenance rescue tests on PostgreSQL 17/18; sparse-history and 501-attempt rescue regressions on PostgreSQL 17. Broader rescue tests continue on both engines. --- awa-model/src/lib.rs | 4 +- awa-model/src/queue_storage.rs | 136 ++++++-- awa-worker/src/client.rs | 6 +- awa-worker/src/maintenance.rs | 418 ++++++++++++++++++++---- awa/tests/queue_storage_runtime_test.rs | 182 +++++++++++ docs/configuration.md | 8 +- docs/grafana/README.md | 2 +- docs/troubleshooting.md | 6 +- 8 files changed, 654 insertions(+), 108 deletions(-) diff --git a/awa-model/src/lib.rs b/awa-model/src/lib.rs index acff1469e..2ed204076 100644 --- a/awa-model/src/lib.rs +++ b/awa-model/src/lib.rs @@ -46,8 +46,8 @@ pub use partitioned_queue::{ }; pub use queue_storage::{ ClaimedEntry, ClaimedRuntimeJob, PruneOutcome, QueueCounts, QueueStorage, QueueStorageConfig, - RescueBatch, RescueBatchOptions, RescueSlotOrder, RotateOutcome, SkipReason, - TerminalDeltaRollupOutcome, + RescueBatch, RescueBatchOptions, RescueScanCursor, RescueScanProgress, RescueSlotOrder, + RotateOutcome, SkipReason, TerminalDeltaRollupOutcome, }; pub use storage::StorageStatus; diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 6a8084048..ade6d7ef6 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -379,11 +379,30 @@ pub struct RescueBatchOptions { #[derive(Debug, Default)] pub struct RescueBatch { pub rescued: Vec, - /// A scan stopped at its row limit, so more attempts may already be - /// eligible. False means the batch rescued everything it found. + /// A scan window or rescue quota was exhausted, so another bounded + /// batch may find eligible attempts. pub saturated: bool, - /// Claim slots this batch searched and found nothing to rescue in. + /// Claim slots whose bounded scans completed without eligible attempts. pub idle_slots: Vec, + /// Last cursor reached by each bounded receipt scan. A caller continuing + /// batches can stop a sweep once its cursor returns to its first window. + pub scan_progress: Vec, +} + +/// Cursor progress from a bounded receipt rescue scan. +#[derive(Debug, Clone)] +pub struct RescueScanProgress { + pub slot: i32, + /// One cursor per representation, in a stable order. A slot is drained + /// only after every representation has completed its sweep. + pub cursors: Vec, +} + +/// Progress of one independently ordered receipt representation. +#[derive(Debug, Clone)] +pub struct RescueScanCursor { + pub position: Option<(DateTime, i64, i64)>, + pub saturated: bool, } /// Receipt claims one rescue batch took, and what it found on the way. @@ -392,8 +411,22 @@ struct ReceiptRescueBatch { rescued: Vec, saturated: bool, idle_slots: Vec, + scan_progress: Vec, } +type ReceiptRescueScan = ( + Vec, + bool, + Option<(DateTime, i64, i64)>, +); +type ReceiptRescueScanRow = ( + sqlx::types::Json>, + bool, + Option>, + Option, + Option, +); + /// Where a rescue sends an exhausted attempt on a queue whose DLQ is enabled. struct RescueDlq<'a> { reason: &'static str, @@ -2201,7 +2234,7 @@ struct ExistingReadyRow { payload: serde_json::Value, } -#[derive(Debug, Clone, sqlx::FromRow)] +#[derive(Debug, Clone, serde::Deserialize, sqlx::FromRow)] struct DeletedLeaseRow { ready_slot: i32, ready_generation: i64, @@ -10913,7 +10946,7 @@ impl QueueStorage { ) -> Result<(), AwaError> { let mut remaining = RECEIPT_RESCUE_BATCH_LIMIT; for slot in self.rescue_slot_order_tx(tx, options).await? { - let slot_rescued = self + let (slot_rescued, scan_full, cursor) = self .rescue_stale_receipt_claims_for_slot_tx( tx, slot, @@ -10926,9 +10959,17 @@ impl QueueStorage { remaining, ) .await?; - if slot_rescued.is_empty() { + batch.saturated |= scan_full; + if !scan_full && slot_rescued.is_empty() { batch.idle_slots.push(slot); } + batch.scan_progress.push(RescueScanProgress { + slot, + cursors: vec![RescueScanCursor { + position: cursor, + saturated: scan_full, + }], + }); remaining = remaining.saturating_sub(slot_rescued.len() as i64); batch.rescued.extend(slot_rescued); if remaining == 0 { @@ -10976,7 +11017,7 @@ impl QueueStorage { cutoff: DateTime, scan_limit: i64, rescue_limit: i64, - ) -> Result, AwaError> { + ) -> Result { let schema = self.schema(); let claim_child = claim_child_name(schema, slot as usize); let claim_batch_child = claim_batch_child_name(schema, slot as usize); @@ -11309,7 +11350,7 @@ impl QueueStorage { .map_err(map_sqlx_error)? .into_iter() .unzip(); - let rescued: Vec = sqlx::query_as(&format!( + let (rescued, scan_full, cursor_at, cursor_job_id, cursor_run_lease): ReceiptRescueScanRow = sqlx::query_as(&format!( r#" WITH {candidate_ctes}, stale_row_locked AS ( @@ -11505,11 +11546,9 @@ impl QueueStorage { rescue_cursor_run_lease = advance_target.run_lease FROM advance_target WHERE slots.slot = $1 - RETURNING slots.slot + RETURNING slots.rescue_cursor_claimed_at, slots.rescue_cursor_job_id, slots.rescue_cursor_run_lease ), - cursor_advance AS ( - SELECT count(*) FROM advance_cursor - ) + rescued_rows AS ( SELECT stale_locked.ready_slot, stale_locked.ready_generation, @@ -11528,7 +11567,13 @@ impl QueueStorage { ON closed_locked.claim_slot = stale_locked.claim_slot AND closed_locked.job_id = stale_locked.job_id AND closed_locked.run_lease = stale_locked.run_lease - CROSS JOIN cursor_advance + ) + SELECT + COALESCE((SELECT jsonb_agg(to_jsonb(rescued_rows)) FROM rescued_rows), '[]'::jsonb), + (SELECT count(*) >= $3 FROM candidates), + (SELECT rescue_cursor_claimed_at FROM advance_cursor), + (SELECT rescue_cursor_job_id FROM advance_cursor), + (SELECT rescue_cursor_run_lease FROM advance_cursor) "# )) .bind(slot) @@ -11537,10 +11582,18 @@ impl QueueStorage { .bind(rescue_limit) .bind(&locked_job_ids) .bind(&locked_run_leases) - .fetch_all(tx.as_mut()) + .fetch_one(tx.as_mut()) .await .map_err(map_sqlx_error)?; - Ok(rescued) + let scan_full = scan_full || rescued.0.len() as i64 >= rescue_limit; + Ok(( + rescued.0, + scan_full, + cursor_at + .zip(cursor_job_id) + .zip(cursor_run_lease) + .map(|((at, id), lease)| (at, id, lease)), + )) } /// Receipt-side counterpart to `rescue_expired_deadlines`: scans @@ -11568,7 +11621,7 @@ impl QueueStorage { ) -> Result<(), AwaError> { let mut remaining = RECEIPT_RESCUE_BATCH_LIMIT; for slot in self.rescue_slot_order_tx(tx, options).await? { - let slot_rescued = self + let (slot_rescued, scan_full, cursor) = self .rescue_expired_receipt_deadlines_for_slot_tx( tx, slot, @@ -11580,9 +11633,17 @@ impl QueueStorage { remaining, ) .await?; - if slot_rescued.is_empty() { + batch.saturated |= scan_full; + if !scan_full && slot_rescued.is_empty() { batch.idle_slots.push(slot); } + batch.scan_progress.push(RescueScanProgress { + slot, + cursors: vec![RescueScanCursor { + position: cursor, + saturated: scan_full, + }], + }); remaining = remaining.saturating_sub(slot_rescued.len() as i64); batch.rescued.extend(slot_rescued); if remaining == 0 { @@ -11640,9 +11701,9 @@ impl QueueStorage { slot: i32, scan_limit: i64, rescue_limit: i64, - ) -> Result, AwaError> { + ) -> Result { if rescue_limit <= 0 { - return Ok(Vec::new()); + return Ok((Vec::new(), false, None)); } let schema = self.schema(); @@ -11779,7 +11840,7 @@ impl QueueStorage { .map_err(map_sqlx_error)? .into_iter() .unzip(); - let rescued: Vec = sqlx::query_as(&format!( + let (rescued, scan_full, cursor_at, cursor_job_id, cursor_run_lease): ReceiptRescueScanRow = sqlx::query_as(&format!( r#" WITH {candidate_ctes}, expired_locked AS ( @@ -11857,11 +11918,9 @@ impl QueueStorage { deadline_cursor_run_lease = advance_target.run_lease FROM advance_target WHERE slots.slot = $1 - RETURNING slots.slot + RETURNING slots.deadline_cursor_deadline_at, slots.deadline_cursor_job_id, slots.deadline_cursor_run_lease ), - cursor_advance AS ( - SELECT count(*) FROM advance_cursor - ) + rescued_rows AS ( SELECT expired_locked.ready_slot, expired_locked.ready_generation, @@ -11880,7 +11939,13 @@ impl QueueStorage { ON inserted.claim_slot = expired_locked.claim_slot AND inserted.job_id = expired_locked.job_id AND inserted.run_lease = expired_locked.run_lease - CROSS JOIN cursor_advance + ) + SELECT + COALESCE((SELECT jsonb_agg(to_jsonb(rescued_rows)) FROM rescued_rows), '[]'::jsonb), + (SELECT count(*) >= $2 FROM candidates), + (SELECT deadline_cursor_deadline_at FROM advance_cursor), + (SELECT deadline_cursor_job_id FROM advance_cursor), + (SELECT deadline_cursor_run_lease FROM advance_cursor) "# )) .bind(slot) @@ -11888,10 +11953,18 @@ impl QueueStorage { .bind(rescue_limit) .bind(&locked_job_ids) .bind(&locked_run_leases) - .fetch_all(tx.as_mut()) + .fetch_one(tx.as_mut()) .await .map_err(map_sqlx_error)?; - Ok(rescued) + let scan_full = scan_full || rescued.0.len() as i64 >= rescue_limit; + Ok(( + rescued.0, + scan_full, + cursor_at + .zip(cursor_job_id) + .zip(cursor_run_lease) + .map(|((at, id), lease)| (at, id, lease)), + )) } pub async fn load_job(&self, pool: &PgPool, job_id: i64) -> Result, AwaError> { @@ -14618,12 +14691,15 @@ impl QueueStorage { } saturated |= receipts.saturated; let idle_slots = receipts.idle_slots; + let scan_progress = receipts.scan_progress; let rescued_receipts = receipts.rescued; if deleted.is_empty() && rescued_receipts.is_empty() { tx.commit().await.map_err(map_sqlx_error)?; return Ok(RescueBatch { idle_slots, + scan_progress, + saturated, ..RescueBatch::default() }); } @@ -14663,6 +14739,7 @@ impl QueueStorage { rescued, saturated, idle_slots, + scan_progress, }) } @@ -14752,12 +14829,15 @@ impl QueueStorage { } saturated |= receipts.saturated; let idle_slots = receipts.idle_slots; + let scan_progress = receipts.scan_progress; let receipt_deleted = receipts.rescued; if deleted.is_empty() && receipt_deleted.is_empty() { tx.commit().await.map_err(map_sqlx_error)?; return Ok(RescueBatch { idle_slots, + scan_progress, + saturated, ..RescueBatch::default() }); } @@ -14785,6 +14865,7 @@ impl QueueStorage { rescued, saturated, idle_slots, + scan_progress, }) } @@ -14887,6 +14968,7 @@ impl QueueStorage { rescued, saturated, idle_slots: Vec::new(), + scan_progress: Vec::new(), }) } diff --git a/awa-worker/src/client.rs b/awa-worker/src/client.rs index 30b28b674..aa91ec7e4 100644 --- a/awa-worker/src/client.rs +++ b/awa-worker/src/client.rs @@ -773,8 +773,10 @@ impl ClientBuilder { /// Set how many attempts one queue-storage rescue pass rescues before it /// stops (default: 5000). With a pass a second while a backlog remains, - /// this caps a drain at about this many attempts per second per rescue - /// kind, and rescued retries become due no faster than this per second. + /// this bounds the number rescued per second per kind, allowing one + /// final batch beyond the cap. Zero disables follow-up batches and early + /// re-arms. Retried attempts retain their normal backoff; their due times + /// can cluster, so this does not cap the promotion rate. pub fn rescue_pass_max_rows(mut self, max_rows: usize) -> Self { self.rescue_pass_max_rows = Some(max_rows); self diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index 893a13001..1fd02c7c7 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -7,10 +7,10 @@ use awa_model::cron::{ #[cfg(test)] use awa_model::SkipReason; use awa_model::{ - JobRow, JobState, PeriodicJob, PruneOutcome, RescueBatch, RescueBatchOptions, RescueSlotOrder, - RotateOutcome, TerminalDeltaRollupOutcome, + JobRow, JobState, PeriodicJob, PruneOutcome, RescueBatch, RescueBatchOptions, + RescueScanProgress, RescueSlotOrder, RotateOutcome, TerminalDeltaRollupOutcome, }; -use chrono::{DateTime, TimeDelta, Utc}; +use chrono::{DateTime, Utc}; use croner::Cron; use sqlx::pool::PoolConnection; use sqlx::{PgPool, Postgres}; @@ -231,10 +231,6 @@ pub(crate) const DEFAULT_RESCUE_PASS_BUDGET: Duration = Duration::from_secs(1); /// Default cap on attempts one queue-storage rescue pass rescues. pub(crate) const DEFAULT_RESCUE_PASS_MAX_ROWS: usize = 5_000; -/// Attempts one rescue batch rescues at most per source (lease rows, receipt -/// claims), each batch in its own transaction. -const RESCUE_BATCH_ROWS: usize = 500; - /// While a rescue kind still has a backlog, its next pass runs after this /// rather than after its full interval. With the per-pass row cap this bounds /// a drain to about `rescue_pass_max_rows` attempts per second per kind. @@ -249,8 +245,7 @@ const RESCUE_FOLLOW_UP_SCAN_LIMIT: usize = 2_000; /// a long drain of the oldest slot leaves the rest at least that share. const RESCUE_OTHERS_FIRST_EVERY: usize = 4; -/// Runtime snapshots the leader reads before letting a stale-heartbeat pass -/// take more than one batch. A runtime is `fresh` while its snapshot is +/// Runtime snapshots the leader reads before accelerating stale-heartbeat rescue. A runtime is `fresh` while its snapshot is /// within three snapshot intervals; one that is neither fresh nor silent for /// the whole heartbeat staleness ($1, milliseconds) could be a live runtime /// that is stalled rather than gone. `last_gone_heartbeat` bounds the last @@ -277,7 +272,7 @@ const RESCUE_RUNTIME_EVIDENCE_SQL: &str = r#" "#; /// The queue-storage rescue kinds a leader drains in passes of batches. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum RescueKind { Heartbeat, Deadline, @@ -333,7 +328,7 @@ impl RescuePassEnd { /// fleet-wide: every runtime snapshot is either fresh with its heartbeat /// loop alive, or silent for at least the heartbeat staleness. That must /// hold at every sample for a full staleness window before a stale-heartbeat -/// pass may take more than one batch, so a live runtime coming out of a stall +/// rescue may run extra batches or early passes, so a live runtime coming out of a stall /// has had the whole window to refresh its heartbeats first. A gap between /// samples longer than half the staleness restarts the window, because a /// stall the leader did not sample could have hidden behind it. @@ -363,6 +358,96 @@ impl RescueOwnerEvidence { } } +/// A sweep survives early re-arms: closed history must not keep the leader +/// scanning once a cursor has made a full revolution. The first window is +/// already visited, so crossing its end after wrapping completes the sweep. +#[derive(Debug, Default)] +struct RescueSweep { + cursors: HashMap>, + idle_slots: Vec, + last_revisit: Option, + batches_started: usize, +} + +#[derive(Debug, Default)] +struct RescueSweepCursor { + first: Option<(DateTime, i64, i64)>, + previous: Option<(DateTime, i64, i64)>, + wrapped: bool, + done: bool, +} + +impl RescueSweep { + /// A long drain must revisit slots whose claims have become eligible + /// since their last scan. Keep unfinished cursors so a large history + /// sweep still converges across these ordinary-cadence revisits. + fn revisit(&mut self, now: Instant, interval: Duration) { + if self + .last_revisit + .is_none_or(|last| now.saturating_duration_since(last) >= interval) + { + self.idle_slots.clear(); + for cursor in self.cursors.values_mut().flatten() { + if cursor.done { + *cursor = RescueSweepCursor::default(); + } + } + self.last_revisit = Some(now); + } + } + + fn observe(&mut self, progress: RescueScanProgress) { + let cursors = self.cursors.entry(progress.slot).or_default(); + cursors.resize_with(progress.cursors.len(), RescueSweepCursor::default); + for (cursor, progress) in cursors.iter_mut().zip(progress.cursors) { + if !progress.saturated { + cursor.done = true; + } + if let Some(position) = progress.position { + if let Some(first) = cursor.first { + let crossed = cursor.previous.is_some_and(|previous| position <= previous); + cursor.done |= (cursor.wrapped && crossed) + || ((cursor.wrapped || crossed) && position >= first); + cursor.wrapped |= crossed; + } else { + cursor.first = Some(position); + } + cursor.previous = Some(position); + } + } + if !cursors.is_empty() && cursors.iter().all(|cursor| cursor.done) { + self.idle_slots.push(progress.slot); + } + } +} + +/// An early re-arm may run after owner evidence disappears. Keep ordinary +/// first batches on their original cadence; every accelerated batch needs +/// the same owner cutoff as follow-up batches. +#[derive(Debug, Default)] +struct RescueHeartbeatCadence { + last_baseline: Option, +} + +impl RescueHeartbeatCadence { + fn select( + &mut self, + now: Instant, + interval: Duration, + cutoff: Option>, + ) -> Option>> { + if self + .last_baseline + .is_none_or(|last| now.saturating_duration_since(last) >= interval) + { + self.last_baseline = Some(now); + Some(None) + } else { + cutoff.map(Some) + } + } +} + /// Per-queue or global retention policy for completed and failed/cancelled jobs. #[derive(Debug, Clone)] pub struct RetentionPolicy { @@ -827,9 +912,8 @@ pub struct MaintenanceService { rescue_pass_budget: Duration, rescue_pass_max_rows: usize, rescue_owner_evidence: std::sync::Mutex, - /// Earliest `run_at` for the next attempt a queue-storage rescue sends - /// back to `retryable`; see [`Self::rescue_retry_floor`]. - rescue_retry_horizon: std::sync::Mutex>>, + rescue_heartbeat_cadence: std::sync::Mutex, + rescue_sweeps: std::sync::Mutex>, rescue_others_first_offset: AtomicUsize, promote_interval: Duration, cleanup_interval: Duration, @@ -902,7 +986,8 @@ impl MaintenanceService { rescue_pass_budget: DEFAULT_RESCUE_PASS_BUDGET, rescue_pass_max_rows: DEFAULT_RESCUE_PASS_MAX_ROWS, rescue_owner_evidence: std::sync::Mutex::new(RescueOwnerEvidence::default()), - rescue_retry_horizon: std::sync::Mutex::new(None), + rescue_heartbeat_cadence: std::sync::Mutex::new(RescueHeartbeatCadence::default()), + rescue_sweeps: std::sync::Mutex::new(HashMap::new()), rescue_others_first_offset: AtomicUsize::new(0), promote_interval: Duration::from_millis(250), cleanup_interval: Duration::from_secs(60), @@ -1019,7 +1104,8 @@ impl MaintenanceService { } /// Set how many attempts one queue-storage rescue pass rescues before it - /// stops (default: 5000). A pass always finishes the batch it started. + /// stops (default: 5000). A pass always finishes the batch it started; + /// zero keeps one batch per ordinary interval, as a zero pass budget does. pub fn rescue_pass_max_rows(mut self, max_rows: usize) -> Self { self.rescue_pass_max_rows = max_rows; self @@ -1178,6 +1264,20 @@ impl MaintenanceService { // Per-branch overrun tracker. Issue #242 — observability only; // the architectural split of this `tokio::select!` is deferred // to v0.7 conditional on the overrun counter showing fleet hits. + *self + .rescue_owner_evidence + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + RescueOwnerEvidence::default(); + *self + .rescue_heartbeat_cadence + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + RescueHeartbeatCadence::default(); + self.rescue_sweeps + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); let branch_tracker = MaintenanceBranchTracker::new(); // Per-segment prune backoff (#169). Gates the prune step of // each rotate branch so a pinned MVCC snapshot doesn't make @@ -1642,13 +1742,41 @@ impl MaintenanceService { }; let budget = self.rescue_pass_budget.min(interval / 2); let dlq_enabled = |queue: &str| self.dlq_policy.enabled_for(queue); - let mut bulk_heartbeat_cutoff: Option>> = None; + let bulk_heartbeat_cutoff = if kind == RescueKind::Heartbeat { + self.stale_claim_owners_gone().await + } else { + None + }; + let first_heartbeat_cutoff = if kind == RescueKind::Heartbeat { + let selected = self + .rescue_heartbeat_cadence + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .select(started, interval, bulk_heartbeat_cutoff); + let Some(cutoff) = selected else { + self.rescue_sweeps + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&kind); + return false; + }; + cutoff + } else { + None + }; let mut rescued_total = 0usize; let mut oldest_attempt: Option> = None; let mut batches = 0usize; - let mut idle_slots: Vec = Vec::new(); + let mut sweep = self + .rescue_sweeps + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&kind) + .unwrap_or_default(); + sweep.revisit(started, interval); let pass_end = loop { - let slot_order = if batches % RESCUE_OTHERS_FIRST_EVERY == RESCUE_OTHERS_FIRST_EVERY - 1 + let slot_order = if sweep.batches_started % RESCUE_OTHERS_FIRST_EVERY + == RESCUE_OTHERS_FIRST_EVERY - 1 { RescueSlotOrder::OthersFirst( self.rescue_others_first_offset @@ -1659,16 +1787,20 @@ impl MaintenanceService { }; let options = RescueBatchOptions { slot_order, - skip_slots: idle_slots.clone(), - retry_not_before: self.rescue_retry_floor(), + skip_slots: sweep.idle_slots.clone(), + retry_not_before: None, cursor_scan_limit: (batches > 0).then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), }; let outcome = match kind { RescueKind::Heartbeat => { // Follow-up batches take only claims last heartbeated // before a runtime known to be gone stopped reporting. - let staleness = bulk_heartbeat_cutoff - .flatten() + let cutoff = if batches == 0 { + first_heartbeat_cutoff + } else { + bulk_heartbeat_cutoff + }; + let staleness = cutoff .and_then(|cutoff| (Utc::now() - cutoff).to_std().ok()) .map_or(self.heartbeat_staleness, |since_cutoff| { since_cutoff.max(self.heartbeat_staleness) @@ -1697,10 +1829,12 @@ impl MaintenanceService { } }; batches += 1; + sweep.batches_started = sweep.batches_started.wrapping_add(1); let RescueBatch { rescued, saturated, idle_slots: batch_idle_slots, + scan_progress, } = match outcome { Ok(batch) => batch, Err(err) => { @@ -1709,14 +1843,11 @@ impl MaintenanceService { break RescuePassEnd::Error; } }; - idle_slots.extend(batch_idle_slots); + sweep.idle_slots.extend(batch_idle_slots); + for progress in scan_progress { + sweep.observe(progress); + } rescued_total += rescued.len(); - self.advance_rescue_retry_horizon( - rescued - .iter() - .filter(|job| job.state == JobState::Retryable) - .count(), - ); if let Some(attempted_at) = rescued.iter().filter_map(|job| job.attempted_at).min() { oldest_attempt = Some(oldest_attempt.map_or(attempted_at, |oldest| oldest.min(attempted_at))); @@ -1727,17 +1858,11 @@ impl MaintenanceService { if !saturated { break RescuePassEnd::Drained; } - if budget.is_zero() { + if budget.is_zero() || self.rescue_pass_max_rows == 0 { break RescuePassEnd::TimeBudget; } - if kind == RescueKind::Heartbeat { - let cutoff = match bulk_heartbeat_cutoff { - Some(cutoff) => cutoff, - None => *bulk_heartbeat_cutoff.insert(self.stale_claim_owners_gone().await), - }; - if cutoff.is_none() { - break RescuePassEnd::OwnersUnconfirmed; - } + if kind == RescueKind::Heartbeat && bulk_heartbeat_cutoff.is_none() { + break RescuePassEnd::OwnersUnconfirmed; } if rescued_total >= self.rescue_pass_max_rows { break RescuePassEnd::RowLimit; @@ -1767,7 +1892,17 @@ impl MaintenanceService { "Queue-storage rescue pass drained a backlog" ); } - pass_end.backlog_remains() && !budget.is_zero() + let rearm = pass_end.backlog_remains() + && !budget.is_zero() + && self.rescue_pass_max_rows > 0 + && (kind != RescueKind::Heartbeat || bulk_heartbeat_cutoff.is_some()); + if rearm { + self.rescue_sweeps + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(kind, sweep); + } + rearm } async fn process_queue_storage_rescues( @@ -1794,7 +1929,7 @@ impl MaintenanceService { } /// Sample the runtime snapshots and update [`RescueOwnerEvidence`]. - /// Returns the heartbeat cutoff for follow-up stale-heartbeat batches + /// Returns the heartbeat cutoff for accelerated stale-heartbeat batches /// when the evidence holds and some runtime is known to be gone; `None` /// keeps the pass to one batch. A failed read counts as no evidence. async fn stale_claim_owners_gone(&self) -> Option> { @@ -1837,34 +1972,6 @@ impl MaintenanceService { } } - /// The earliest `run_at` for the next attempt a queue-storage rescue - /// sends back to `retryable`. Rescued attempts become due no faster than - /// `rescue_pass_max_rows` per second across all rescue kinds, so a large - /// drain reaches promotion and the ready lanes spread over time rather - /// than all at once. `None` while rescue runs below that rate. - fn rescue_retry_floor(&self) -> Option> { - let horizon = *self - .rescue_retry_horizon - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - horizon.filter(|horizon| *horizon > Utc::now()) - } - - fn advance_rescue_retry_horizon(&self, retried: usize) { - if retried == 0 { - return; - } - let per_second = self.rescue_pass_max_rows.max(RESCUE_BATCH_ROWS) as i64; - let spread = TimeDelta::microseconds(retried as i64 * 1_000_000 / per_second); - let now = Utc::now(); - let mut horizon = self - .rescue_retry_horizon - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let start = horizon.map_or(now, |horizon| horizon.max(now)); - *horizon = Some(start + spread); - } - /// Batched canonical stale-heartbeat rescue with the #388 row-at-a-time /// fallback on unique-claim conflict. Exhausted attempts fail in place, /// as callback timeouts do, so a job that keeps killing or hanging its @@ -4471,6 +4578,179 @@ mod tests { ); } + #[test] + fn rescue_accelerated_first_batch_requires_owner_evidence() { + let mut cadence = RescueHeartbeatCadence::default(); + let now = Instant::now(); + let interval = Duration::from_secs(30); + let cutoff = Utc::now(); + assert_eq!(cadence.select(now, interval, None), Some(None)); + assert_eq!( + cadence.select(now + Duration::from_secs(1), interval, Some(cutoff)), + Some(Some(cutoff)) + ); + // The preceding accelerated pass re-armed the timer, but owners + // became unconfirmed before it fired: no ordinary batch may slip in. + assert_eq!( + cadence.select(now + Duration::from_secs(2), interval, None), + None + ); + assert_eq!(cadence.select(now + interval, interval, None), Some(None)); + } + + #[test] + fn rescue_closed_history_sweep_stops_after_wrap() { + let mut sweep = RescueSweep::default(); + let stamp = Utc::now(); + // First window ends midway around a slot. Sweep state survives + // the pass's time budget and the next early timer fire. + for id in [20, 40, 60, 10] { + sweep.observe(RescueScanProgress { + slot: 1, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, id, 1)), + saturated: true, + }], + }); + assert!(sweep.idle_slots.is_empty()); + } + sweep.observe(RescueScanProgress { + slot: 1, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, 20, 1)), + saturated: true, + }], + }); + assert_eq!(sweep.idle_slots, vec![1]); + + let mut one = RescueSweep::default(); + for _ in 0..2 { + one.observe(RescueScanProgress { + slot: 0, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, 1, 1)), + saturated: true, + }], + }); + } + assert_eq!(one.idle_slots, vec![0]); + } + + #[test] + fn rescue_sweep_revisits_newly_eligible_slots_at_the_ordinary_interval() { + let mut sweep = RescueSweep::default(); + let now = Instant::now(); + let interval = Duration::from_secs(30); + let stamp = Utc::now(); + sweep.revisit(now, interval); + // Slot A's deadlines are in the future; B keeps a large pass open. + sweep.observe(RescueScanProgress { + slot: 0, + cursors: vec![awa_model::RescueScanCursor { + position: None, + saturated: false, + }], + }); + sweep.observe(RescueScanProgress { + slot: 1, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, 100, 1)), + saturated: true, + }], + }); + sweep.revisit(now + interval / 2, interval); + assert_eq!(sweep.idle_slots, vec![0]); + // A can now be expired (or contain claims from a reused generation). + sweep.revisit(now + interval, interval); + assert!(sweep.idle_slots.is_empty()); + assert!(!sweep.cursors[&0][0].done); + assert_eq!( + sweep.cursors[&1][0].first, + Some((stamp, 100, 1)), + "revisiting an idle slot must not restart an unfinished sweep" + ); + } + + #[test] + fn rescue_sweep_detects_a_window_crossing_its_origin() { + let mut sweep = RescueSweep::default(); + let stamp = Utc::now(); + // A window can cross the origin and wrap again before its last key. + // Its end alone need never equal or exceed the original last key. + for id in [10_000, 1_999, 3_999, 5_999, 7_999, 9_999, 1_998] { + sweep.observe(RescueScanProgress { + slot: 0, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, id, 1)), + saturated: true, + }], + }); + } + assert_eq!(sweep.idle_slots, vec![0]); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn rescue_closed_history_stops_rearming() { + let _guard = db_test_mutex().lock().await; + let pool = setup_pool(4).await; + reset_schema(&pool).await; + migrations::run(&pool).await.expect("migrations"); + let config = QueueStorageConfig::default(); + let store = QueueStorage::new(config.clone()).expect("queue storage"); + store.install(&pool).await.expect("install"); + sqlx::query( + "INSERT INTO awa.lease_claims + (claim_slot, job_id, run_lease, ready_slot, ready_generation, + queue, priority, attempt, max_attempts, lane_seq, claimed_at, closed_at, deadline_at) + SELECT 0, id, 1, 0, 0, 'closed_history', 0, 1, 3, id, + now() - interval '1 hour', now(), now() - interval '1 minute' + FROM generate_series(1, 10001) AS id", + ) + .execute(&pool) + .await + .expect("seed more closed history than a scan window"); + let runtime = + QueueStorageRuntime::new(config, Duration::from_secs(1), Duration::from_secs(1)) + .expect("runtime"); + let service = MaintenanceService::new( + pool.clone(), + metrics_for_test(), + Arc::new(AtomicBool::new(true)), + Arc::new(AtomicBool::new(true)), + CancellationToken::new(), + Arc::new(Vec::new()), + InFlightMap::default(), + RuntimeStorage::QueueStorage(runtime.clone()), + Arc::new(HashMap::new()), + Arc::new(HashMap::new()), + ) + .rescue_pass_budget(Duration::from_millis(200)); + let mut drained = false; + for _ in 0..20 { + if !service + .rescue_queue_storage_pass(RescueKind::Deadline, &runtime, false) + .await + { + drained = true; + break; + } + } + assert!( + drained, + "closed history must finish a bounded sweep instead of rearming forever" + ); + assert!(service.rescue_sweeps.lock().expect("sweeps").is_empty()); + // Zero is a documented way to retain the ordinary one-batch cadence. + let service = service.rescue_pass_max_rows(0); + assert!( + !service + .rescue_queue_storage_pass(RescueKind::Deadline, &runtime, false) + .await + ); + assert!(service.rescue_sweeps.lock().expect("sweeps").is_empty()); + pool.close().await; + } + #[test] fn rescue_owner_evidence_needs_a_full_settled_staleness_window() { let staleness = Duration::from_secs(90); diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index a060369f8..237e1cee6 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -5610,6 +5610,188 @@ async fn test_queue_storage_receipt_deadline_rescue_cursor_advances_over_termina ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_rescue_batch_keeps_a_partially_rescued_scan_active() { + let (_db_guard, pool) = setup_pool(10).await; + let schema = "awa_rescue_partial_window"; + let queue = "rescue_partial_window"; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: schema.into(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 1, + lease_claim_receipts: true, + claim_slot_count: 2, + }, + ) + .await; + for id in 0..501 { + enqueue_job( + &pool, + &store, + &CompleteJob { id }, + InsertOpts { + queue: queue.into(), + ..Default::default() + }, + ) + .await; + } + let claims = store + .claim_runtime_batch(&pool, queue, 501, Duration::from_secs(30)) + .await + .expect("claim a backlog larger than one rescue batch"); + assert_eq!(claims.len(), 501); + for table in ["lease_claims", "lease_claim_batches"] { + sqlx::query(&format!("UPDATE {schema}.{table} SET claimed_at = now() - interval '1 hour', deadline_at = now() - interval '1 minute'")) + .execute(&pool).await.expect("expire attempts"); + } + let batch = store + .rescue_expired_deadlines_batch(&pool, &|_| false, &Default::default()) + .await + .expect("first rescue batch"); + assert_eq!(batch.rescued.len(), 500); + assert!(batch.saturated); + let slot = claims[0].claim.claim_slot; + assert!(!batch.idle_slots.contains(&slot)); + assert!( + batch + .scan_progress + .iter() + .filter(|scan| scan.slot == slot) + .flat_map(|scan| &scan.cursors) + .any(|cursor| cursor.saturated), + "a partially rescued candidate window must not complete that slot's sweep" + ); + let tail = store + .rescue_expired_deadlines_batch(&pool, &|_| false, &Default::default()) + .await + .expect("rescue the remaining attempt"); + assert_eq!(tail.rescued.len(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_rescue_batch_continues_past_closed_scan_windows() { + for (deadline, fresh_history) in [(false, false), (true, false), (false, true)] { + let (_db_guard, pool) = setup_pool(10).await; + let suffix = if deadline { + "deadline" + } else if fresh_history { + "fresh" + } else { + "heartbeat" + }; + let schema = format!("awa_rescue_sparse_{suffix}"); + let queue = format!("rescue_sparse_{suffix}"); + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: schema.clone(), + queue_slot_count: 4, + lease_slot_count: 2, + queue_stripe_count: 1, + lease_claim_receipts: true, + claim_slot_count: 2, + }, + ) + .await; + for id in 0..3 { + enqueue_job( + &pool, + &store, + &CompleteJob { id }, + InsertOpts { + queue: queue.clone(), + ..Default::default() + }, + ) + .await; + } + let claims = store + .claim_runtime_batch(&pool, &queue, 3, Duration::from_secs(30)) + .await + .expect("claim sparse backlog"); + assert_eq!(claims.len(), 3); + for claim in &claims { + age_receipt_claim( + &pool, + &store, + claim.job.id, + claim.job.run_lease, + Duration::from_secs(300), + ) + .await; + } + if deadline { + sqlx::query(&format!( + "UPDATE {schema}.lease_claims SET deadline_at = now() - interval '1 second'" + )) + .execute(&pool) + .await + .expect("expire deadlines"); + } + if fresh_history { + for claim in &claims[..2] { + sqlx::query(&format!("INSERT INTO {schema}.attempt_state (job_id, run_lease, heartbeat_at, updated_at) VALUES ($1, $2, now(), now())")) + .bind(claim.job.id).bind(claim.job.run_lease).execute(&pool).await.expect("fresh heartbeat history"); + } + } else { + store + .complete_runtime_batch(&pool, &claims[..2]) + .await + .expect("complete history"); + } + let options = awa::model::RescueBatchOptions { + cursor_scan_limit: Some(1), + ..Default::default() + }; + let batch = if deadline { + store + .rescue_expired_deadlines_batch(&pool, &|_| false, &options) + .await + } else { + store + .rescue_stale_heartbeats_batch(&pool, Duration::from_secs(90), &|_| false, &options) + .await + } + .expect("walk first closed window"); + assert!(batch.rescued.is_empty()); + assert!( + batch.saturated, + "a full closed scan window must continue toward unseen abandoned claims ({suffix})" + ); + assert!( + !batch.idle_slots.contains(&claims[0].claim.claim_slot), + "cursor progress is not an idle slot" + ); + let mut rescued = Vec::new(); + for _ in 0..3 { + let batch = if deadline { + store + .rescue_expired_deadlines_batch(&pool, &|_| false, &options) + .await + } else { + store + .rescue_stale_heartbeats_batch( + &pool, + Duration::from_secs(90), + &|_| false, + &options, + ) + .await + } + .expect("continue sparse scan"); + rescued.extend(batch.rescued); + } + assert_eq!( + rescued.iter().map(|job| job.id).collect::>(), + vec![claims[2].job.id] + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_queue_storage_receipt_rescue_cursor_sweeps_past_fresh_claims_and_wraps() { let (_db_guard, pool) = setup_pool(10).await; diff --git a/docs/configuration.md b/docs/configuration.md index c7d7950da..ca588bc53 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -477,18 +477,18 @@ If you suspend a handler with `wait_for_callback()` and the external system neve A runtime that is killed (OOM, `SIGKILL`, a lost node) leaves every attempt it had claimed open. On queue storage those open claims also stop the claim ring from rotating back into their slot, so a large backlog has to be rescued quickly, not 500 attempts per rescue interval. -Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pass of batches. A batch rescues up to 500 attempts per source (lease rows and receipt claims) in its own short transaction. The pass keeps starting batches while the previous one came back full, until its time budget is spent or it has rescued `rescue_pass_max_rows`. While a backlog remains, the next pass of that kind runs one second later instead of after its interval, so a drain proceeds at up to about `rescue_pass_max_rows` attempts per second per rescue kind. +Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pass of batches. A batch scans a bounded claim window and rescues up to 500 attempts per source (lease rows and receipt claims) in its own short transaction. The pass keeps starting batches while a scan window or rescue batch came back full, until its time budget is spent or it has rescued `rescue_pass_max_rows`. While a backlog remains, the next pass of that kind runs one second later instead of after its interval, so a drain proceeds at up to about `rescue_pass_max_rows` attempts per second per rescue kind. | Knob | Default | What it does | | --- | --- | --- | | `ClientBuilder::rescue_pass_budget` (Rust) / `rescue_pass_budget_ms` (Python kwarg) | `1s` | How long one pass keeps starting batches. Capped at half the rescue interval. `Duration::ZERO` / `0` keeps every pass to one batch. | -| `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic) per pass and so per second. | +| `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic), with up to one final batch beyond the cap. `0` keeps one batch per ordinary interval without early re-arms. | How a pass chooses and paces its work: - **Oldest claims first.** Batches visit claim slots from the oldest to the current one, because the ring cannot rotate back into the oldest slot until its claims are closed. Every fourth batch visits the other slots first, so a long drain of one slot does not starve the rest. A slot a batch found nothing in is skipped for the rest of the pass. -- **Stale heartbeats need evidence that the owner is gone.** A stale heartbeat cannot tell a dead runtime from a stalled one (a database stall, a long GC pause, a network partition). A stale-heartbeat pass takes more than one batch only when the runtime snapshots in `awa.runtime_instances` show every runtime either fresh with its heartbeat loop alive, or silent for at least `heartbeat_staleness`, and that has held at every pass for a whole `heartbeat_staleness` window. A live runtime coming out of a stall has had that window to refresh its heartbeats, so its claims are not stale by the time a drain can reach them. The extra batches also take only claims last heartbeated before the most recent of those gone runtimes stopped reporting, so a live runtime whose heartbeat pass merely lags behind `heartbeat_staleness` (one holding many claims under load, say) is not drained in bulk; its claims stay at one batch per pass, as before. The leader samples the snapshots once per pass, so this evidence needs `heartbeat_rescue_interval` below half of `heartbeat_staleness` (the defaults, `30s` and `90s`, qualify); otherwise stale-heartbeat rescue stays at one batch per pass. The per-attempt staleness rule itself is unchanged. Deadline and callback-timeout rescues act on a deadline that has passed, so they drain without this evidence. -- **Retries are spread out.** A rescued attempt that goes back to `retryable` keeps its backoff, and rescued retries across all kinds become due no faster than `rescue_pass_max_rows` per second, so promotion moves a large drain into the ready lanes over time instead of all at once. +- **Stale heartbeats need evidence that the owner is gone.** A stale heartbeat cannot tell a dead runtime from a stalled one (a database stall, a long GC pause, a network partition). Stale-heartbeat rescue runs extra batches or early passes only when the runtime snapshots in `awa.runtime_instances` show every runtime either fresh with its heartbeat loop alive, or silent for at least `heartbeat_staleness`, and that has held at every pass for a whole `heartbeat_staleness` window. A live runtime coming out of a stall has had that window to refresh its heartbeats, so its claims are not stale by the time a drain can reach them. Every accelerated batch, including an early pass's first batch, also takes only claims last heartbeated before the most recent of those gone runtimes stopped reporting, so a live runtime whose heartbeat pass merely lags behind `heartbeat_staleness` (one holding many claims under load, say) is not drained in bulk; its claims stay at one batch per ordinary rescue interval, as before. The leader samples the snapshots once per pass, so this evidence needs `heartbeat_rescue_interval` below half of `heartbeat_staleness` (the defaults, `30s` and `90s`, qualify); otherwise stale-heartbeat rescue stays at one batch per pass. After a runtime stops reporting, the default `90s` staleness and `30s` rescue interval usually require about 3–3.5 minutes before acceleration: the runtime must first qualify as gone, then evidence must remain settled for a full staleness window. A new maintenance leader starts that evidence window again. The per-attempt staleness rule itself is unchanged. Deadline and callback-timeout rescues act on a deadline that has passed, so they drain without this evidence. +- **Retries retain their backoff.** Rescue bounds closures and retry writes per pass. Backoffs and promotion delays can make retries become due together; `rescue_pass_max_rows` does not cap the rate at which promotion admits them to the ready lanes. - **Exhausted attempts are dead-lettered in the same batches.** On a DLQ-enabled queue an attempt rescued on its last try goes to the DLQ in its batch's transaction (reason `heartbeat_stale`, `deadline_exceeded` or `callback_timeout`), so a drain can move many jobs to the DLQ at once. `awa.job.dlq_moved` counts them by reason. A pass holds the maintenance leader's loop for up to its budget plus one batch, so promotion and the other maintenance branches can wait about that long while a backlog drains. Watch a drain with `awa.maintenance.rescue.pass_size` (attempts per pass, by `awa.rescue.kind` and `awa.rescue.pass_end`: `drained`, `time_budget`, `row_limit`, `owners_unconfirmed` or `error`) and `awa.maintenance.rescue.oldest_attempt_age` (the age of the oldest attempt the latest pass rescued, `0` when it rescued nothing). An `oldest_attempt_age` far above `heartbeat_staleness` or the queue's deadline means rescue is behind. diff --git a/docs/grafana/README.md b/docs/grafana/README.md index 5a7d1dce0..712a1dea9 100644 --- a/docs/grafana/README.md +++ b/docs/grafana/README.md @@ -158,7 +158,7 @@ All metrics use the `awa` OTel meter name and are exported via OTLP to your conf | `awa.maintenance.promote_duration` | Histogram (s) | state | Promotion time | | `awa.maintenance.rescues` | Counter | rescue_kind | Jobs rescued | | `awa.maintenance.rescue.pass_size` | Histogram | rescue_kind, rescue_pass_end | Attempts rescued per queue-storage rescue pass, and why the pass stopped (`drained`, `time_budget`, `row_limit`, `owners_unconfirmed`, `error`) | -| `awa.maintenance.rescue.oldest_attempt_age` | Gauge (s) | rescue_kind | Age of the oldest attempt the latest queue-storage rescue pass rescued; `0` when it rescued none | +| `awa.maintenance.rescue.oldest_attempt_age` | Gauge (s) | rescue_kind | Age of the oldest attempt the latest queue-storage rescue pass on this process rescued; `0` when it rescued none. A former leader retains its last value until it runs another pass; filter to the current leader | ### Heartbeat diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7883706f0..cc1282833 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -296,7 +296,7 @@ Symptoms on queue storage, usually after a high-throughput runtime was OOM-kille The dead runtime's claims stay open until rescue closes them, and the ring cannot rotate back into a slot that still holds open claims. Rescue drains such a backlog in passes of 500-attempt batches ([Draining a backlog of abandoned claims](configuration.md#rescue-backlog)). To see where it is: - `awa.maintenance.rescue.oldest_attempt_age` by `awa.rescue.kind` shows how old the attempts rescue is closing are. It should fall towards `heartbeat_staleness` (or the queue's deadline) as the backlog drains. -- `awa.maintenance.rescue.pass_size` by `awa.rescue.pass_end` shows why passes stop. `time_budget` or `row_limit` means a drain is running at its configured rate. `owners_unconfirmed` means stale-heartbeat rescue is waiting for evidence that the claims' runtime is gone and is taking one batch per pass. +- `awa.maintenance.rescue.pass_size` by `awa.rescue.pass_end` shows why passes stop. `time_budget` or `row_limit` means a drain is running at its configured rate. `owners_unconfirmed` means stale-heartbeat rescue is waiting for evidence that the claims' runtime is gone and is taking one batch per ordinary rescue interval. If stale-heartbeat passes stay at `owners_unconfirmed`, look at the runtime snapshots: @@ -306,9 +306,9 @@ FROM awa.runtime_instances ORDER BY last_seen_at DESC; ``` -A runtime that has stopped reporting but has not yet been silent for `heartbeat_staleness`, or a reporting runtime with `heartbeat_alive = false`, holds the drain back until it is either gone or healthy. The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. +A runtime that has stopped reporting but has not yet been silent for `heartbeat_staleness`, or a reporting runtime with `heartbeat_alive = false`, holds the drain back until it is either gone or healthy. The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. With the defaults, acceleration usually starts about 3–3.5 minutes after the last runtime snapshot, and a leadership change restarts the evidence window. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. -To drain faster, raise `rescue_pass_max_rows` and `rescue_pass_budget`, keeping in mind that a pass holds the maintenance leader for its budget and writes up to `rescue_pass_max_rows` closures and retry rows per pass. +To drain faster, raise `rescue_pass_max_rows` and `rescue_pass_budget`, keeping in mind that a pass holds the maintenance leader for its budget and can exceed `rescue_pass_max_rows` by one final batch of closures and retry rows. Retry due times can cluster; this cap does not bound promotion throughput. ## Dead Tuples Growing In Queue Storage From 9361839829ce9528ee52cfce46bcfc76ab3089f1 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 21:52:37 +1300 Subject: [PATCH 06/22] perf(queue-storage): complete metadata jobs by compact claim identity Address compact claims by their batch ID and item index, validating the job, attempt and receipt before closing. Avoid expanding every retained compact batch for each metadata or retry completion flush, and avoid locking immutable shared batch rows. Preserve per-attempt advisory serialization, row-local prelocking and the fresh deciding snapshot. A 50,000-claim recovery rehearsal with heartbeat protection timed out waiting 120 seconds for retries on PostgreSQL 17 and 18 before this change. With keyed completion, the same full rehearsal passes in 116.35s and 123.30s total. A new regression rejects a wrong item index, completes while the immutable batch row is locked elsewhere, and checks retained tags and metadata. Validation: cargo fmt --all; SQLX_OFFLINE=true cargo clippy --all-targets --all-features -- -D warnings; SQLX_OFFLINE=true cargo build --workspace; eight focused metadata, stale-completion, cancellation, idempotence and finalization-bound tests plus the shutdown-waits test on PostgreSQL 17 and 18. --- CHANGELOG.md | 11 +++- awa-model/src/queue_storage.rs | 44 ++++++++++----- awa/tests/queue_storage_runtime_test.rs | 74 +++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9527d89a4..9a3dd1d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- Queue-storage completion of retried jobs or jobs carrying metadata now + addresses compact claims by their batch identity and item index, as on main. + It no longer expands every retained claim batch for each completion flush, + so bounded finalization can drain large recovery backlogs without that scan. + - **A completion backlog no longer gets finished jobs rescued and run again.** An attempt stops heartbeating when its handler returns, and its finalize then waits for the completion batcher. The runtime released the @@ -13,9 +18,9 @@ Notable changes between releases. Detailed migration notes for storage transitio slower than claiming (for example, retried jobs, whose completions take the slower per-row path) the queue of finished, unheartbeated attempts grew without bound. Attempts that waited longer than `heartbeat_staleness` were - rescued and executed again, some of them twice. At most `2 × batch size × - completion shards` attempts (1,024 by default), and at least one per - worker, now wait for finalization. Past that, a finished attempt keeps its + rescued and executed again, some of them twice. The bound on attempts waiting + for finalization is `max(worker count, 2 × batch size × completion shards)`. + Past that, a finished attempt keeps its worker slot and its heartbeat until there is room, so the dispatcher stops claiming until finalization catches up. - **`admin::job_kind_overviews` on queue storage no longer depends on the diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 930aab715..3190ebf6e 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -6534,14 +6534,22 @@ impl QueueStorage { .expect("receipt-backed slow completion requires receipt_id") }) .collect(); + let receipt_claim_batch_ids: Vec> = receipt_claimed + .iter() + .map(|entry| entry.claim.claim_batch_id) + .collect(); + let receipt_claim_batch_indices: Vec> = receipt_claimed + .iter() + .map(|entry| entry.claim.claim_batch_index) + .collect(); let closure_rel = format!("{schema}.lease_claim_closures"); let closure_batch_rel = format!("{schema}.lease_claim_closure_batches"); let closed_evidence = receipt_closed_evidence_sql(schema, &closure_rel, &closure_batch_rel, "claims"); let updated: Vec<(i64, i64)> = sqlx::query_as(&format!( r#" - WITH completed(claim_slot, job_id, run_lease, receipt_id) AS ( - SELECT * FROM unnest($1::int[], $2::bigint[], $3::bigint[], $4::bigint[]) + WITH completed(claim_slot, job_id, run_lease, receipt_id, claim_batch_id, claim_batch_index) AS ( + SELECT * FROM unnest($1::int[], $2::bigint[], $3::bigint[], $4::bigint[], $5::bigint[], $6::int[]) ), locked_row_claims AS ( SELECT claims.claim_slot, claims.job_id, claims.run_lease @@ -6568,18 +6576,23 @@ impl QueueStorage { items.job_id, items.run_lease, items.receipt_id - FROM {schema}.lease_claim_batches AS claim_batches - CROSS JOIN LATERAL unnest( - claim_batches.job_ids, - claim_batches.run_leases, - claim_batches.receipt_ids - ) AS items(job_id, run_lease, receipt_id) - JOIN completed - ON completed.claim_slot = claim_batches.claim_slot - AND completed.job_id = items.job_id - AND completed.run_lease = items.run_lease - AND completed.receipt_id = items.receipt_id - WHERE NOT EXISTS ( + FROM completed + JOIN {schema}.lease_claim_batches AS claim_batches + ON claim_batches.claim_slot = completed.claim_slot + AND claim_batches.batch_id = completed.claim_batch_id + CROSS JOIN LATERAL ( + SELECT + claim_batches.job_ids[completed.claim_batch_index] AS job_id, + claim_batches.run_leases[completed.claim_batch_index] AS run_lease, + claim_batches.receipt_ids[completed.claim_batch_index] AS receipt_id + ) AS items + WHERE completed.claim_batch_id IS NOT NULL + AND completed.claim_batch_index IS NOT NULL + AND completed.claim_batch_index BETWEEN 1 AND claim_batches.claimed_count + AND completed.job_id = items.job_id + AND completed.run_lease = items.run_lease + AND completed.receipt_id = items.receipt_id + AND NOT EXISTS ( SELECT 1 FROM {schema}.lease_claim_closures AS closures WHERE closures.claim_slot = claim_batches.claim_slot @@ -6616,7 +6629,6 @@ impl QueueStorage { WHERE dlq.job_id = items.job_id AND dlq.run_lease = items.run_lease ) - FOR UPDATE OF claim_batches ), locked_claims AS ( SELECT claim_slot, job_id, run_lease FROM locked_row_claims @@ -6708,6 +6720,8 @@ impl QueueStorage { .bind(&receipt_job_ids) .bind(&receipt_run_leases) .bind(&receipt_receipt_ids) + .bind(&receipt_claim_batch_ids) + .bind(&receipt_claim_batch_indices) .fetch_all(tx.as_mut()) .await .map_err(map_sqlx_error)?; diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index 8174c7d66..4765116c1 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -15514,3 +15514,77 @@ async fn test_queue_storage_receipt_claims_are_locked_before_the_heartbeat_state .expect("join heartbeat") .expect("heartbeat after the rewrite"); } + +/// Metadata completions validate the exact compact item and do not lock the +/// immutable batch shared with other attempts. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_metadata_completion_validates_and_does_not_lock_compact_batch() { + let (_db_guard, pool) = setup_pool(10).await; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: "awa_qs_metadata_completion_identity".into(), + ..Default::default() + }, + ) + .await; + let queue = "metadata_completion_identity"; + for id in 0..2 { + enqueue_job( + &pool, + &store, + &CompleteJob { id }, + InsertOpts { + queue: queue.into(), + tags: vec!["retained".into()], + metadata: serde_json::json!({"source": "recovery"}), + ..Default::default() + }, + ) + .await; + } + let jobs = store + .claim_runtime_batch(&pool, queue, 2, Duration::ZERO) + .await + .expect("claim compact jobs with metadata"); + assert_eq!(jobs.len(), 2); + assert_eq!(jobs[0].claim.claim_batch_id, jobs[1].claim.claim_batch_id); + let mut invalid = jobs[0].clone(); + invalid.claim.claim_batch_index = jobs[1].claim.claim_batch_index; + assert!(store + .complete_runtime_batch(&pool, &[invalid]) + .await + .expect("reject a different batch item") + .is_empty()); + + let mut holder = pool.begin().await.expect("begin batch holder"); + sqlx::query(&format!( + "SELECT batch_id FROM {}.lease_claim_batches \ + WHERE claim_slot = $1 AND batch_id = $2 FOR UPDATE", + store.schema() + )) + .bind(jobs[0].claim.claim_slot) + .bind(jobs[0].claim.claim_batch_id.expect("compact batch")) + .execute(holder.as_mut()) + .await + .expect("hold the immutable batch row"); + let completed = tokio::time::timeout( + ci_timing::scaled_timeout(Duration::from_secs(2)), + store.complete_runtime_batch(&pool, &jobs), + ) + .await + .expect("completion must not wait on an unrelated batch-row lock") + .expect("complete metadata jobs"); + assert_eq!(completed.len(), 2); + holder.rollback().await.expect("release batch row"); + for job in jobs { + let durable = store + .load_job(&pool, job.job.id) + .await + .expect("read completed job") + .expect("completed job"); + assert_eq!(durable.state, JobState::Completed); + assert_eq!(durable.tags, vec!["retained".to_string()]); + assert_eq!(durable.metadata, serde_json::json!({"source": "recovery"})); + } +} From e01f87f1f412cdce3c88993d72ea36d986174387 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 21:55:29 +1300 Subject: [PATCH 07/22] Keep finalizing attempts live through completion acknowledgement --- CHANGELOG.md | 6 + awa-model/src/queue_storage.rs | 246 ++++++++++++++- awa-worker/src/executor.rs | 40 ++- awa-worker/src/heartbeat.rs | 50 +++- awa-worker/src/runtime.rs | 35 ++- awa/tests/heartbeat_pass_benchmark_test.rs | 187 ++++++++++++ awa/tests/queue_storage_runtime_test.rs | 329 +++++++++++++++++++++ 7 files changed, 869 insertions(+), 24 deletions(-) create mode 100644 awa/tests/heartbeat_pass_benchmark_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a3dd1d7d..13a08de78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ Notable changes between releases. Detailed migration notes for storage transitio It no longer expands every retained claim batch for each completion flush, so bounded finalization can drain large recovery backlogs without that scan. +- Finished attempts keep heartbeating until their durable finalization finishes, + including while completion writes are stalled. Compact receipt heartbeats + address claims by batch identity and skip attempts a closer already holds, + so one stalled completion cannot block the rest of a heartbeat pass. Late + heartbeats cannot recreate state for a completed attempt. + - **A completion backlog no longer gets finished jobs rescued and run again.** An attempt stops heartbeating when its handler returns, and its finalize then waits for the completion batcher. The runtime released the diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 3190ebf6e..306709025 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -154,6 +154,39 @@ pub struct ClaimedRuntimeJob { pub unique_states: Option, } +/// Identity of a compact receipt claim for heartbeat purposes: addresses +/// `lease_claim_batches` by primary key and the attempt by item index, so a +/// heartbeat does not have to scan every open claim batch. Built with +/// [`ClaimHeartbeatRef::from_claim`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ClaimHeartbeatRef { + pub job_id: i64, + pub run_lease: i64, + pub claim_slot: i32, + pub claim_batch_id: i64, + pub claim_batch_index: i32, + pub receipt_id: i64, +} + +impl ClaimHeartbeatRef { + /// The heartbeat identity of a compact receipt claim, or `None` for + /// legacy or row-local claims. + pub fn from_claim(job_id: i64, run_lease: i64, claim: &ClaimedEntry) -> Option { + if !claim.lease_claim_receipt { + return None; + } + Some(Self { + job_id, + run_lease, + claim_slot: claim.claim_slot, + claim_batch_id: claim.claim_batch_id?, + claim_batch_index: claim.claim_batch_index?, + receipt_id: claim.receipt_id?, + }) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, sqlx::FromRow)] pub struct QueueClaimerLease { pub claimer_slot: i16, @@ -9820,6 +9853,47 @@ impl QueueStorage { Ok(()) } + /// Take the closers' attempt locks without waiting, returning the attempts + /// locked. A heartbeat must not queue behind a closer: an attempt whose lock + /// is held is being finalized or rescued right now, and rescue skips it for + /// as long as the lock is held. Taking the locks in a statement of their own + /// means the statement that then decides what is still open starts after + /// every closer that held a lock first has committed (#614), so a heartbeat + /// cannot recreate state for an attempt that has just been closed. + async fn try_lock_receipt_attempt_keys_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + jobs: &[(i64, i64)], + ) -> Result, AwaError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let job_ids: Vec = jobs.iter().map(|(job_id, _)| *job_id).collect(); + let run_leases: Vec = jobs.iter().map(|(_, run_lease)| *run_lease).collect(); + sqlx::query_as::<_, (i64, i64)>( + r#" + WITH attempts AS MATERIALIZED ( + SELECT DISTINCT + input.job_id, + input.run_lease, + pg_catalog.hashtextextended( + format('awa.receipt.complete:%s:%s', input.job_id, input.run_lease), + 0 + ) AS lock_key + FROM unnest($1::bigint[], $2::bigint[]) AS input(job_id, run_lease) + ) + SELECT job_id, run_lease + FROM attempts + WHERE pg_catalog.pg_try_advisory_xact_lock(lock_key) + "#, + ) + .bind(&job_ids) + .bind(&run_leases) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error) + } + async fn ensure_running_leases_from_receipts_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, @@ -10114,7 +10188,6 @@ impl QueueStorage { WHERE dlq.job_id = items.job_id AND dlq.run_lease = items.run_lease ) - FOR UPDATE OF claim_batches ), claim_refs AS ( SELECT * FROM row_claim_refs @@ -10163,6 +10236,19 @@ impl QueueStorage { .iter() .map(|(job_id, run_lease, _)| (*job_id, *run_lease)) .collect(); + let locked: HashSet<_> = self + .try_lock_receipt_attempt_keys_tx(tx, &receipt_jobs) + .await? + .into_iter() + .collect(); + let jobs: Vec<_> = jobs + .iter() + .filter(|(job_id, run_lease, _)| locked.contains(&(*job_id, *run_lease))) + .collect(); + if jobs.is_empty() { + return Ok(0); + } + let receipt_jobs: Vec<_> = jobs.iter().map(|(id, lease, _)| (*id, *lease)).collect(); self.lock_receipt_attempts_tx(tx, &receipt_jobs).await?; let schema = self.schema(); @@ -10231,7 +10317,6 @@ impl QueueStorage { WHERE dlq.job_id = items.job_id AND dlq.run_lease = items.run_lease ) - FOR UPDATE OF claim_batches ), claim_refs AS ( SELECT * FROM row_claim_refs @@ -13037,6 +13122,141 @@ impl QueueStorage { Ok(()) } + /// Heartbeat compact receipt claims by their batch identity. Each claim + /// batch is addressed by primary key and the attempt by item index, with + /// index-only closure probes and no batch row lock, so the cost is + /// O(in-flight attempts) rather than O(open claim batches). Attempts that + /// a closer holds right now are skipped rather than waited for. + pub async fn heartbeat_claims_batch( + &self, + pool: &PgPool, + claims: &[ClaimHeartbeatRef], + ) -> Result { + if claims.is_empty() { + return Ok(0); + } + let schema = self.schema(); + let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let pairs: Vec<(i64, i64)> = claims + .iter() + .map(|claim| (claim.job_id, claim.run_lease)) + .collect(); + let locked: HashSet<(i64, i64)> = self + .try_lock_receipt_attempt_keys_tx(&mut tx, &pairs) + .await? + .into_iter() + .collect(); + let claims: Vec<&ClaimHeartbeatRef> = claims + .iter() + .filter(|claim| locked.contains(&(claim.job_id, claim.run_lease))) + .collect(); + if claims.is_empty() { + tx.commit().await.map_err(map_sqlx_error)?; + return Ok(0); + } + let job_ids: Vec = claims.iter().map(|claim| claim.job_id).collect(); + let run_leases: Vec = claims.iter().map(|claim| claim.run_lease).collect(); + let claim_slots: Vec = claims.iter().map(|claim| claim.claim_slot).collect(); + let batch_ids: Vec = claims.iter().map(|claim| claim.claim_batch_id).collect(); + let batch_indices: Vec = claims.iter().map(|claim| claim.claim_batch_index).collect(); + let receipt_ids: Vec = claims.iter().map(|claim| claim.receipt_id).collect(); + let updated: i64 = sqlx::query_scalar(&format!( + r#" + WITH inflight(job_id, run_lease, claim_slot, batch_id, batch_index, receipt_id) AS ( + SELECT * FROM unnest($1::bigint[], $2::bigint[], $3::int[], $4::bigint[], $5::int[], $6::bigint[]) + ), + open_claims AS ( + SELECT inflight.job_id, inflight.run_lease + FROM inflight + JOIN {schema}.lease_claim_batches AS claim_batches + ON claim_batches.claim_slot = inflight.claim_slot + AND claim_batches.batch_id = inflight.batch_id + WHERE inflight.batch_index BETWEEN 1 AND claim_batches.claimed_count + AND claim_batches.job_ids[inflight.batch_index] = inflight.job_id + AND claim_batches.run_leases[inflight.batch_index] = inflight.run_lease + AND claim_batches.receipt_ids[inflight.batch_index] = inflight.receipt_id + AND NOT EXISTS ( + SELECT 1 FROM {schema}.lease_claim_closures AS closures + WHERE closures.claim_slot = inflight.claim_slot + AND closures.job_id = inflight.job_id + AND closures.run_lease = inflight.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM {schema}.lease_claim_closure_batches AS closure_batches + WHERE closure_batches.claim_slot = inflight.claim_slot + AND closure_batches.receipt_ranges @> inflight.receipt_id + ) + AND NOT EXISTS ( + SELECT 1 FROM {schema}.done_entries AS done + WHERE done.job_id = inflight.job_id AND done.run_lease = inflight.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM {schema}.deferred_jobs AS deferred + WHERE deferred.job_id = inflight.job_id AND deferred.run_lease = inflight.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM {schema}.dlq_entries AS dlq + WHERE dlq.job_id = inflight.job_id AND dlq.run_lease = inflight.run_lease + ) + ), + upserted AS ( + INSERT INTO {schema}.attempt_state (job_id, run_lease, heartbeat_at, updated_at) + SELECT open_claims.job_id, open_claims.run_lease, clock_timestamp(), clock_timestamp() + FROM open_claims + ON CONFLICT (job_id, run_lease) + DO UPDATE SET + heartbeat_at = clock_timestamp(), + updated_at = clock_timestamp() + RETURNING job_id + ) + SELECT count(*)::bigint FROM upserted + "# + )) + .bind(&job_ids) + .bind(&run_leases) + .bind(&claim_slots) + .bind(&batch_ids) + .bind(&batch_indices) + .bind(&receipt_ids) + .fetch_one(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + tx.commit().await.map_err(map_sqlx_error)?; + Ok(updated as usize) + } + + /// Lock legacy lease rows without waiting behind a finalizer. Keep this + /// separate from the updating statement: the latter must use a fresh + /// snapshot and must not mix partition pruning with row locking (PG18). + async fn lock_live_leases_for_heartbeat_tx( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + jobs: &[(i64, i64)], + ) -> Result, AwaError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let job_ids: Vec = jobs.iter().map(|(job_id, _)| *job_id).collect(); + let run_leases: Vec = jobs.iter().map(|(_, run_lease)| *run_lease).collect(); + sqlx::query_as::<_, (i64, i64)>(&format!( + r#" + SELECT lease.job_id, lease.run_lease + FROM {table} AS lease + JOIN unnest($1::bigint[], $2::bigint[]) AS input(job_id, run_lease) + ON lease.job_id = input.job_id AND lease.run_lease = input.run_lease + WHERE lease.state = 'running' + ORDER BY lease.job_id, lease.run_lease + FOR UPDATE OF lease SKIP LOCKED + "#, + table = self.leases_table(), + )) + .bind(&job_ids) + .bind(&run_leases) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error) + } + pub async fn heartbeat_batch( &self, pool: &PgPool, @@ -13057,14 +13277,18 @@ impl QueueStorage { // dead entries per write under sustained churn). Compat // reads in `LeaseJobRow` SELECTs and the `awa.jobs` view // COALESCE attempt_state.heartbeat_at first. + let locked = self.try_lock_receipt_attempt_keys_tx(&mut tx, jobs).await?; updated += self - .upsert_attempt_state_from_receipts_tx(&mut tx, jobs) + .upsert_attempt_state_from_receipts_tx(&mut tx, &locked) .await?; } else { // Legacy non-receipts mode (custom schemas with // `lease_claim_receipts=FALSE`): the `leases.heartbeat_at` // write is still the heartbeat home, since there is no // upsert_attempt_state path firing. + let jobs = self + .lock_live_leases_for_heartbeat_tx(&mut tx, jobs) + .await?; let job_ids: Vec = jobs.iter().map(|(job_id, _)| *job_id).collect(); let run_leases: Vec = jobs.iter().map(|(_, run_lease)| *run_lease).collect(); let result = sqlx::query(&format!( @@ -13115,9 +13339,19 @@ impl QueueStorage { self.upsert_attempt_state_progress_from_receipts_tx(&mut tx, jobs) .await? } else { - // Legacy non-receipts mode keeps the old CTE shape that - // updates leases.heartbeat_at + leases.progress - // (via attempt_state upsert) in a single round-trip. + // Prelock independently so a finalizer cannot stall the entire + // heartbeat pass. Skipped progress snapshots remain unacknowledged + // in the worker and are retried on the next pass. + let keys: Vec<_> = jobs.iter().map(|(id, lease, _)| (*id, *lease)).collect(); + let locked: HashSet<_> = self + .lock_live_leases_for_heartbeat_tx(&mut tx, &keys) + .await? + .into_iter() + .collect(); + let jobs: Vec<_> = jobs + .iter() + .filter(|(id, lease, _)| locked.contains(&(*id, *lease))) + .collect(); let schema = self.schema(); let job_ids: Vec = jobs.iter().map(|(job_id, _, _)| *job_id).collect(); let run_leases: Vec = jobs.iter().map(|(_, run_lease, _)| *run_lease).collect(); diff --git a/awa-worker/src/executor.rs b/awa-worker/src/executor.rs index 99b8c9c18..a392b3a31 100644 --- a/awa-worker/src/executor.rs +++ b/awa-worker/src/executor.rs @@ -365,6 +365,14 @@ impl JobExecutor { let in_flight_state = InFlightState { cancel: cancel.clone(), progress: progress_state.clone(), + claim: queue_storage_claim.as_ref().and_then(|claim| { + awa_model::queue_storage::ClaimHeartbeatRef::from_claim( + job_id, + job_run_lease, + claim, + ) + }), + finalizing: false, }; in_flight.insert((job_id, job_run_lease), in_flight_state); if let Some(counter) = queue_in_flight.get(&job_queue) { @@ -443,12 +451,17 @@ impl JobExecutor { guard.clone_latest() }; - // Leaving `in_flight` ends the attempt's heartbeat, so only a - // bounded number of finished attempts may wait for finalization. - // While that queue is full the attempt keeps its worker slot and - // its heartbeat, and the dispatcher stops claiming: a completion - // backlog cannot outgrow `heartbeat_staleness` and get finished - // attempts rescued and run again. + // The attempt stays in `in_flight`, and so keeps its heartbeat, + // until its finalize is over: a finalize that waits behind a + // completion backlog must not look like a dead worker to rescue. + // Only a bounded number of finished attempts may wait for + // finalization; while that queue is full the attempt also keeps + // its worker slot, so the dispatcher stops claiming. + in_flight.mark_finalizing((job_id, job_run_lease)); + let heartbeat = HeartbeatUntilFinalized { + in_flight: in_flight.clone(), + key: (job_id, job_run_lease), + }; let finalize_permit = match result { Some(_) => tokio::select! { permit = finalize_capacity.acquire_owned() => { @@ -458,7 +471,6 @@ impl JobExecutor { }, None => None, }; - in_flight.remove((job_id, job_run_lease)); if let Some(counter) = queue_in_flight.get(&job_queue) { counter.fetch_sub(1, Ordering::SeqCst); } @@ -511,6 +523,7 @@ impl JobExecutor { outcome => break outcome, } }; + drop(heartbeat); drop(finalize_permit); match &outcome { @@ -594,6 +607,19 @@ impl JobExecutor { } } +/// Keeps a finished attempt in the heartbeat set until its finalize is over, +/// and takes it out however the finalizer ends. +struct HeartbeatUntilFinalized { + in_flight: InFlightMap, + key: (i64, i64), +} + +impl Drop for HeartbeatUntilFinalized { + fn drop(&mut self) { + self.in_flight.remove(self.key); + } +} + /// How many times a finalize transaction is re-run after Postgres aborts it /// to break a lock cycle before the failure is surfaced. const COMPLETION_DEADLOCK_RETRIES: u32 = 3; diff --git a/awa-worker/src/heartbeat.rs b/awa-worker/src/heartbeat.rs index 462b33eb7..655e02729 100644 --- a/awa-worker/src/heartbeat.rs +++ b/awa-worker/src/heartbeat.rs @@ -1,5 +1,6 @@ use crate::runtime::{InFlightMap, RunLease}; use crate::storage::{QueueStorageRuntime, RuntimeStorage}; +use awa_model::queue_storage::ClaimHeartbeatRef; use sqlx::PgPool; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -76,8 +77,9 @@ impl HeartbeatService { fields(interval_ms = self.interval.as_millis() as u64) )] async fn heartbeat_once(&self) { - let mut all_keys: Vec<(i64, RunLease)> = self.in_flight.keys(); - all_keys.sort_unstable(); + let mut in_flight = self.in_flight.claim_refs(); + in_flight.sort_unstable_by_key(|(key, _)| *key); + let all_keys: Vec<(i64, RunLease)> = in_flight.iter().map(|(key, _)| *key).collect(); if all_keys.is_empty() { return; @@ -104,7 +106,14 @@ impl HeartbeatService { .await; } RuntimeStorage::QueueStorage(runtime) => { - self.heartbeat_once_queue_storage(runtime, &heartbeat_only, &pending_progress) + let (claimed, plain): (Vec<_>, Vec<_>) = in_flight + .iter() + .filter(|(key, _)| !progress_keys.contains(key)) + .partition(|(_, claim)| claim.is_some()); + let claimed: Vec = + claimed.into_iter().filter_map(|(_, claim)| claim).collect(); + let plain: Vec<(i64, RunLease)> = plain.into_iter().map(|(key, _)| key).collect(); + self.heartbeat_once_queue_storage(runtime, &claimed, &plain, &pending_progress) .await; } } @@ -131,7 +140,7 @@ impl HeartbeatService { AND jobs.run_lease = inflight.run_lease WHERE jobs.state = 'running' ORDER BY jobs.id - FOR UPDATE OF jobs + FOR UPDATE OF jobs SKIP LOCKED ) UPDATE awa.jobs_hot AS jobs SET heartbeat_at = now() @@ -178,7 +187,7 @@ impl HeartbeatService { AND jobs.run_lease = inflight.run_lease WHERE jobs.state = 'running' ORDER BY jobs.id - FOR UPDATE OF jobs + FOR UPDATE OF jobs SKIP LOCKED ) UPDATE awa.jobs_hot AS jobs SET heartbeat_at = now(), @@ -204,7 +213,13 @@ impl HeartbeatService { .iter() .map(|(id, lease, gen, _)| (*id, *lease, *gen)) .collect(); - self.in_flight.ack_progress(&acked); + if result.rows_affected() as usize == chunk.len() { + self.in_flight.ack_progress(&acked); + } else { + // Re-send successful snapshots too: replay is safe, + // whereas acknowledging a skipped write loses progress. + self.in_flight.clear_in_flight_progress(&acked); + } } Err(err) => { warn!(error = %err, "Failed to send heartbeat+progress batch"); @@ -222,9 +237,26 @@ impl HeartbeatService { async fn heartbeat_once_queue_storage( &self, runtime: &QueueStorageRuntime, + claimed: &[ClaimHeartbeatRef], heartbeat_only: &[(i64, RunLease)], pending_progress: &[(i64, i64, u64, serde_json::Value)], ) { + for chunk in claimed.chunks(self.batch_size) { + match runtime + .store + .heartbeat_claims_batch(&self.pool, chunk) + .await + { + Ok(updated) => { + self.metrics.heartbeat_batches.add(1, &[]); + debug!(count = chunk.len(), updated, "Heartbeat claim batch sent"); + } + Err(err) => { + warn!(error = %err, "Failed to send heartbeat claim batch"); + } + } + } + for chunk in heartbeat_only.chunks(self.batch_size) { match runtime.store.heartbeat_batch(&self.pool, chunk).await { Ok(updated) => { @@ -258,7 +290,11 @@ impl HeartbeatService { .iter() .map(|(id, lease, gen, _)| (*id, *lease, *gen)) .collect(); - self.in_flight.ack_progress(&acked); + if updated == chunk.len() { + self.in_flight.ack_progress(&acked); + } else { + self.in_flight.clear_in_flight_progress(&acked); + } } Err(err) => { warn!(error = %err, "Failed to send heartbeat+progress batch"); diff --git a/awa-worker/src/runtime.rs b/awa-worker/src/runtime.rs index 4de28684a..5a87d4428 100644 --- a/awa-worker/src/runtime.rs +++ b/awa-worker/src/runtime.rs @@ -5,6 +5,8 @@ use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::sync::RwLock; +use awa_model::queue_storage::ClaimHeartbeatRef; + const IN_FLIGHT_SHARDS: usize = 64; pub(crate) type RunLease = i64; @@ -17,6 +19,13 @@ type InFlightShard = RwLock>; pub(crate) struct InFlightState { pub cancel: CancelFlag, pub progress: Arc>, + /// Compact receipt claim identity, when the attempt was dispatched from + /// queue storage with one; lets the heartbeat address the claim batch + /// directly. + pub claim: Option, + /// The handler has returned and the attempt waits for its finalize. It is + /// still heartbeated, but its progress is settled by the finalize. + pub finalizing: bool, } /// Mutable progress buffer shared between handler and heartbeat service. @@ -170,13 +179,26 @@ impl InFlightRegistry { guard.get(&key).map(|s| s.cancel.clone()) } - pub fn keys(&self) -> Vec { - let mut keys = Vec::new(); + /// Every attempt to heartbeat, with its compact claim identity, if any. + pub fn claim_refs(&self) -> Vec<(InFlightKey, Option)> { + let mut refs = Vec::new(); for shard in &*self.shards { let guard = shard.read().expect("in_flight shard poisoned"); - keys.extend(guard.keys().copied()); + refs.extend(guard.iter().map(|(key, state)| (*key, state.claim))); + } + refs + } + + /// The handler has returned: keep heartbeating the attempt until its + /// finalize is done, and stop flushing its progress. + pub fn mark_finalizing(&self, key: InFlightKey) { + let mut guard = self + .shard_for(&key) + .write() + .expect("in_flight shard poisoned"); + if let Some(state) = guard.get_mut(&key) { + state.finalizing = true; } - keys } pub fn flags(&self) -> Vec { @@ -198,6 +220,9 @@ impl InFlightRegistry { for shard in &*self.shards { let guard = shard.read().expect("in_flight shard poisoned"); for (&(job_id, run_lease), state) in guard.iter() { + if state.finalizing { + continue; + } let mut progress = lock_progress(&state.progress); if progress.has_pending() && progress.in_flight.is_none() { if let Some(ref value) = progress.latest { @@ -283,6 +308,8 @@ mod tests { InFlightState { cancel: Arc::new(AtomicBool::new(false)), progress: progress.clone(), + claim: None, + finalizing: false, }, ); diff --git a/awa/tests/heartbeat_pass_benchmark_test.rs b/awa/tests/heartbeat_pass_benchmark_test.rs new file mode 100644 index 000000000..489f29677 --- /dev/null +++ b/awa/tests/heartbeat_pass_benchmark_test.rs @@ -0,0 +1,187 @@ +//! Heartbeat pass cost against in-flight claim count. Ignored by default: +//! +//! ```text +//! DATABASE_URL=postgres://postgres:test@localhost:15432/awa_test \ +//! cargo test --release -p awa --test heartbeat_pass_benchmark_test -- --ignored --nocapture +//! ``` +//! +//! Each size claims that many jobs through the claim path the dispatcher uses, +//! in claims of `AWA_HEARTBEAT_BENCH_CLAIM_BATCH` jobs (default 16, as a busy +//! runtime refills a few freed workers at a time), and times a heartbeat pass +//! over them in the heartbeat service's 500-attempt chunks. +//! `AWA_HEARTBEAT_BENCH_SIZES` overrides the sizes. + +use awa::model::queue_storage::ClaimHeartbeatRef; +use awa::model::{insert, migrations, QueueStorage, QueueStorageConfig}; +use awa::{InsertOpts, JobArgs}; +use serde::{Deserialize, Serialize}; +use sqlx::postgres::{PgConnection, PgPoolOptions}; +use sqlx::{Connection, PgPool}; +use std::time::{Duration, Instant}; + +const HEARTBEAT_CHUNK: usize = 500; +const PASSES: usize = 6; + +#[derive(Debug, Serialize, Deserialize, JobArgs)] +struct HeartbeatBenchJob { + n: i64, +} + +fn database_url() -> String { + std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://postgres:test@localhost:15432/awa_test".to_string()) +} + +fn with_database(url: &str, database: &str) -> String { + let (base, _) = url + .rsplit_once('/') + .expect("database URL has a database name"); + format!("{base}/{database}") +} + +async fn fresh_database(name: &str) -> PgPool { + let mut admin = PgConnection::connect(&with_database(&database_url(), "postgres")) + .await + .expect("admin connection"); + sqlx::raw_sql(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .execute(&mut admin) + .await + .expect("drop leftover database"); + sqlx::raw_sql(&format!("CREATE DATABASE {name}")) + .execute(&mut admin) + .await + .expect("create database"); + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&with_database(&database_url(), name)) + .await + .expect("bench pool"); + migrations::run(&pool).await.expect("migrate"); + pool +} + +async fn drop_database(name: &str) { + let mut admin = PgConnection::connect(&with_database(&database_url(), "postgres")) + .await + .expect("admin connection"); + let _ = sqlx::raw_sql(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)")) + .execute(&mut admin) + .await; +} + +#[tokio::test] +#[ignore = "benchmark; creates and drops its own database"] +async fn heartbeat_pass_time_by_in_flight_claims() { + let sizes: Vec = std::env::var("AWA_HEARTBEAT_BENCH_SIZES") + .ok() + .map(|sizes| { + sizes + .split(',') + .filter_map(|size| size.trim().parse().ok()) + .collect() + }) + .unwrap_or_else(|| vec![1_000, 10_000, 25_000]); + let claim_batch: i64 = std::env::var("AWA_HEARTBEAT_BENCH_CLAIM_BATCH") + .ok() + .and_then(|size| size.parse().ok()) + .unwrap_or(16); + let server: String = { + let pool = fresh_database(&format!("awa_hb_bench_{}", std::process::id())).await; + let version = sqlx::query_scalar("SHOW server_version") + .fetch_one(&pool) + .await + .expect("server version"); + pool.close().await; + version + }; + for size in sizes { + let name = format!("awa_hb_bench_{}", std::process::id()); + let pool = fresh_database(&name).await; + let store = QueueStorage::new(QueueStorageConfig::default()).expect("queue storage"); + store.install(&pool).await.expect("install queue storage"); + awa_testing::setup::activate_queue_storage(&pool).await; + let queue = "heartbeat_bench"; + for first in (0..size as i64).step_by(1_000) { + let params: Vec<_> = (first..(first + 1_000).min(size as i64)) + .map(|n| { + insert::params_with( + &HeartbeatBenchJob { n }, + InsertOpts { + queue: queue.to_string(), + ..Default::default() + }, + ) + .expect("params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue"); + } + let mut claimed = Vec::with_capacity(size); + while claimed.len() < size { + let batch = store + .claim_runtime_batch(&pool, queue, claim_batch, Duration::ZERO) + .await + .expect("claim"); + assert!(!batch.is_empty(), "claimed {} of {size}", claimed.len()); + claimed.extend(batch); + } + sqlx::query("ANALYZE") + .execute(&pool) + .await + .expect("analyze"); + + // The first pass creates every attempt's heartbeat row; later passes + // refresh them. + let mut passes = Vec::with_capacity(PASSES); + let mut refreshed = 0; + for _ in 0..PASSES { + let started = Instant::now(); + refreshed = heartbeat_pass(&store, &pool, &claimed).await; + passes.push(started.elapsed()); + } + let first = passes[0]; + let mut later = passes[1..].to_vec(); + later.sort_unstable(); + eprintln!( + "heartbeat pass: server={server} in_flight={size} claim_batch={claim_batch} \ + refreshed={refreshed} first_ms={} median_ms={} max_ms={}", + first.as_millis(), + later[later.len() / 2].as_millis(), + later[later.len() - 1].as_millis() + ); + assert_eq!(refreshed, size, "every in-flight attempt is refreshed"); + pool.close().await; + drop_database(&name).await; + } +} + +async fn heartbeat_pass( + store: &QueueStorage, + pool: &PgPool, + claimed: &[awa::model::ClaimedRuntimeJob], +) -> usize { + // As the heartbeat service splits them: compact receipt claims by their + // batch identity, anything else by attempt key. + let mut claims = Vec::new(); + let mut plain = Vec::new(); + for job in claimed { + match ClaimHeartbeatRef::from_claim(job.job.id, job.job.run_lease, &job.claim) { + Some(claim) => claims.push(claim), + None => plain.push((job.job.id, job.job.run_lease)), + } + } + let mut refreshed = 0; + for chunk in claims.chunks(HEARTBEAT_CHUNK) { + refreshed += store + .heartbeat_claims_batch(pool, chunk) + .await + .expect("heartbeat claims"); + } + for chunk in plain.chunks(HEARTBEAT_CHUNK) { + refreshed += store.heartbeat_batch(pool, chunk).await.expect("heartbeat"); + } + refreshed +} diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index 4765116c1..daa942d98 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -2156,6 +2156,132 @@ async fn test_queue_storage_runtime_stops_claiming_while_finalization_is_stalled ); } +/// A finished attempt keeps heartbeating until its finalize is over, so a +/// finalize that waits behind a stalled completion batcher for several +/// staleness windows is not mistaken for a dead worker and run again. +/// Holding the first job's receipt-completion lock stalls the batcher's flush; +/// the attempts that flush has locked are skipped by rescue for as long as it +/// waits, and every other finished attempt must still be heartbeated. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_finalizing_attempts_keep_heartbeating_through_a_stalled_finalize() { + const WORKERS: u32 = 64; + const JOBS: i64 = 1_600; + let (_db_guard, pool) = setup_pool(10).await; + let queue = "qs_finalize_heartbeat"; + let store_config = QueueStorageConfig { + schema: "awa_qs_finalize_heartbeat".to_string(), + ..Default::default() + }; + let store = create_store_with_config(&pool, store_config.clone()).await; + for first in (0..JOBS).step_by(500) { + let params: Vec<_> = (first..(first + 500).min(JOBS)) + .map(|n| { + insert::params_with( + &FinalizeBacklogJob { n }, + InsertOpts { + queue: queue.to_string(), + ..Default::default() + }, + ) + .expect("params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue backlog"); + } + + // Only the first job's lock: a held lock also keeps the heartbeat off that + // attempt, which is right only while a closer is working on it. + let first_job: i64 = sqlx::query_scalar(&format!( + "SELECT job_id FROM {}.ready_entries WHERE queue = $1 ORDER BY lane_seq LIMIT 1", + store.schema() + )) + .bind(queue) + .fetch_one(&pool) + .await + .expect("first job"); + let mut lock_holder = pool.acquire().await.expect("lock connection"); + sqlx::query( + "SELECT pg_catalog.pg_advisory_lock(pg_catalog.hashtextextended( + format('awa.receipt.complete:%s:%s', $1::bigint, 1), 0))", + ) + .bind(first_job) + .execute(&mut *lock_holder) + .await + .expect("stall the first job's finalization"); + + let staleness = ci_timing::scaled_staleness(Duration::from_secs(1)); + let runs: Arc>> = Arc::default(); + let recorded = runs.clone(); + let client = Client::builder(pool.clone()) + .queue( + queue, + QueueConfig { + max_workers: WORKERS, + poll_interval: Duration::from_millis(10), + deadline_duration: Duration::ZERO, + claim_batch_size: WORKERS as usize, + ..QueueConfig::default() + }, + ) + .queue_storage( + store_config, + Duration::from_secs(60), + Duration::from_millis(50), + ) + .claim_rotate_interval(Duration::from_secs(60)) + .register::(move |_, ctx| { + *recorded + .lock() + .expect("runs") + .entry(ctx.job.id) + .or_insert(0) += 1; + async { Ok(JobResult::Completed) } + }) + .heartbeat_interval(staleness / 5) + .heartbeat_staleness(staleness) + .heartbeat_rescue_interval(Duration::from_millis(100)) + .leader_election_interval(Duration::from_millis(100)) + .leader_check_interval(Duration::from_millis(50)) + .build() + .expect("build finalize heartbeat client"); + client + .start() + .await + .expect("start finalize heartbeat client"); + + // Hold the stall for several staleness windows once claiming has stopped. + let deadline = Instant::now() + ci_timing::scaled_timeout(Duration::from_secs(30)); + while runs.lock().expect("runs").len() < WORKERS as usize * 2 { + assert!(Instant::now() < deadline, "the runtime did not claim"); + tokio::time::sleep(Duration::from_millis(25)).await; + } + tokio::time::sleep(staleness * 4).await; + + sqlx::query("SELECT pg_catalog.pg_advisory_unlock_all()") + .execute(&mut *lock_holder) + .await + .expect("release the stalled finalization"); + drop(lock_holder); + let deadline = Instant::now() + ci_timing::scaled_timeout(Duration::from_secs(60)); + while completed_terminal_count(&pool, &store, queue).await < JOBS { + assert!(Instant::now() < deadline, "the backlog did not drain"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + client.shutdown(Duration::from_secs(10)).await; + + let runs = runs.lock().expect("runs"); + assert_eq!(runs.len() as i64, JOBS); + let rerun: Vec<_> = runs + .iter() + .filter(|(_, count)| **count > 1) + .take(5) + .collect(); + assert!(rerun.is_empty(), "jobs ran more than once: {rerun:?}"); +} + /// ADR-023 claim-ring control-plane smoke test. Exercises /// `rotate_claims`, the busy-check, and `prune_oldest_claims` on an /// empty schema: rotate cycles through every slot, prune is a noop @@ -15515,6 +15641,209 @@ async fn test_queue_storage_receipt_claims_are_locked_before_the_heartbeat_state .expect("heartbeat after the rewrite"); } +/// A finalizing attempt must not hold up unrelated heartbeats, and an old +/// heartbeat snapshot must not recreate state after durable completion. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_heartbeat_skips_closers_and_never_revives_closed_claims() { + let (_db_guard, pool) = setup_pool(10).await; + let config = QueueStorageConfig { + schema: "awa_qs_heartbeat_closure".to_string(), + ..Default::default() + }; + let store = create_store_with_config(&pool, config).await; + let queue = "heartbeat_closure"; + for id in 0..2 { + enqueue_job( + &pool, + &store, + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + ..Default::default() + }, + ) + .await; + } + let jobs = store + .claim_runtime_batch(&pool, queue, 2, Duration::ZERO) + .await + .expect("claim two compact receipts"); + assert_eq!(jobs.len(), 2); + let claims: Vec<_> = jobs + .iter() + .map(|job| { + awa::model::queue_storage::ClaimHeartbeatRef::from_claim( + job.job.id, + job.job.run_lease, + &job.claim, + ) + .expect("compact claim identity") + }) + .collect(); + let mut closer = pool.begin().await.expect("begin closer"); + sqlx::query( + "SELECT pg_advisory_xact_lock(hashtextextended( + format('awa.receipt.complete:%s:%s', $1::bigint, $2::bigint), 0))", + ) + .bind(jobs[0].job.id) + .bind(jobs[0].job.run_lease) + .execute(closer.as_mut()) + .await + .expect("hold one attempt's closer lock"); + let refreshed = tokio::time::timeout( + ci_timing::scaled_timeout(Duration::from_secs(2)), + store.heartbeat_claims_batch(&pool, &claims), + ) + .await + .expect("a closer cannot stall the heartbeat pass") + .expect("heartbeat the unlocked claim"); + assert_eq!(refreshed, 1); + closer.rollback().await.expect("release closer"); + + let completed = store + .complete_runtime_batch(&pool, &jobs) + .await + .expect("durable completion"); + assert_eq!(completed.len(), 2); + assert_eq!( + store + .heartbeat_claims_batch(&pool, &claims) + .await + .expect("a cached heartbeat snapshot after completion"), + 0, + ); + let remaining: i64 = sqlx::query_scalar(&format!( + "SELECT count(*) FROM {}.attempt_state WHERE job_id = ANY($1)", + store.schema() + )) + .bind(jobs.iter().map(|job| job.job.id).collect::>()) + .fetch_one(&pool) + .await + .expect("read attempt state"); + assert_eq!( + remaining, 0, + "a late heartbeat must not revive a closed attempt" + ); +} + +/// A progress snapshot captured before a handler returns must not wait behind +/// its finalizer and prevent unrelated attempts from heartbeating. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_pending_progress_skips_a_closer_and_can_retry() { + progress_snapshot_skips_closer(true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_queue_storage_legacy_pending_progress_skips_a_closer_and_can_retry() { + progress_snapshot_skips_closer(false).await; +} + +async fn progress_snapshot_skips_closer(receipts: bool) { + let (_db_guard, pool) = setup_pool(10).await; + let store = create_store_with_config( + &pool, + QueueStorageConfig { + schema: if receipts { + "awa_qs_pending_progress_closer".to_string() + } else { + "awa_qs_pending_progress_closer_legacy".to_string() + }, + lease_claim_receipts: receipts, + ..Default::default() + }, + ) + .await; + let queue = "pending_progress_closer"; + for id in 0..2 { + enqueue_job( + &pool, + &store, + &CompleteJob { id }, + InsertOpts { + queue: queue.to_string(), + ..Default::default() + }, + ) + .await; + } + let jobs = store + .claim_runtime_batch(&pool, queue, 2, Duration::ZERO) + .await + .expect("claim compact receipts"); + assert_eq!(jobs.len(), 2); + let pending: Vec<_> = jobs + .iter() + .map(|job| { + ( + job.job.id, + job.job.run_lease, + serde_json::json!({"step": 2}), + ) + }) + .collect(); + let mut closer = pool.begin().await.expect("begin closer"); + if receipts { + sqlx::query( + "SELECT pg_advisory_xact_lock(hashtextextended( + format('awa.receipt.complete:%s:%s', $1::bigint, $2::bigint), 0))", + ) + .bind(jobs[0].job.id) + .bind(jobs[0].job.run_lease) + .execute(closer.as_mut()) + .await + .expect("hold a closer lock after the progress snapshot"); + } else { + sqlx::query(&format!( + "SELECT 1 FROM {}.leases WHERE job_id = $1 AND run_lease = $2 FOR UPDATE", + store.schema() + )) + .bind(jobs[0].job.id) + .bind(jobs[0].job.run_lease) + .execute(closer.as_mut()) + .await + .expect("hold a legacy finalizer row lock"); + } + let updated = tokio::time::timeout( + ci_timing::scaled_timeout(Duration::from_secs(2)), + store.heartbeat_progress_batch(&pool, &pending), + ) + .await + .expect("a stale progress snapshot cannot block the heartbeat pass") + .expect("write the unlocked snapshot"); + assert_eq!(updated, 1); + let pairs: Vec<_> = jobs + .iter() + .map(|job| (job.job.id, job.job.run_lease)) + .collect(); + assert_eq!( + tokio::time::timeout( + ci_timing::scaled_timeout(Duration::from_secs(2)), + store.heartbeat_batch(&pool, &pairs), + ) + .await + .expect("heartbeat-only passes must also skip a closer") + .expect("heartbeat unlocked claims"), + 1, + ); + closer.rollback().await.expect("release closer"); + assert_eq!( + store + .heartbeat_progress_batch(&pool, &pending) + .await + .expect("retry the unacknowledged snapshots"), + 2, + ); + let progress: Vec = sqlx::query_scalar(&format!( + "SELECT progress FROM {}.attempt_state WHERE job_id = ANY($1) ORDER BY job_id", + store.schema() + )) + .bind(jobs.iter().map(|job| job.job.id).collect::>()) + .fetch_all(&pool) + .await + .expect("read durable progress"); + assert_eq!(progress, vec![serde_json::json!({"step": 2}); 2]); +} + /// Metadata completions validate the exact compact item and do not lock the /// immutable batch shared with other attempts. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From d1be1ebf16da4e5e632a19f540c6afcc25c69457 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 22:13:22 +1300 Subject: [PATCH 08/22] test(rescue): require durable recovery with no repeated executions Wait for committed completed terminal facts, require every recovered job to run exactly once on attempt 2 with its rescue recorded, and reject additional rescues of either live or recovered attempts. Keep the existing timeouts. The strict test compiles and all mandatory Rust workspace prechecks pass; final combined PostgreSQL acceptance is pending. --- awa/tests/bulk_rescue_test.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index 7bcdd3afd..f5680a61d 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -627,7 +627,11 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { wait_until( scaled_timeout(Duration::from_secs(120)), "the retried jobs to complete", - || async { recovered.lock().expect("recovered").len() as i64 >= ABANDONED - exhausted }, + || async { + count(&pool, format!( + "SELECT count(*) FROM awa.terminal_jobs WHERE queue = '{ABANDONED_QUEUE}' AND state = 'completed'" + )).await == ABANDONED - exhausted + }, ) .await; recovery.shutdown(Duration::from_secs(10)).await; @@ -636,12 +640,12 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { assert_eq!(recovered.len() as i64, ABANDONED - exhausted); let unexpected: Vec<_> = recovered .iter() - .filter(|(_, runs)| runs.first() != Some(&(2, true))) + .filter(|(_, runs)| runs.as_slice() != [(2, true)]) .take(5) .collect(); assert!( unexpected.is_empty(), - "each retried job first runs on attempt 2, with the rescue recorded: {unexpected:?}" + "each retried job runs exactly once on attempt 2, with the rescue recorded: {unexpected:?}" ); } @@ -668,19 +672,13 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { 0, "rescue took a claim from the live runtime" ); - // On this line a retried job's finished attempt can still be rescued - // from the recovery runtime, as it was before bounded rescue, so only - // the live runtime's claims are checked here. { let rescues = rescues.lock().expect("rescues"); - let live_rescued: Vec<_> = rescues - .iter() - .filter(|(_, rescued)| rescued.iter().any(|(queue, _)| *queue == LIVE_QUEUE)) - .take(5) - .collect(); + assert_eq!(rescues.len() as i64, ABANDONED); + let unexpected = rescued_twice(&rescues); assert!( - live_rescued.is_empty(), - "no claim of the live runtime was rescued: {live_rescued:?}" + unexpected.is_empty(), + "neither live nor recovered attempts may be rescued: {unexpected:?}" ); } let live_runs = live_runs.lock().expect("live runs"); From 675188eb6c60247522f9aa0e5766a59f522beda8 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Fri, 6 Nov 2026 22:45:17 +1300 Subject: [PATCH 09/22] test(worker): gate stalled completion after attempt locks --- awa/tests/queue_storage_runtime_test.rs | 75 +++++++++++++++++++------ 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/awa/tests/queue_storage_runtime_test.rs b/awa/tests/queue_storage_runtime_test.rs index e45b62370..e0b998538 100644 --- a/awa/tests/queue_storage_runtime_test.rs +++ b/awa/tests/queue_storage_runtime_test.rs @@ -2388,9 +2388,9 @@ async fn test_queue_storage_runtime_stops_claiming_while_finalization_is_stalled /// A finished attempt keeps heartbeating until its finalize is over, so a /// finalize that waits behind a stalled completion batcher for several /// staleness windows is not mistaken for a dead worker and run again. -/// Holding the first job's receipt-completion lock stalls the batcher's flush; -/// the attempts that flush has locked are skipped by rescue for as long as it -/// waits, and every other finished attempt must still be heartbeated. +/// An independent trigger gate stalls a completion after it has acquired all +/// of its attempts' locks. That batch remains protected from rescue while +/// every other finished attempt must still be heartbeated. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_queue_storage_finalizing_attempts_keep_heartbeating_through_a_stalled_finalize() { const WORKERS: u32 = 64; @@ -2421,27 +2421,48 @@ async fn test_queue_storage_finalizing_attempts_keep_heartbeating_through_a_stal .expect("enqueue backlog"); } - // Only the first job's lock: a held lock also keeps the heartbeat off that - // attempt, which is right only while a closer is working on it. - let first_job: i64 = sqlx::query_scalar(&format!( - "SELECT job_id FROM {}.ready_entries WHERE queue = $1 ORDER BY lane_seq LIMIT 1", - store.schema() + // The trigger runs after the completer owns every attempt lock in its + // flush. Holding an attempt lock directly would suppress its heartbeat + // before completion owns it, racing rescue when the test unlocks it. + sqlx::raw_sql(&format!( + r#" + CREATE FUNCTION {schema}.hold_completion_for_heartbeat_test() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.outcome = 'completed' THEN + PERFORM pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtextextended('awa.test.finalize-heartbeat-gate', 0) + ); + END IF; + RETURN NEW; + END + $$; + CREATE TRIGGER hold_completion_for_heartbeat_test + BEFORE INSERT ON {schema}.lease_claim_closure_batches + FOR EACH ROW EXECUTE FUNCTION {schema}.hold_completion_for_heartbeat_test(); + "#, + schema = store.schema(), )) - .bind(queue) - .fetch_one(&pool) + .execute(&pool) .await - .expect("first job"); + .expect("install the completion gate"); let mut lock_holder = pool.acquire().await.expect("lock connection"); + let lock_holder_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *lock_holder) + .await + .expect("gate holder PID"); sqlx::query( - "SELECT pg_catalog.pg_advisory_lock(pg_catalog.hashtextextended( - format('awa.receipt.complete:%s:%s', $1::bigint, 1), 0))", + "SELECT pg_catalog.pg_advisory_lock( + pg_catalog.hashtextextended('awa.test.finalize-heartbeat-gate', 0))", ) - .bind(first_job) .execute(&mut *lock_holder) .await - .expect("stall the first job's finalization"); + .expect("hold the independent completion gate"); - let staleness = ci_timing::scaled_staleness(Duration::from_secs(1)); + // Sequential heartbeat chunks can need several WAL-sync round trips + // under contention. Allow margin for those writes while blocking the + // completion for four full staleness windows below. + let staleness = ci_timing::scaled_staleness(Duration::from_secs(5)); let runs: Arc>> = Arc::default(); let recorded = runs.clone(); let client = Client::builder(pool.clone()) @@ -2483,8 +2504,26 @@ async fn test_queue_storage_finalizing_attempts_keep_heartbeating_through_a_stal // Hold the stall for several staleness windows once claiming has stopped. let deadline = Instant::now() + ci_timing::scaled_timeout(Duration::from_secs(30)); - while runs.lock().expect("runs").len() < WORKERS as usize * 2 { - assert!(Instant::now() < deadline, "the runtime did not claim"); + loop { + let completion_at_gate: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event = 'advisory' + AND $1 = ANY(pg_catalog.pg_blocking_pids(pid)) + )", + ) + .bind(lock_holder_pid) + .fetch_one(&pool) + .await + .expect("observe the completion gate"); + if completion_at_gate && runs.lock().expect("runs").len() >= WORKERS as usize * 2 { + break; + } + assert!( + Instant::now() < deadline, + "the runtime did not queue attempts behind the completion gate" + ); tokio::time::sleep(Duration::from_millis(25)).await; } tokio::time::sleep(staleness * 4).await; From c149591b14b4f81a045085aff470516e45daf660 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 7 Nov 2026 03:35:13 +1300 Subject: [PATCH 10/22] test(rescue): count retry states in one snapshot (0.6) --- awa/tests/bulk_rescue_test.rs | 134 ++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 14 deletions(-) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index f5680a61d..7f3c4b621 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -70,6 +70,119 @@ async fn count(pool: &PgPool, sql: String) -> i64 { .unwrap_or_else(|err| panic!("{sql}: {err}")) } +/// Retry promotion transfers a row between these states in one transaction. +/// Read both counts from one statement so a commit between reads cannot count +/// the same job once in deferred storage and again in ready storage. +async fn waiting_retry_counts(pool: &PgPool, queue: &str) -> (i64, i64) { + let overview = awa::model::admin::queue_overview(pool, queue) + .await + .expect("retry accounting overview") + .expect("retry accounting queue"); + (overview.retryable, overview.available) +} + +#[tokio::test] +async fn retry_accounting_uses_one_snapshot_during_promotion() { + let db = BulkRescueDatabase::create().await; + let store = QueueStorage::new(QueueStorageConfig::default()).expect("queue storage"); + let queue = "bulk_rescue_retry_snapshot"; + let params = awa::model::insert::params_with( + &AbandonedJob { n: 0 }, + InsertOpts { + queue: queue.into(), + max_attempts: 3, + ..Default::default() + }, + ) + .expect("retry accounting job"); + store + .enqueue_params_batch(db.pool(), &[params]) + .await + .expect("enqueue retry accounting job"); + let claims = store + .claim_runtime_batch(db.pool(), queue, 1, Duration::ZERO) + .await + .expect("claim retry accounting job"); + assert_eq!(claims.len(), 1); + store + .retry_after( + db.pool(), + claims[0].job.id, + claims[0].job.run_lease, + Duration::ZERO, + None, + ) + .await + .expect("retry accounting job") + .expect("claimed job becomes retryable"); + assert_eq!(waiting_retry_counts(db.pool(), queue).await, (1, 0)); + + // Hold the observer's only connection after its first statement. A real + // promotion commits before any second observer statement can start. The + // old split read therefore counts this one job twice; the overview's two + // counts always describe the same statement snapshot. + let armed = Arc::new(AtomicBool::new(false)); + let read_finished = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let observer = PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(scaled_timeout(Duration::from_secs(10))) + .after_release({ + let armed = armed.clone(); + let read_finished = read_finished.clone(); + let release = release.clone(); + move |_, _| { + let gate = armed.swap(false, Ordering::SeqCst); + let read_finished = read_finished.clone(); + let release = release.clone(); + Box::pin(async move { + if gate { + read_finished.notify_one(); + release.notified().await; + } + Ok(true) + }) + } + }) + .connect(&db.url()) + .await + .expect("retry accounting observer pool"); + let primed = observer.acquire().await.expect("prime observer connection"); + drop(primed); + tokio::time::timeout(scaled_timeout(Duration::from_secs(10)), async { + while observer.num_idle() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("observer's initial connection is idle before arming"); + armed.store(true, Ordering::SeqCst); + let promote = async { + let observed = tokio::time::timeout( + scaled_timeout(Duration::from_secs(10)), + read_finished.notified(), + ) + .await; + if observed.is_err() { + release.notify_one(); + panic!("retry accounting observer did not finish its first statement"); + } + let promoted = store + .promote_due(db.pool(), awa::JobState::Retryable, 1) + .await; + release.notify_one(); + assert_eq!(promoted.expect("promote between accounting reads"), 1); + }; + let (counts, ()) = tokio::time::timeout(scaled_timeout(Duration::from_secs(10)), async { + tokio::join!(waiting_retry_counts(&observer, queue), promote) + }) + .await + .expect("bounded retry accounting interleaving"); + observer.close().await; + assert_eq!(counts.0 + counts.1, 1, "one job is waiting to retry"); + assert_eq!(waiting_retry_counts(db.pool(), queue).await, (0, 1)); +} + async fn running(pool: &PgPool, store: &QueueStorage, queue: &str) -> i64 { store .queue_counts(pool, queue) @@ -175,7 +288,12 @@ struct BulkRescueDatabase { impl BulkRescueDatabase { async fn create() -> Self { - let name = format!("awa_bulk_rescue_{}", std::process::id()); + static NEXT_DATABASE: AtomicU64 = AtomicU64::new(0); + let name = format!( + "awa_bulk_rescue_{}_{}", + std::process::id(), + NEXT_DATABASE.fetch_add(1, Ordering::Relaxed) + ); let mut admin = PgConnection::connect(&with_database(&database_url(), "postgres")) .await .expect("admin connection"); @@ -546,19 +664,7 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { exhausted, "every one-attempt job is dead-lettered by its rescue" ); - let retryable = count( - &pool, - format!( - "SELECT count(*) FROM awa.deferred_jobs - WHERE queue = '{ABANDONED_QUEUE}' AND state = 'retryable'" - ), - ) - .await; - let promoted = store - .queue_counts(&pool, ABANDONED_QUEUE) - .await - .expect("abandoned queue counts") - .available; + let (retryable, promoted) = waiting_retry_counts(&pool, ABANDONED_QUEUE).await; assert_eq!( retryable + promoted, ABANDONED - exhausted, From fa9c50049626b73a29c3ae4e0b331c8e4f46802a Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 7 Nov 2026 04:56:06 +1300 Subject: [PATCH 11/22] fix(worker): keep confirmed heartbeat rescue cursor windows small Use the existing follow-up scan window from the first heartbeat batch when confirmed-owner evidence and both acceleration controls are enabled. Keep ordinary, unconfirmed and disabled paths on the wider initial scan. Exercise sparse closed-prefix progress, exact-once disposition, fresh protection and disabled/unconfirmed wide coverage through real maintenance passes. SQL, cutoffs, lock ordering and timing bounds are unchanged. --- CHANGELOG.md | 4 + awa-worker/src/maintenance.rs | 259 +++++++++++++++++++++++++++++++++- 2 files changed, 259 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa2fa8386..942efb3d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- Keep confirmed-owner bulk heartbeat rescue's first cursor window small so + short passes can reach follow-up batches. Unconfirmed owners and + configurations that disable acceleration retain the wider initial scan. + - Queue-storage rescue drains large abandoned-claim backlogs in bounded transactions, controlled by `rescue_pass_budget` and a soft `rescue_pass_max_rows` cap. Cursor sweeps continue past closed or freshly diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index 53689d6ce..d553c53ad 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -238,9 +238,9 @@ pub(crate) const DEFAULT_RESCUE_PASS_MAX_ROWS: usize = 5_000; /// a drain to about `rescue_pass_max_rows` attempts per second per kind. const RESCUE_BACKLOG_INTERVAL: Duration = Duration::from_secs(1); -/// Claims each slot's cursor walk examines in a batch that follows a full -/// one in the same pass. The first batch keeps the store's wider default so -/// a pass can still walk past long runs of closed claims. +/// Claims each slot's cursor walk examines during bulk heartbeat rescue and +/// follow-up batches. Other first batches keep the store's wider default to +/// walk past long closed prefixes when acceleration is unavailable or disabled. const RESCUE_FOLLOW_UP_SCAN_LIMIT: usize = 2_000; /// One batch in this many visits the other claim slots before the oldest, so @@ -2033,7 +2033,12 @@ impl MaintenanceService { slot_order, skip_slots: sweep.idle_slots.clone(), retry_not_before: None, - cursor_scan_limit: (batches > 0).then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), + cursor_scan_limit: (batches > 0 + || (kind == RescueKind::Heartbeat + && bulk_heartbeat_cutoff.is_some() + && !budget.is_zero() + && self.rescue_pass_max_rows > 0)) + .then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), }; let outcome = match kind { RescueKind::Heartbeat => { @@ -4900,6 +4905,252 @@ mod tests { assert_eq!(sweep.idle_slots, vec![0]); } + struct SparseHeartbeatFixture { + pool: PgPool, + runtime: QueueStorageRuntime, + service: MaintenanceService, + stale_job: i64, + fresh_job: i64, + fresh_runtime: Uuid, + } + + impl SparseHeartbeatFixture { + async fn new(budget: Duration, max_rows: usize, confirmed: bool) -> Self { + let pool = setup_pool(4).await; + reset_schema(&pool).await; + migrations::run(&pool).await.expect("migrations"); + let config = QueueStorageConfig::default(); + let store = QueueStorage::new(config.clone()).expect("queue storage"); + store.install(&pool).await.expect("install"); + let jobs: Vec<_> = (0..2) + .map(|id| awa_model::InsertParams { + kind: "sparse_heartbeat".into(), + args: serde_json::json!({ "id": id }), + opts: awa_model::InsertOpts { + queue: "sparse_heartbeat".into(), + max_attempts: 3, + ..Default::default() + }, + }) + .collect(); + store + .enqueue_params_batch(&pool, &jobs) + .await + .expect("enqueue"); + let claims = store + .claim_runtime_batch(&pool, "sparse_heartbeat", 2, Duration::from_secs(3600)) + .await + .expect("claim real jobs"); + assert_eq!(claims.len(), 2); + let slot = claims[0].claim.claim_slot; + assert_eq!(claims[1].claim.claim_slot, slot); + let stamp = Utc::now() - chrono::Duration::hours(2); + + // Keep the old row-local representation valid on both versions. + // Closed history needs no payload; the two open claims retain their + // real ready rows so rescue exercises the actual disposition path. + sqlx::query("DELETE FROM awa.lease_claim_batches WHERE queue = 'sparse_heartbeat'") + .execute(&pool) + .await + .expect("replace compact claims with legacy rows"); + for (index, claim) in claims.iter().enumerate() { + sqlx::query( + "INSERT INTO awa.lease_claims + (claim_slot, receipt_id, job_id, run_lease, ready_slot, ready_generation, + queue, priority, attempt, max_attempts, lane_seq, enqueue_shard, claimed_at, deadline_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,clock_timestamp()+interval '1 hour') + ON CONFLICT (claim_slot,job_id,run_lease) DO UPDATE + SET claimed_at=EXCLUDED.claimed_at, deadline_at=EXCLUDED.deadline_at", + ) + .bind(slot).bind(claim.claim.receipt_id).bind(claim.job.id).bind(claim.job.run_lease) + .bind(claim.claim.ready_slot).bind(claim.claim.ready_generation).bind(&claim.claim.queue) + .bind(claim.claim.priority).bind(claim.job.attempt).bind(claim.job.max_attempts) + .bind(claim.claim.lane_seq).bind(claim.claim.enqueue_shard) + .bind(stamp + chrono::Duration::microseconds(if index == 0 { 3500 } else { 4500 })) + .execute(&pool).await.expect("legacy open receipt"); + } + sqlx::query( + "INSERT INTO awa.lease_claims + (claim_slot,job_id,run_lease,ready_slot,ready_generation,queue,priority, + attempt,max_attempts,lane_seq,claimed_at,closed_at) + SELECT $1,1000000+id,1,0,0,'sparse_history',0,1,3,id, + $2::timestamptz+id*interval '1 microsecond',clock_timestamp() + FROM generate_series(1,6000) AS id", + ) + .bind(slot) + .bind(stamp) + .execute(&pool) + .await + .expect("sparse closed prefix"); + let stale_job = claims[0].job.id; + let fresh_job = claims[1].job.id; + sqlx::query( + "INSERT INTO awa.attempt_state(job_id,run_lease,heartbeat_at,updated_at) + VALUES($1,$2,clock_timestamp(),clock_timestamp())", + ) + .bind(fresh_job) + .bind(claims[1].job.run_lease) + .execute(&pool) + .await + .expect("protect fresh claim"); + let fresh_runtime = Uuid::new_v4(); + for (id, seen, heartbeat_alive) in [ + (Uuid::new_v4(), stamp + chrono::Duration::hours(1), false), + (fresh_runtime, Utc::now(), true), + ] { + sqlx::query( + "INSERT INTO awa.runtime_instances + (instance_id,hostname,pid,version,started_at,last_seen_at,snapshot_interval_ms, + healthy,postgres_connected,poll_loop_alive,heartbeat_alive,shutting_down,leader) + VALUES($1,'sparse_test',1,'0.6.13',$2,$2,1000,true,true,true,$3,false,false)", + ) + .bind(id).bind(seen).bind(heartbeat_alive) + .execute(&pool).await.expect("runtime evidence"); + } + let runtime = + QueueStorageRuntime::new(config, Duration::from_secs(1), Duration::from_secs(1)) + .expect("runtime"); + let service = MaintenanceService::new( + pool.clone(), + metrics_for_test(), + Arc::new(AtomicBool::new(true)), + Arc::new(AtomicBool::new(true)), + CancellationToken::new(), + Arc::new(Vec::new()), + InFlightMap::default(), + RuntimeStorage::QueueStorage(runtime.clone()), + Arc::new(HashMap::new()), + Arc::new(HashMap::new()), + ) + .rescue_pass_budget(budget) + .rescue_pass_max_rows(max_rows); + if confirmed { + let now = Instant::now(); + *service + .rescue_owner_evidence + .lock() + .expect("owner evidence") = RescueOwnerEvidence { + steady_since: Some(now - service.heartbeat_staleness * 2), + last_sample: Some(now), + }; + } + Self { + pool, + runtime, + service, + stale_job, + fresh_job, + fresh_runtime, + } + } + + async fn pass(&self) -> bool { + sqlx::query("UPDATE awa.runtime_instances SET last_seen_at=clock_timestamp() WHERE instance_id=$1") + .bind(self.fresh_runtime).execute(&self.pool).await.expect("fresh owner snapshot"); + self.service + .rescue_queue_storage_pass(RescueKind::Heartbeat, &self.runtime, false) + .await + } + + async fn assert_dispositions(&self) { + let stale = self + .runtime + .store + .load_job(&self.pool, self.stale_job) + .await + .expect("stale job lookup") + .expect("stale job"); + assert_eq!(stale.state, JobState::Retryable); + let fresh = self + .runtime + .store + .load_job(&self.pool, self.fresh_job) + .await + .expect("fresh job lookup") + .expect("fresh job"); + assert_eq!( + fresh.state, + JobState::Running, + "fresh heartbeat stays protected" + ); + let closures: i64 = + sqlx::query_scalar("SELECT count(*) FROM awa.lease_claim_closures WHERE job_id=$1") + .bind(self.stale_job) + .fetch_one(&self.pool) + .await + .expect("closure count"); + assert_eq!(closures, 1, "sparse stale attempt is rescued exactly once"); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn rescue_confirmed_heartbeat_sparse_sweep_progresses_and_stops() { + let _guard = db_test_mutex().lock().await; + // A positive budget below any database round trip makes each pass + // finish its first batch and rearm; no elapsed-time assertion is used. + let run = SparseHeartbeatFixture::new(Duration::from_nanos(1), 5000, true).await; + assert!(run.pass().await, "closed prefix leaves a resumable sweep"); + let cursor: i64 = sqlx::query_scalar( + "SELECT rescue_cursor_job_id FROM awa.claim_ring_slots WHERE slot=0", + ) + .fetch_one(&run.pool) + .await + .expect("first persisted cursor"); + assert_eq!( + cursor, 1002000, + "confirmed bulk starts with one small window" + ); + let stale = run + .runtime + .store + .load_job(&run.pool, run.stale_job) + .await + .expect("unvisited job lookup") + .expect("unvisited job"); + assert_eq!( + stale.state, + JobState::Running, + "sparse job lies beyond the first window" + ); + let mut drained = false; + for _ in 0..12 { + if !run.pass().await { + drained = true; + break; + } + } + assert!( + drained, + "persisted sparse sweep converges instead of rearming forever" + ); + assert!(run.service.rescue_sweeps.lock().expect("sweeps").is_empty()); + run.assert_dispositions().await; + run.pool.close().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn rescue_disabled_acceleration_keeps_wide_first_heartbeat_window() { + let _guard = db_test_mutex().lock().await; + for (budget, max_rows) in [(Duration::ZERO, 5000), (Duration::from_secs(1), 0)] { + let run = SparseHeartbeatFixture::new(budget, max_rows, true).await; + assert!(!run.pass().await, "disabled acceleration never rearms"); + run.assert_dispositions().await; + run.pool.close().await; + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn rescue_unconfirmed_heartbeat_keeps_wide_first_window() { + let _guard = db_test_mutex().lock().await; + let run = SparseHeartbeatFixture::new(Duration::from_nanos(1), 5000, false).await; + assert!( + !run.pass().await, + "unconfirmed owners keep the ordinary one-batch cadence" + ); + run.assert_dispositions().await; + run.pool.close().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn rescue_closed_history_stops_rearming() { let _guard = db_test_mutex().lock().await; From 826727d73ced953ecdb42132ed9f7866bc7ba031 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 7 Nov 2026 10:02:55 +1300 Subject: [PATCH 12/22] fix(worker): preserve ordinary rescue ticks under jitter --- CHANGELOG.md | 3 + awa-worker/src/maintenance.rs | 314 ++++++++++++++++++++++++++++------ docs/configuration.md | 3 +- docs/troubleshooting.md | 2 +- 4 files changed, 269 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 942efb3d0..ff19d2960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- Ordinary rescue ticks retain their cadence under scheduler jitter, while + backlog re-arms keep the heartbeat owner-evidence gate. + - Keep confirmed-owner bulk heartbeat rescue's first cursor window small so short passes can reach follow-up batches. Unconfirmed owners and configurations that disable acceleration retain the wider initial scan. diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index d553c53ad..baac2960d 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -248,11 +248,12 @@ const RESCUE_FOLLOW_UP_SCAN_LIMIT: usize = 2_000; const RESCUE_OTHERS_FIRST_EVERY: usize = 4; /// Runtime snapshots the leader reads before accelerating stale-heartbeat rescue. A runtime is `fresh` while its snapshot is -/// within three snapshot intervals; one that is neither fresh nor silent for -/// the whole heartbeat staleness ($1, milliseconds) could be a live runtime -/// that is stalled rather than gone. `last_gone_heartbeat` bounds the last -/// heartbeat any gone runtime can have written: it wrote a snapshot at least -/// every interval while it lived. +/// within three effective snapshot intervals. An unconfirmed phase exists only +/// when 3 * max(snapshot_interval, 1s) < heartbeat staleness ($1, milliseconds). +/// With a larger interval a stalled live runtime can pass directly from fresh +/// to gone; this evidence is a bounded heuristic, not proof of process death. +/// `last_gone_heartbeat` estimates the last heartbeat of a gone runtime from +/// its final snapshot and configured interval. const RESCUE_RUNTIME_EVIDENCE_SQL: &str = r#" SELECT count(*) FILTER (WHERE fresh AND NOT heartbeat_alive)::bigint AS fresh_without_heartbeat, @@ -330,10 +331,11 @@ impl RescuePassEnd { /// fleet-wide: every runtime snapshot is either fresh with its heartbeat /// loop alive, or silent for at least the heartbeat staleness. That must /// hold at every sample for a full staleness window before a stale-heartbeat -/// rescue may run extra batches or early passes, so a live runtime coming out of a stall -/// has had the whole window to refresh its heartbeats first. A gap between -/// samples longer than half the staleness restarts the window, because a -/// stall the leader did not sample could have hidden behind it. +/// rescue may run extra batches or early passes. This settled window may have +/// started before a runtime stalled. If freshness lasts at least the staleness, +/// that runtime can pass directly from fresh to gone without resetting it. +/// A gap between samples longer than half the staleness restarts the window, +/// because a stall the leader did not sample could have hidden behind it. #[derive(Debug, Default)] struct RescueOwnerEvidence { steady_since: Option, @@ -360,6 +362,40 @@ impl RescueOwnerEvidence { } } +/// A normal interval tick is a baseline visit even if scheduler jitter makes +/// its actual start slightly earlier than one interval after the last start. +/// A reset tick is accelerated work and must retain the owner/cadence gate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RescueTickOrigin { + Ordinary, + Backlog, +} + +struct RescueTimer { + interval: tokio::time::Interval, + next_origin: RescueTickOrigin, +} + +impl RescueTimer { + fn new(period: Duration) -> Self { + Self { + interval: leader_loop_interval(period), + next_origin: RescueTickOrigin::Ordinary, + } + } + + async fn tick(&mut self) -> RescueTickOrigin { + self.interval.tick().await; + // Consume even when the caller's branch cooldown skips this tick. + std::mem::replace(&mut self.next_origin, RescueTickOrigin::Ordinary) + } + + fn reset_after(&mut self, duration: Duration) { + self.interval.reset_after(duration); + self.next_origin = RescueTickOrigin::Backlog; + } +} + /// A sweep survives early re-arms: closed history must not keep the leader /// scanning once a cursor has made a full revolution. The first window is /// already visited, so crossing its end after wrapping completes the sweep. @@ -383,10 +419,11 @@ impl RescueSweep { /// A long drain must revisit slots whose claims have become eligible /// since their last scan. Keep unfinished cursors so a large history /// sweep still converges across these ordinary-cadence revisits. - fn revisit(&mut self, now: Instant, interval: Duration) { - if self - .last_revisit - .is_none_or(|last| now.saturating_duration_since(last) >= interval) + fn revisit(&mut self, now: Instant, interval: Duration, origin: RescueTickOrigin) { + if origin == RescueTickOrigin::Ordinary + || self + .last_revisit + .is_none_or(|last| now.saturating_duration_since(last) >= interval) { self.idle_slots.clear(); for cursor in self.cursors.values_mut().flatten() { @@ -437,10 +474,12 @@ impl RescueHeartbeatCadence { now: Instant, interval: Duration, cutoff: Option>, + origin: RescueTickOrigin, ) -> Option>> { - if self - .last_baseline - .is_none_or(|last| now.saturating_duration_since(last) >= interval) + if origin == RescueTickOrigin::Ordinary + || self + .last_baseline + .is_none_or(|last| now.saturating_duration_since(last) >= interval) { self.last_baseline = Some(now); Some(None) @@ -1446,9 +1485,9 @@ impl MaintenanceService { self.leader.store(true, Ordering::SeqCst); // Run maintenance tasks as leader - let mut heartbeat_rescue_timer = leader_loop_interval(self.heartbeat_rescue_interval); - let mut deadline_rescue_timer = leader_loop_interval(self.deadline_rescue_interval); - let mut callback_rescue_timer = leader_loop_interval(self.callback_rescue_interval); + let mut heartbeat_rescue_timer = RescueTimer::new(self.heartbeat_rescue_interval); + let mut deadline_rescue_timer = RescueTimer::new(self.deadline_rescue_interval); + let mut callback_rescue_timer = RescueTimer::new(self.callback_rescue_interval); let mut promote_timer = leader_loop_interval(self.promote_interval); let mut cleanup_timer = leader_loop_interval(self.cleanup_interval); let mut cron_sync_timer = leader_loop_interval(self.cron_sync_interval); @@ -1574,27 +1613,27 @@ impl MaintenanceService { let _ = Self::release_leader(&mut leader_conn).await; return; } - _ = heartbeat_rescue_timer.tick() => { + origin = heartbeat_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_stale_heartbeats", self.heartbeat_rescue_interval, &self.metrics) { - let backlog = self.rescue_stale_heartbeats().await; + let backlog = self.rescue_stale_heartbeats(origin).await; timer.finish(); if backlog { heartbeat_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.heartbeat_rescue_interval)); } } } - _ = deadline_rescue_timer.tick() => { + origin = deadline_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_expired_deadlines", self.deadline_rescue_interval, &self.metrics) { - let backlog = self.rescue_expired_deadlines().await; + let backlog = self.rescue_expired_deadlines(origin).await; timer.finish(); if backlog { deadline_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.deadline_rescue_interval)); } } } - _ = callback_rescue_timer.tick() => { + origin = callback_rescue_timer.tick() => { if let Some(timer) = branch_tracker.try_begin("rescue_expired_callbacks", self.callback_rescue_interval, &self.metrics) { - let backlog = self.rescue_expired_callbacks().await; + let backlog = self.rescue_expired_callbacks(origin).await; timer.finish(); if backlog { callback_rescue_timer.reset_after(RESCUE_BACKLOG_INTERVAL.min(self.callback_rescue_interval)); @@ -1932,21 +1971,21 @@ impl MaintenanceService { /// Returns whether the queue-storage pass left a backlog it may keep /// draining, so the next pass should run before the full interval. #[tracing::instrument(skip(self), name = "maintenance.rescue_stale")] - async fn rescue_stale_heartbeats(&self) -> bool { + async fn rescue_stale_heartbeats(&self, origin: RescueTickOrigin) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_stale_heartbeats().await; self.process_heartbeat_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { return self - .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, false) + .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, false, origin) .await; } false } RuntimeStorage::QueueStorage(runtime) => { let backlog = self - .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, true) + .rescue_queue_storage_pass(RescueKind::Heartbeat, runtime, true, origin) .await; let outcome = self.rescue_canonical_stale_heartbeats().await; self.process_heartbeat_rescues(outcome, false, None).await; @@ -1966,9 +2005,10 @@ impl MaintenanceService { /// stale-heartbeat pass takes more than one batch only when /// [`RescueOwnerEvidence`] holds, and its later batches take only claims /// last heartbeated before the most recently gone runtime stopped - /// reporting. A live runtime whose heartbeats merely lag has refreshed its - /// claims since then. Deadline and callback rescues act on a deadline that - /// has passed whoever owns the attempt. + /// reporting. This reduces the chance of draining a lagging live runtime, + /// but snapshot freshness and sampling cannot prove an owner is dead. + /// Deadline and callback rescues act on a deadline that has passed whoever + /// owns the attempt. /// /// Returns whether the pass stopped with a backlog it may keep draining, /// so the next pass should run before the full interval. @@ -1977,6 +2017,7 @@ impl MaintenanceService { kind: RescueKind, runtime: &QueueStorageRuntime, signal_local: bool, + origin: RescueTickOrigin, ) -> bool { let started = Instant::now(); let interval = match kind { @@ -1996,7 +2037,7 @@ impl MaintenanceService { .rescue_heartbeat_cadence .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .select(started, interval, bulk_heartbeat_cutoff); + .select(started, interval, bulk_heartbeat_cutoff, origin); let Some(cutoff) = selected else { self.rescue_sweeps .lock() @@ -2017,7 +2058,7 @@ impl MaintenanceService { .unwrap_or_else(std::sync::PoisonError::into_inner) .remove(&kind) .unwrap_or_default(); - sweep.revisit(started, interval); + sweep.revisit(started, interval, origin); let pass_end = loop { let slot_order = if sweep.batches_started % RESCUE_OTHERS_FIRST_EVERY == RESCUE_OTHERS_FIRST_EVERY - 1 @@ -2384,21 +2425,21 @@ impl MaintenanceService { /// #456 mirror: sweeps both storage planes; see /// [`Self::rescue_stale_heartbeats`]. #[tracing::instrument(skip(self), name = "maintenance.rescue_deadline")] - async fn rescue_expired_deadlines(&self) -> bool { + async fn rescue_expired_deadlines(&self, origin: RescueTickOrigin) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_expired_deadlines().await; self.process_deadline_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { return self - .rescue_queue_storage_pass(RescueKind::Deadline, runtime, false) + .rescue_queue_storage_pass(RescueKind::Deadline, runtime, false, origin) .await; } false } RuntimeStorage::QueueStorage(runtime) => { let backlog = self - .rescue_queue_storage_pass(RescueKind::Deadline, runtime, true) + .rescue_queue_storage_pass(RescueKind::Deadline, runtime, true, origin) .await; let outcome = self.rescue_canonical_expired_deadlines().await; self.process_deadline_rescues(outcome, false, None).await; @@ -2549,21 +2590,21 @@ impl MaintenanceService { /// doubly here because callback-carrying jobs never migrate off the /// canonical plane during a transition. #[tracing::instrument(skip(self), name = "maintenance.rescue_callback_timeout")] - async fn rescue_expired_callbacks(&self) -> bool { + async fn rescue_expired_callbacks(&self, origin: RescueTickOrigin) -> bool { match &self.storage { RuntimeStorage::Canonical => { let outcome = self.rescue_canonical_expired_callbacks().await; self.process_callback_rescues(outcome, true, None).await; if let Some(runtime) = self.active_standby_queue_storage().await { return self - .rescue_queue_storage_pass(RescueKind::Callback, runtime, false) + .rescue_queue_storage_pass(RescueKind::Callback, runtime, false, origin) .await; } false } RuntimeStorage::QueueStorage(runtime) => { let backlog = self - .rescue_queue_storage_pass(RescueKind::Callback, runtime, true) + .rescue_queue_storage_pass(RescueKind::Callback, runtime, true, origin) .await; let outcome = self.rescue_canonical_expired_callbacks().await; self.process_callback_rescues(outcome, false, None).await; @@ -4540,8 +4581,12 @@ mod tests { Arc::new(HashMap::new()), ); - service.rescue_stale_heartbeats().await; - service.rescue_expired_callbacks().await; + service + .rescue_stale_heartbeats(RescueTickOrigin::Backlog) + .await; + service + .rescue_expired_callbacks(RescueTickOrigin::Backlog) + .await; // Post-flip the `awa.jobs` view exposes only the queue-storage // plane, so assert against the physical canonical tables: both @@ -4794,24 +4839,176 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + async fn ordinary_rescue_ticks_survive_jitter_and_revisit_finished_slots() { + let interval = Duration::from_secs(30); + let mut timer = RescueTimer::new(interval); + timer.tick().await; + let mut cadence = RescueHeartbeatCadence::default(); + let mut sweep = RescueSweep::default(); + tokio::time::advance(interval + Duration::from_millis(2)).await; + let first_origin = timer.tick().await; + assert_eq!(first_origin, RescueTickOrigin::Ordinary); + let first_start = tokio::time::Instant::now().into_std(); + assert_eq!( + cadence.select(first_start, interval, None, first_origin), + Some(None) + ); + sweep.revisit(first_start, interval, first_origin); + let stamp = Utc::now(); + for (slot, saturated) in [(0, false), (1, true)] { + sweep.observe(RescueScanProgress { + slot, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, 100, 1)), + saturated, + }], + }); + } + assert_eq!(sweep.idle_slots, vec![0]); + tokio::time::advance(interval - Duration::from_millis(2)).await; + let next_origin = timer.tick().await; + let next_start = tokio::time::Instant::now().into_std(); + assert_eq!( + next_start.duration_since(first_start), + interval - Duration::from_millis(2) + ); + assert_eq!(next_origin, RescueTickOrigin::Ordinary); + assert_eq!( + cadence.select(next_start, interval, None, next_origin), + Some(None), + "an ordinary tick must not disappear because the prior tick started 2ms late" + ); + sweep.revisit(next_start, interval, next_origin); + assert!( + sweep.idle_slots.is_empty(), + "newly eligible finished slots must be revisited" + ); + assert!(!sweep.cursors[&0][0].done); + assert_eq!( + sweep.cursors[&1][0].first, + Some((stamp, 100, 1)), + "ordinary revisits must retain unfinished sweep progress" + ); + } + + #[test] + fn ordinary_rescue_revisit_preserves_unfinished_cursor_under_jitter() { + let interval = Duration::from_secs(30); + let now = Instant::now(); + let first_start = now + interval + Duration::from_millis(2); + let second_start = now + interval * 2; + let stamp = Utc::now(); + let mut sweep = RescueSweep::default(); + sweep.revisit(first_start, interval, RescueTickOrigin::Ordinary); + for (slot, saturated) in [(0, false), (1, true)] { + sweep.observe(RescueScanProgress { + slot, + cursors: vec![awa_model::RescueScanCursor { + position: Some((stamp, 100, 1)), + saturated, + }], + }); + } + assert_eq!(sweep.idle_slots, vec![0]); + sweep.revisit(second_start, interval, RescueTickOrigin::Ordinary); + assert!( + sweep.idle_slots.is_empty(), + "finished slot must be revisited on the ordinary tick" + ); + assert!(!sweep.cursors[&0][0].done); + assert_eq!( + sweep.cursors[&1][0].first, + Some((stamp, 100, 1)), + "unfinished sweep progress must survive the same ordinary revisit" + ); + } + + #[tokio::test(start_paused = true)] + async fn delayed_rescue_baseline_does_not_turn_early_rearm_into_baseline() { + let interval = Duration::from_secs(30); + let mut timer = RescueTimer::new(interval); + timer.tick().await; + tokio::time::advance(Duration::from_secs(52)).await; + let first_origin = timer.tick().await; + let first_start = tokio::time::Instant::now().into_std(); + let mut cadence = RescueHeartbeatCadence::default(); + assert_eq!( + cadence.select(first_start, interval, None, first_origin), + Some(None) + ); + timer.reset_after(Duration::from_secs(8)); + let origin = timer.tick().await; + let now = tokio::time::Instant::now().into_std(); + assert_eq!(now.duration_since(first_start), Duration::from_secs(8)); + assert_eq!(origin, RescueTickOrigin::Backlog); + assert_eq!(cadence.select(now, interval, None, origin), None, + "a tick scheduled 30s but serviced at52s must not make backlog60s an unguarded baseline"); + let cutoff = Utc::now(); + assert_eq!( + cadence.select(now, interval, Some(cutoff), origin), + Some(Some(cutoff)), + "an early re-arm must retain the owner cutoff" + ); + let origin = timer.tick().await; + let now = tokio::time::Instant::now().into_std(); + assert_eq!(origin, RescueTickOrigin::Ordinary); + assert_eq!(now.duration_since(first_start), Duration::from_secs(38)); + assert_eq!(cadence.select(now, interval, None, origin), Some(None)); + } + + #[tokio::test(start_paused = true)] + async fn rescue_backlog_tick_is_consumed_even_when_branch_cooldown_skips_it() { + let (tracker, clock, cooldown_until) = tracker_flipped_by_250ms_bodies(); + let metrics = metrics_for_test(); + let mut timer = RescueTimer::new(Duration::from_secs(30)); + timer.tick().await; + timer.reset_after(TICK_100MS); + let origin = timer.tick().await; + assert_eq!(origin, RescueTickOrigin::Backlog); + let skipped_at = clock + Duration::from_nanos(1); + assert!(skipped_at < cooldown_until); + assert!(tracker + .try_begin_at("cleanup", TICK_100MS, &metrics, true, skipped_at) + .is_none()); + // tick() consumed the marker before try_begin could skip the work. + assert_eq!(timer.tick().await, RescueTickOrigin::Ordinary); + } + #[test] fn rescue_accelerated_first_batch_requires_owner_evidence() { let mut cadence = RescueHeartbeatCadence::default(); let now = Instant::now(); let interval = Duration::from_secs(30); let cutoff = Utc::now(); - assert_eq!(cadence.select(now, interval, None), Some(None)); assert_eq!( - cadence.select(now + Duration::from_secs(1), interval, Some(cutoff)), + cadence.select(now, interval, None, RescueTickOrigin::Backlog), + Some(None) + ); + assert_eq!( + cadence.select( + now + Duration::from_secs(1), + interval, + Some(cutoff), + RescueTickOrigin::Backlog + ), Some(Some(cutoff)) ); // The preceding accelerated pass re-armed the timer, but owners // became unconfirmed before it fired: no ordinary batch may slip in. assert_eq!( - cadence.select(now + Duration::from_secs(2), interval, None), + cadence.select( + now + Duration::from_secs(2), + interval, + None, + RescueTickOrigin::Backlog + ), None ); - assert_eq!(cadence.select(now + interval, interval, None), Some(None)); + assert_eq!( + cadence.select(now + interval, interval, None, RescueTickOrigin::Backlog), + Some(None) + ); } #[test] @@ -4858,7 +5055,7 @@ mod tests { let now = Instant::now(); let interval = Duration::from_secs(30); let stamp = Utc::now(); - sweep.revisit(now, interval); + sweep.revisit(now, interval, RescueTickOrigin::Backlog); // Slot A's deadlines are in the future; B keeps a large pass open. sweep.observe(RescueScanProgress { slot: 0, @@ -4874,10 +5071,10 @@ mod tests { saturated: true, }], }); - sweep.revisit(now + interval / 2, interval); + sweep.revisit(now + interval / 2, interval, RescueTickOrigin::Backlog); assert_eq!(sweep.idle_slots, vec![0]); // A can now be expired (or contain claims from a reused generation). - sweep.revisit(now + interval, interval); + sweep.revisit(now + interval, interval, RescueTickOrigin::Backlog); assert!(sweep.idle_slots.is_empty()); assert!(!sweep.cursors[&0][0].done); assert_eq!( @@ -5048,7 +5245,12 @@ mod tests { sqlx::query("UPDATE awa.runtime_instances SET last_seen_at=clock_timestamp() WHERE instance_id=$1") .bind(self.fresh_runtime).execute(&self.pool).await.expect("fresh owner snapshot"); self.service - .rescue_queue_storage_pass(RescueKind::Heartbeat, &self.runtime, false) + .rescue_queue_storage_pass( + RescueKind::Heartbeat, + &self.runtime, + false, + RescueTickOrigin::Backlog, + ) .await } @@ -5190,7 +5392,12 @@ mod tests { let mut drained = false; for _ in 0..20 { if !service - .rescue_queue_storage_pass(RescueKind::Deadline, &runtime, false) + .rescue_queue_storage_pass( + RescueKind::Deadline, + &runtime, + false, + RescueTickOrigin::Backlog, + ) .await { drained = true; @@ -5206,7 +5413,12 @@ mod tests { let service = service.rescue_pass_max_rows(0); assert!( !service - .rescue_queue_storage_pass(RescueKind::Deadline, &runtime, false) + .rescue_queue_storage_pass( + RescueKind::Deadline, + &runtime, + false, + RescueTickOrigin::Backlog + ) .await ); assert!(service.rescue_sweeps.lock().expect("sweeps").is_empty()); diff --git a/docs/configuration.md b/docs/configuration.md index 186c650ba..086683010 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -488,7 +488,8 @@ Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pa How a pass chooses and paces its work: - **Oldest claims first.** Batches visit claim slots from the oldest to the current one, because the ring cannot rotate back into the oldest slot until its claims are closed. Every fourth batch visits the other slots first, so a long drain of one slot does not starve the rest. A slot a batch found nothing in is skipped for the rest of the pass. -- **Stale heartbeats need evidence that the owner is gone.** A stale heartbeat cannot tell a dead runtime from a stalled one (a database stall, a long GC pause, a network partition). Stale-heartbeat rescue runs extra batches or early passes only when the runtime snapshots in `awa.runtime_instances` show every runtime either fresh with its heartbeat loop alive, or silent for at least `heartbeat_staleness`, and that has held at every pass for a whole `heartbeat_staleness` window. A live runtime coming out of a stall has had that window to refresh its heartbeats, so its claims are not stale by the time a drain can reach them. Every accelerated batch, including an early pass's first batch, also takes only claims last heartbeated before the most recent of those gone runtimes stopped reporting, so a live runtime whose heartbeat pass merely lags behind `heartbeat_staleness` (one holding many claims under load, say) is not drained in bulk; its claims stay at one batch per ordinary rescue interval, as before. The leader samples the snapshots once per pass, so this evidence needs `heartbeat_rescue_interval` below half of `heartbeat_staleness` (the defaults, `30s` and `90s`, qualify); otherwise stale-heartbeat rescue stays at one batch per pass. After a runtime stops reporting, the default `90s` staleness and `30s` rescue interval usually require about 3–3.5 minutes before acceleration: the runtime must first qualify as gone, then evidence must remain settled for a full staleness window. A new maintenance leader starts that evidence window again. The per-attempt staleness rule itself is unchanged. Deadline and callback-timeout rescues act on a deadline that has passed, so they drain without this evidence. +- **Stale-heartbeat acceleration uses runtime evidence.** A stale heartbeat cannot distinguish a dead runtime from a stalled one. Extra batches and early passes require every snapshot in `awa.runtime_instances` to be either fresh with its heartbeat loop alive or silent for at least `heartbeat_staleness`, with that condition observed for a full staleness window. Accelerated batches also require the claim's last heartbeat to precede the latest gone-runtime cutoff. The per-attempt staleness condition is unchanged. + This is a bounded heuristic, not proof that an owner has exited. To expose an unconfirmed phase between fresh and gone, each runtime needs `3 × max(runtime_snapshot_interval, 1s) < heartbeat_staleness`. With 5-second snapshots and 12-second staleness, a stalled live runtime goes directly from fresh to gone; steady evidence may predate the stall, so acceleration can rescue its stale claims faster than the ordinary one-batch cadence. The leader must also sample frequently enough: gaps longer than half the staleness reset the evidence window, and short stalls between samples can be missed. The defaults (10-second snapshots, 90-second staleness, 30-second rescue interval) provide an unconfirmed phase; after a runtime stops reporting, acceleration usually begins after about 3–3.5 minutes. A new maintenance leader resets the evidence window. Deadline and callback rescues use their expired deadline and do not need owner evidence. - **Retries retain their backoff.** Rescue bounds closures and retry writes per pass. Backoffs and promotion delays can make retries become due together; `rescue_pass_max_rows` does not cap the rate at which promotion admits them to the ready lanes. - **Exhausted attempts are dead-lettered in the same batches.** On a DLQ-enabled queue an attempt rescued on its last try goes to the DLQ in its batch's transaction (reason `heartbeat_stale`, `deadline_exceeded` or `callback_timeout`), so a drain can move many jobs to the DLQ at once. `awa.job.dlq_moved` counts them by reason. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cc1282833..7db6c337e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -306,7 +306,7 @@ FROM awa.runtime_instances ORDER BY last_seen_at DESC; ``` -A runtime that has stopped reporting but has not yet been silent for `heartbeat_staleness`, or a reporting runtime with `heartbeat_alive = false`, holds the drain back until it is either gone or healthy. The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. With the defaults, acceleration usually starts about 3–3.5 minutes after the last runtime snapshot, and a leadership change restarts the evidence window. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. +A snapshot that is no longer fresh but is younger than `heartbeat_staleness` holds acceleration back, as does a fresh snapshot with `heartbeat_alive = false`. The unconfirmed phase requires `3 × max(runtime_snapshot_interval, 1s) < heartbeat_staleness` for every runtime; otherwise a stalled live runtime can pass directly from fresh to gone ([owner-evidence limits](configuration.md#rescue-backlog)). The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. With the defaults, acceleration usually starts about 3–3.5 minutes after the last runtime snapshot, and a leadership change restarts the evidence window. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. To drain faster, raise `rescue_pass_max_rows` and `rescue_pass_budget`, keeping in mind that a pass holds the maintenance leader for its budget and can exceed `rescue_pass_max_rows` by one final batch of closures and retry rows. Retry due times can cluster; this cap does not bound promotion throughput. From fe00551dd5b052bbd13650a17a559ac1524d8ca1 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 14:22:53 +1300 Subject: [PATCH 13/22] refactor(rescue): drop the unused retry_not_before batch option Rescued retries keep their ordinary backoff, and nothing sets the floor any more, so the option and its plumbing through settlement go. The callback batch keeps its options parameter for symmetry with the other kinds; none of them apply to callback waits. --- awa-model/src/queue_storage.rs | 19 +++++-------------- awa-worker/src/maintenance.rs | 1 - 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 451c62c94..4a03f20ea 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -398,9 +398,6 @@ pub struct RescueBatchOptions { /// Claim slots to leave out, typically ones an earlier batch of the same /// pass found nothing in ([`RescueBatch::idle_slots`]). pub skip_slots: Vec, - /// Earliest `run_at` for a rescued attempt that goes back to `retryable`. - /// Its backoff still applies when that is later. - pub retry_not_before: Option>, /// How many claims past its cursor each slot's stale-heartbeat or /// row-local deadline walk examines (default 10,000). Every claim /// examined costs its closure checks, so a batch that follows one that @@ -15168,7 +15165,6 @@ impl QueueStorage { rows: Vec, error: &str, dlq: RescueDlq<'_>, - retry_not_before: Option>, ) -> Result, AwaError> { if rows.is_empty() { return Ok(Vec::new()); @@ -15196,10 +15192,9 @@ impl QueueStorage { let exhausted = row.attempt >= row.max_attempts; payload.push_error(lifecycle_error(error, row.attempt, exhausted)); if !exhausted { - let backoff_at = retry_run_ats.next().ok_or_else(|| { + let run_at = retry_run_ats.next().ok_or_else(|| { AwaError::Validation("rescue backoff missing for a retryable attempt".into()) })?; - let run_at = retry_not_before.map_or(backoff_at, |floor| floor.max(backoff_at)); let deferred = row.clone().into_deferred_row( JobState::Retryable, run_at, @@ -16300,7 +16295,6 @@ impl QueueStorage { reason: "heartbeat_stale", enabled: dlq_enabled, }, - options.retry_not_before, ) .await?; rescued.extend( @@ -16312,7 +16306,6 @@ impl QueueStorage { reason: "heartbeat_stale", enabled: dlq_enabled, }, - options.retry_not_before, ) .await?, ); @@ -16504,7 +16497,6 @@ impl QueueStorage { reason: "deadline_exceeded", enabled: dlq_enabled, }, - options.retry_not_before, ) .await?; attempt.commit().await.map_err(map_sqlx_error)?; @@ -16541,17 +16533,17 @@ impl QueueStorage { } /// One callback-timeout rescue batch in its own transaction: up to - /// 500 attempts. `slot_order` does not apply; - /// callback waits live in `leases`, not in the claim ring. + /// 500 attempts. The options do not apply: callback waits live in + /// `leases`, not in the claim ring. #[tracing::instrument( - skip(self, pool, dlq_enabled, options), + skip(self, pool, dlq_enabled, _options), name = "queue_storage.rescue_expired_callbacks" )] pub async fn rescue_expired_callbacks_batch( &self, pool: &PgPool, dlq_enabled: &(dyn Fn(&str) -> bool + Sync), - options: &RescueBatchOptions, + _options: &RescueBatchOptions, ) -> Result { let schema = self.schema(); let mut tx = pool.begin().await.map_err(map_sqlx_error)?; @@ -16650,7 +16642,6 @@ impl QueueStorage { reason: "callback_timeout", enabled: dlq_enabled, }, - options.retry_not_before, ) .await?; attempt.commit().await.map_err(map_sqlx_error)?; diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index 89bc9f78e..00d8c5878 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -2195,7 +2195,6 @@ impl MaintenanceService { let options = RescueBatchOptions { slot_order, skip_slots: sweep.idle_slots.clone(), - retry_not_before: None, cursor_scan_limit: (batches > 0 || (kind == RescueKind::Heartbeat && bulk_heartbeat_cutoff.is_some() From b9fc504c931c6382c9f693165e7999117ecdfb71 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 14:22:53 +1300 Subject: [PATCH 14/22] perf(rescue): give deadline passes the narrow first cursor window A deadline pass that may take several batches now starts with the 2,000-claim window that follow-up batches and confirmed stale-heartbeat passes already use, instead of 10,000. Every claim in the window pays its closure checks, so on a dense backlog the wide first window spent most of a batch evaluating claims it could not take: a 10,000-claim deadline batch of row-local claims took about 680 ms of statement time against about 220 ms for a 2,000-claim one. The pass keeps taking batches while windows come back full, so long closed prefixes are still walked; a pass limited to one batch keeps the wide window. Against 200,000 abandoned row-local claims with default settings the drain went from about 820 to about 1,020 claims a second, and the slowest deadline statement from 605 ms to 221 ms. --- awa-worker/src/maintenance.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/awa-worker/src/maintenance.rs b/awa-worker/src/maintenance.rs index 00d8c5878..ed5cad751 100644 --- a/awa-worker/src/maintenance.rs +++ b/awa-worker/src/maintenance.rs @@ -243,9 +243,12 @@ pub(crate) const DEFAULT_RESCUE_PASS_MAX_ROWS: usize = 5_000; /// a drain to about `rescue_pass_max_rows` attempts per second per kind. const RESCUE_BACKLOG_INTERVAL: Duration = Duration::from_secs(1); -/// Claims each slot's cursor walk examines during bulk heartbeat rescue and -/// follow-up batches. Other first batches keep the store's wider default to -/// walk past long closed prefixes when acceleration is unavailable or disabled. +/// Claims each slot's cursor walk examines in every batch of a pass that may +/// take more than one: deadline rescue, and stale-heartbeat rescue once its +/// owners are confirmed gone. Every claim examined costs its closure checks, +/// so a narrow window keeps each batch's transaction short; the pass continues +/// while windows come back full. A pass limited to one batch keeps the store's +/// wider default so it can walk past a long closed prefix. const RESCUE_FOLLOW_UP_SCAN_LIMIT: usize = 2_000; /// One batch in this many visits the other claim slots before the oldest, so @@ -2196,11 +2199,14 @@ impl MaintenanceService { slot_order, skip_slots: sweep.idle_slots.clone(), cursor_scan_limit: (batches > 0 - || (kind == RescueKind::Heartbeat - && bulk_heartbeat_cutoff.is_some() - && !budget.is_zero() - && self.rescue_pass_max_rows > 0)) - .then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), + || (!budget.is_zero() + && self.rescue_pass_max_rows > 0 + && match kind { + RescueKind::Heartbeat => bulk_heartbeat_cutoff.is_some(), + RescueKind::Deadline => true, + RescueKind::Callback => false, + })) + .then_some(RESCUE_FOLLOW_UP_SCAN_LIMIT), }; let outcome = match kind { RescueKind::Heartbeat => { From 9366c34c1e0b1161e039c140b84e50c0cb294e22 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 14:22:53 +1300 Subject: [PATCH 15/22] test(rescue): drain abandoned row-local claims that pin both rings A killed runtime that held claims with deadlines leaves row-local lease_claims rows, not compact batches. The new test builds that shape directly: a backlog over two claim slots that all reference one queue slot, claimed, heartbeated and past their deadlines hours ago, with the dead runtime's snapshot gone. Until the claims close, queue prune skips that queue slot and claim prune skips both claim slots. With default rescue intervals the test requires every claim to close, both rings to rotate past the pinned slots and every rescued job to complete on its second attempt. It runs 20,000 claims by default. Environment variables scale the backlog, override the pass controls, or stop after a fixed time and report the drain rate, which is how the larger measurements were taken. --- awa/tests/bulk_rescue_test.rs | 235 ++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index 7f3c4b621..eed2ac3ab 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -803,3 +803,238 @@ async fn rescue_drains_an_abandoned_backlog_without_rescuing_live_claims() { live_runs.len() ); } + +const ROW_LOCAL_QUEUE: &str = "bulk_rescue_row_local"; + +fn env_u64(name: &str) -> Option { + std::env::var(name).ok().map(|value| { + value + .parse() + .unwrap_or_else(|err| panic!("{name}={value}: {err}")) + }) +} + +async fn ring_generation(pool: &PgPool, ring: &str) -> i64 { + count( + pool, + format!("SELECT generation FROM awa.{ring}_ring_state WHERE singleton"), + ) + .await +} + +/// First-attempt claims still open: a dead runtime's attempts that no rescue +/// has closed. A pruned claim slot takes its closed claims with it. +async fn open_first_attempt_claims(pool: &PgPool) -> i64 { + count( + pool, + format!( + "SELECT count(*) FROM awa.lease_claims + WHERE queue = '{ROW_LOCAL_QUEUE}' AND run_lease = 1 AND closed_at IS NULL" + ), + ) + .await +} + +/// A killed runtime left hundreds of thousands of row-local claims: claims +/// with a deadline, so each has its own `lease_claims` row rather than a +/// compact batch. They span two claim slots, all reference one queue slot, +/// were last heartbeated hours ago and are past their deadlines. Until they +/// close, queue prune skips that queue slot and claim prune skips both claim +/// slots, so neither ring can wrap. With default rescue intervals, rescue +/// must close every claim and both rings must move past the pinned slots. +/// +/// `AWA_ROW_LOCAL_RESCUE_CLAIMS` sets the backlog (default 20,000); +/// `AWA_ROW_LOCAL_RESCUE_PASS_BUDGET_MS` and `AWA_ROW_LOCAL_RESCUE_PASS_MAX_ROWS` +/// override the pass controls, and `AWA_ROW_LOCAL_RESCUE_OBSERVE_SECS` reports +/// the drain rate after that long instead of requiring a full drain. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_test_writer() + .try_init(); + let abandoned = env_u64("AWA_ROW_LOCAL_RESCUE_CLAIMS").unwrap_or(20_000) as i64; + let observe_for = env_u64("AWA_ROW_LOCAL_RESCUE_OBSERVE_SECS").map(Duration::from_secs); + + let db = BulkRescueDatabase::create().await; + let pool = db.pool().clone(); + let store = QueueStorage::new(QueueStorageConfig::default()).expect("queue storage"); + let queue_slots = store.queue_slot_count() as i64; + let claim_slots = store.claim_slot_count() as i64; + + let setup_started = Instant::now(); + for first in (0..abandoned).step_by(5_000) { + let params: Vec<_> = (first..(first + 5_000).min(abandoned)) + .map(|n| { + awa::model::insert::params_with( + &AbandonedJob { n }, + InsertOpts { + queue: ROW_LOCAL_QUEUE.into(), + ..Default::default() + }, + ) + .expect("abandoned params") + }) + .collect(); + store + .enqueue_params_batch(&pool, ¶ms) + .await + .expect("enqueue abandoned jobs"); + } + // Half the backlog in each of two claim slots, as a runtime that kept + // claiming across a claim rotation leaves it. + let mut claimed = 0; + let mut rotated = false; + while claimed < abandoned { + if !rotated && claimed >= abandoned / 2 { + store.rotate_claims(&pool).await.expect("rotate claims"); + rotated = true; + } + let batch = store + .claim_runtime_batch(&pool, ROW_LOCAL_QUEUE, 5_000, Duration::from_secs(300)) + .await + .expect("claim abandoned jobs"); + assert!(!batch.is_empty(), "the backlog should still be claimable"); + claimed += batch.len() as i64; + } + // The runtime died hours ago: its claims, deadlines and last heartbeats + // all predate the staleness window, and its snapshot stopped then too. + sqlx::raw_sql(&format!( + "UPDATE awa.lease_claims + SET claimed_at = claimed_at - interval '20 hours', + deadline_at = deadline_at - interval '20 hours' + WHERE queue = '{ROW_LOCAL_QUEUE}'; + INSERT INTO awa.attempt_state (job_id, run_lease, heartbeat_at, updated_at) + SELECT job_id, run_lease, claimed_at + interval '1 minute', claimed_at + interval '1 minute' + FROM awa.lease_claims WHERE queue = '{ROW_LOCAL_QUEUE}' + ON CONFLICT (job_id, run_lease) DO UPDATE SET heartbeat_at = EXCLUDED.heartbeat_at; + INSERT INTO awa.runtime_instances ( + instance_id, pid, version, started_at, last_seen_at, snapshot_interval_ms, + healthy, postgres_connected, poll_loop_alive, heartbeat_alive, + maintenance_alive, shutting_down, leader, storage_capability, transition_role + ) VALUES ( + gen_random_uuid(), 1, 'killed', now() - interval '21 hours', + now() - interval '19 hours', 10000, true, true, true, true, true, false, false, + 'queue_storage', 'queue_storage_target' + ); + ANALYZE awa.lease_claims; + ANALYZE awa.attempt_state;" + )) + .execute(&pool) + .await + .expect("age the abandoned claims"); + let shape: (i64, i64, i64, i64) = sqlx::query_as(&format!( + "SELECT count(*), count(DISTINCT claim_slot), count(DISTINCT ready_slot), + (SELECT count(*) FROM awa.lease_claim_batches) + FROM awa.lease_claims WHERE queue = '{ROW_LOCAL_QUEUE}'" + )) + .fetch_one(&pool) + .await + .expect("backlog shape"); + assert_eq!( + shape, + (abandoned, 2, 1, 0), + "row-local claims over two claim slots and one queue slot" + ); + let queue_generation = ring_generation(&pool, "queue").await; + let claim_generation = ring_generation(&pool, "claim").await; + eprintln!( + "{abandoned} abandoned row-local claims built in {:?}", + setup_started.elapsed() + ); + + let mut builder = Client::builder(pool.clone()) + .queue_storage( + QueueStorageConfig::default(), + Duration::from_secs(1), + Duration::from_secs(1), + ) + .claim_rotate_interval(Duration::from_secs(1)) + .queue( + ROW_LOCAL_QUEUE, + QueueConfig { + max_workers: 200, + ..QueueConfig::default() + }, + ) + .register::(|_, _| async { Ok(JobResult::Completed) }); + if let Some(budget) = env_u64("AWA_ROW_LOCAL_RESCUE_PASS_BUDGET_MS") { + builder = builder.rescue_pass_budget(Duration::from_millis(budget)); + } + if let Some(max_rows) = env_u64("AWA_ROW_LOCAL_RESCUE_PASS_MAX_ROWS") { + builder = builder.rescue_pass_max_rows(max_rows as usize); + } + let client = builder.build().expect("client"); + let started = Instant::now(); + client.start().await.expect("start client"); + + let mut first_rescue = None; + let mut drained = None; + loop { + let open = open_first_attempt_claims(&pool).await; + if open < abandoned && first_rescue.is_none() { + first_rescue = Some(started.elapsed()); + } + if open == 0 { + drained = Some(started.elapsed()); + break; + } + if let Some(observe_for) = observe_for { + if started.elapsed() >= observe_for { + let rescued = abandoned - open; + let since_first = started.elapsed() - first_rescue.unwrap_or_default(); + eprintln!( + "observed {rescued} of {abandoned} rescued in {:?} ({:?} after the first \ + rescue): {:.1} claims/s", + started.elapsed(), + since_first, + rescued as f64 / since_first.as_secs_f64().max(f64::EPSILON) + ); + break; + } + } + assert!( + started.elapsed() < scaled_timeout(Duration::from_secs(600 + abandoned as u64 / 200)), + "{open} of {abandoned} abandoned claims are still open" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + } + + if let Some(drained) = drained { + let first_rescue = first_rescue.unwrap_or_default(); + eprintln!( + "first rescue {first_rescue:?} after start; all {abandoned} closed {drained:?} after \ + start: {:.1} claims/s from the first rescue", + abandoned as f64 / (drained - first_rescue).as_secs_f64().max(f64::EPSILON) + ); + let unpinned = wait_until( + scaled_timeout(Duration::from_secs(120)), + "both rings to rotate past the slots the backlog pinned", + || async { + ring_generation(&pool, "queue").await >= queue_generation + queue_slots + && ring_generation(&pool, "claim").await >= claim_generation + claim_slots + }, + ) + .await; + eprintln!( + "queue and claim rings rotated past the pinned slots {unpinned:?} after the drain" + ); + wait_until( + scaled_timeout(Duration::from_secs(300 + abandoned as u64 / 500)), + "every rescued job to run on its second attempt", + || async { + count( + &pool, + format!( + "SELECT count(*) FROM awa.terminal_jobs + WHERE queue = '{ROW_LOCAL_QUEUE}' AND state = 'completed'" + ), + ) + .await + == abandoned + }, + ) + .await; + } + client.shutdown(Duration::from_secs(10)).await; +} From 3b0b3844a8bac2147efc4a29b1816dc8c0ad832a Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 14:26:44 +1300 Subject: [PATCH 16/22] test(worker): count rescue select arms that bind their tick origin The scheduler registration check counted only arms written as `_ = timer.tick()`, so the three rescue arms, which bind the tick origin, dropped out of the count after the merge with the prune controls. --- awa-worker/src/runtime_control.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awa-worker/src/runtime_control.rs b/awa-worker/src/runtime_control.rs index 9004dae2d..7fd7560ff 100644 --- a/awa-worker/src/runtime_control.rs +++ b/awa-worker/src/runtime_control.rs @@ -970,7 +970,10 @@ mod tests { .next() .unwrap(); // Every select arm except cancellation must register and gate its work. - let selected = leader.matches("\n _ = ").count(); + let selected = leader.matches("\n _ = ").count() + + leader + .matches("\n origin = ") + .count(); assert_eq!( selected, leader.matches("branch_tracker.try_begin").count() + 2 From 59235491ea9ec4c911413804e20c199c86e9e817 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 15:15:32 +1300 Subject: [PATCH 17/22] perf(rescue): seek receipt rescue cursors through the index The heartbeat and deadline receipt scans compared each claim with the slot's cursor through a cross join on the locked cursor row, so the comparison ran as a join filter. Every scan walked the cursor index from its start and discarded every claim the sweep had already passed: with 790,000 claims in a slot, each batch read and dropped them all twice, once per statement. A drain slowed as it progressed, from about 1,000 claims a second to about 440, and 1.6 million abandoned row-local claims took 48 minutes. The cursor values now come from scalar subqueries on the same locked row, which the planner evaluates once and uses as index conditions, so a scan seeks straight to the cursor. On a slot with 294,000 claims behind the cursor, the deadline candidate scan dropped from 190 ms to 33 ms. The cursor row is still locked by the same statements and an empty cursor row still yields no candidates. --- awa-model/src/queue_storage.rs | 56 +++++++++++++++------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 4a03f20ea..94f84b6e0 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -12041,15 +12041,14 @@ impl QueueStorage { SELECT claim_batches.claimed_at, sum(claim_batches.claimed_count) FILTER ( - WHERE claim_batches.claimed_at > cursor_row.rescue_cursor_claimed_at + WHERE claim_batches.claimed_at > (SELECT rescue_cursor_claimed_at FROM cursor_row) ) OVER ( ORDER BY claim_batches.claimed_at, claim_batches.batch_id ) AS covered FROM {claim_batch_child} AS claim_batches - CROSS JOIN cursor_row WHERE claim_batches.claim_slot = $1 AND claim_batches.claimed_at < $2 - AND claim_batches.claimed_at >= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at >= (SELECT rescue_cursor_claimed_at FROM cursor_row) ) AS running WHERE running.covered >= $3 ), @@ -12073,14 +12072,13 @@ impl QueueStorage { claims.closed_at, false AS compact_batch FROM {claim_child} AS claims - CROSS JOIN cursor_row WHERE claims.claim_slot = $1 AND claims.claimed_at < $2 AND (claims.claimed_at, claims.job_id, claims.run_lease) > ( - cursor_row.rescue_cursor_claimed_at, - cursor_row.rescue_cursor_job_id, - cursor_row.rescue_cursor_run_lease + (SELECT rescue_cursor_claimed_at FROM cursor_row), + (SELECT rescue_cursor_job_id FROM cursor_row), + (SELECT rescue_cursor_run_lease FROM cursor_row) ) ORDER BY claims.claimed_at, claims.job_id, claims.run_lease LIMIT $3 @@ -12104,7 +12102,6 @@ impl QueueStorage { NULL::timestamptz AS closed_at, true AS compact_batch FROM {claim_batch_child} AS claim_batches - CROSS JOIN cursor_row CROSS JOIN after_batch_bound CROSS JOIN LATERAL unnest( claim_batches.job_ids, @@ -12116,14 +12113,14 @@ impl QueueStorage { ) AS items(job_id, run_lease, receipt_id, lane_seq, attempt, max_attempts) WHERE claim_batches.claim_slot = $1 AND claim_batches.claimed_at < $2 - AND claim_batches.claimed_at >= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at >= (SELECT rescue_cursor_claimed_at FROM cursor_row) AND claim_batches.claimed_at <= COALESCE(after_batch_bound.claimed_at, 'infinity'::timestamptz) AND (claim_batches.claimed_at, items.job_id, items.run_lease) > ( - cursor_row.rescue_cursor_claimed_at, - cursor_row.rescue_cursor_job_id, - cursor_row.rescue_cursor_run_lease + (SELECT rescue_cursor_claimed_at FROM cursor_row), + (SELECT rescue_cursor_job_id FROM cursor_row), + (SELECT rescue_cursor_run_lease FROM cursor_row) ) ), after_cursor AS MATERIALIZED ( @@ -12145,15 +12142,14 @@ impl QueueStorage { SELECT claim_batches.claimed_at, sum(claim_batches.claimed_count) FILTER ( - WHERE claim_batches.claimed_at < cursor_row.rescue_cursor_claimed_at + WHERE claim_batches.claimed_at < (SELECT rescue_cursor_claimed_at FROM cursor_row) ) OVER ( ORDER BY claim_batches.claimed_at, claim_batches.batch_id ) AS covered FROM {claim_batch_child} AS claim_batches - CROSS JOIN cursor_row WHERE claim_batches.claim_slot = $1 AND claim_batches.claimed_at < $2 - AND claim_batches.claimed_at <= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at <= (SELECT rescue_cursor_claimed_at FROM cursor_row) ) AS running WHERE running.covered >= $3 ), @@ -12177,15 +12173,14 @@ impl QueueStorage { claims.closed_at, false AS compact_batch FROM {claim_child} AS claims - CROSS JOIN cursor_row WHERE (SELECT count FROM after_count) < $3 AND claims.claim_slot = $1 AND claims.claimed_at < $2 AND (claims.claimed_at, claims.job_id, claims.run_lease) <= ( - cursor_row.rescue_cursor_claimed_at, - cursor_row.rescue_cursor_job_id, - cursor_row.rescue_cursor_run_lease + (SELECT rescue_cursor_claimed_at FROM cursor_row), + (SELECT rescue_cursor_job_id FROM cursor_row), + (SELECT rescue_cursor_run_lease FROM cursor_row) ) ORDER BY claims.claimed_at, claims.job_id, claims.run_lease LIMIT $3 @@ -12209,7 +12204,6 @@ impl QueueStorage { NULL::timestamptz AS closed_at, true AS compact_batch FROM {claim_batch_child} AS claim_batches - CROSS JOIN cursor_row CROSS JOIN before_batch_bound CROSS JOIN LATERAL unnest( claim_batches.job_ids, @@ -12222,14 +12216,14 @@ impl QueueStorage { WHERE (SELECT count FROM after_count) < $3 AND claim_batches.claim_slot = $1 AND claim_batches.claimed_at < $2 - AND claim_batches.claimed_at <= cursor_row.rescue_cursor_claimed_at + AND claim_batches.claimed_at <= (SELECT rescue_cursor_claimed_at FROM cursor_row) AND claim_batches.claimed_at <= COALESCE(before_batch_bound.claimed_at, 'infinity'::timestamptz) AND (claim_batches.claimed_at, items.job_id, items.run_lease) <= ( - cursor_row.rescue_cursor_claimed_at, - cursor_row.rescue_cursor_job_id, - cursor_row.rescue_cursor_run_lease + (SELECT rescue_cursor_claimed_at FROM cursor_row), + (SELECT rescue_cursor_job_id FROM cursor_row), + (SELECT rescue_cursor_run_lease FROM cursor_row) ) ), before_cursor AS MATERIALIZED ( @@ -12756,15 +12750,14 @@ impl QueueStorage { ORDER BY claims.deadline_at, claims.job_id, claims.run_lease ) AS rn FROM {claim_child} AS claims - CROSS JOIN cursor_row WHERE claims.claim_slot = $1 AND claims.deadline_at IS NOT NULL AND claims.deadline_at < clock_timestamp() AND (claims.deadline_at, claims.job_id, claims.run_lease) > ( - cursor_row.deadline_cursor_deadline_at, - cursor_row.deadline_cursor_job_id, - cursor_row.deadline_cursor_run_lease + (SELECT deadline_cursor_deadline_at FROM cursor_row), + (SELECT deadline_cursor_job_id FROM cursor_row), + (SELECT deadline_cursor_run_lease FROM cursor_row) ) ORDER BY claims.deadline_at, claims.job_id, claims.run_lease LIMIT $2 @@ -12781,7 +12774,6 @@ impl QueueStorage { ORDER BY claims.deadline_at, claims.job_id, claims.run_lease ) AS rn FROM {claim_child} AS claims - CROSS JOIN cursor_row CROSS JOIN after_count WHERE after_count.count < $2 AND claims.claim_slot = $1 @@ -12789,9 +12781,9 @@ impl QueueStorage { AND claims.deadline_at < clock_timestamp() AND (claims.deadline_at, claims.job_id, claims.run_lease) <= ( - cursor_row.deadline_cursor_deadline_at, - cursor_row.deadline_cursor_job_id, - cursor_row.deadline_cursor_run_lease + (SELECT deadline_cursor_deadline_at FROM cursor_row), + (SELECT deadline_cursor_job_id FROM cursor_row), + (SELECT deadline_cursor_run_lease FROM cursor_row) ) ORDER BY claims.deadline_at, claims.job_id, claims.run_lease LIMIT (SELECT GREATEST($2 - count, 0) FROM after_count) From e6cac7a79fd819d80e9cbb95d5d7895d6b0d12cf Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 15:56:13 +1300 Subject: [PATCH 18/22] test(rescue): count row-local recovery runs in the handler Queue prune removes completed rows once the ring wraps past them, which a drain of a million claims does long before every recovery run finishes, so the completed-row count never reached the backlog. The handler now counts second-attempt runs and any other attempt, and the test requires exactly one second-attempt run per abandoned job and no others. --- awa/tests/bulk_rescue_test.rs | 36 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index eed2ac3ab..182d7b533 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -943,6 +943,10 @@ async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { setup_started.elapsed() ); + // Completed rows leave the ring once queue prune wraps past them, so the + // handler counts the recovery runs. + let second_attempts = Arc::new(AtomicU64::new(0)); + let other_attempts = Arc::new(AtomicU64::new(0)); let mut builder = Client::builder(pool.clone()) .queue_storage( QueueStorageConfig::default(), @@ -957,7 +961,19 @@ async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { ..QueueConfig::default() }, ) - .register::(|_, _| async { Ok(JobResult::Completed) }); + .register::({ + let second_attempts = second_attempts.clone(); + let other_attempts = other_attempts.clone(); + move |_, ctx| { + let runs = if ctx.job.attempt == 2 { + &second_attempts + } else { + &other_attempts + }; + runs.fetch_add(1, Ordering::SeqCst); + async { Ok(JobResult::Completed) } + } + }); if let Some(budget) = env_u64("AWA_ROW_LOCAL_RESCUE_PASS_BUDGET_MS") { builder = builder.rescue_pass_budget(Duration::from_millis(budget)); } @@ -1022,19 +1038,15 @@ async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { wait_until( scaled_timeout(Duration::from_secs(300 + abandoned as u64 / 500)), "every rescued job to run on its second attempt", - || async { - count( - &pool, - format!( - "SELECT count(*) FROM awa.terminal_jobs - WHERE queue = '{ROW_LOCAL_QUEUE}' AND state = 'completed'" - ), - ) - .await - == abandoned - }, + || async { second_attempts.load(Ordering::SeqCst) >= abandoned as u64 }, ) .await; + assert_eq!(second_attempts.load(Ordering::SeqCst), abandoned as u64); + assert_eq!( + other_attempts.load(Ordering::SeqCst), + 0, + "only the abandoned first attempts are rescued" + ); } client.shutdown(Duration::from_secs(10)).await; } From 2c85dc0f4329c91e521bb1e52f2daf8d7a4eed5e Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 16:05:14 +1300 Subject: [PATCH 19/22] perf(rescue): run rescue batches without JIT The heartbeat and deadline cursor scans are estimated far above their cost, enough to cross PostgreSQL's JIT thresholds. With a 10,000-claim window the stale-heartbeat lock statement executed in about 130 ms but spent 1.6 s compiling, and its closing statement about 3.5 s, so each batch outside owner-confirmed bulk rescue held its cursor row and attempt locks for 4 to 10 seconds against a 200,000 to 1.6 million claim backlog. Each rescue batch transaction now sets jit off locally, as queue drains already do. --- awa-model/src/queue_storage.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 94f84b6e0..96cd6000b 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -31,6 +31,9 @@ const DRAIN_BATCH_SIZE: i64 = 1_000; // which takes seconds against milliseconds of execution. const DRAIN_TRANSACTION_BEGIN: &str = "BEGIN; SET LOCAL jit = off"; const DRAIN_CLAIM_CHECK_ROWS: i64 = 64; +// Rescue cursor scans are estimated far above their real cost, enough to +// JIT-compile them, which takes seconds against about 100 ms of execution. +const RESCUE_TRANSACTION_BEGIN: &str = "BEGIN; SET LOCAL jit = off"; const RECEIPT_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const RECEIPT_DEADLINE_RESCUE_CURSOR_SCAN_LIMIT: i64 = 10_000; const DEFAULT_PRUNE_LOCK_TIMEOUT: Duration = Duration::from_millis(10); @@ -16118,7 +16121,10 @@ impl QueueStorage { options: &RescueBatchOptions, ) -> Result { let schema = self.schema(); - let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let mut tx = pool + .begin_with(RESCUE_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; let cutoff = Utc::now() - TimeDelta::from_std(staleness) .map_err(|err| AwaError::Validation(format!("invalid staleness: {err}")))?; @@ -16347,7 +16353,10 @@ impl QueueStorage { options: &RescueBatchOptions, ) -> Result { let schema = self.schema(); - let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let mut tx = pool + .begin_with(RESCUE_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; let mut excluded = BTreeSet::new(); let mut selected: Option> = None; loop { @@ -16538,7 +16547,10 @@ impl QueueStorage { _options: &RescueBatchOptions, ) -> Result { let schema = self.schema(); - let mut tx = pool.begin().await.map_err(map_sqlx_error)?; + let mut tx = pool + .begin_with(RESCUE_TRANSACTION_BEGIN) + .await + .map_err(map_sqlx_error)?; let mut excluded = BTreeSet::new(); let mut selected: Option> = None; loop { From 3584f7441b73a8e5cd9de1d24c1e9ba9776d140f Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 16:05:14 +1300 Subject: [PATCH 20/22] test(rescue): analyze ready rows before the row-local drain A long-lived backlog has settled statistics. Without them, rescue's ready-body lookups of the freshly built backlog chose the primary key over the job index and took seconds per batch until autovacuum analyzed the partition. --- awa/tests/bulk_rescue_test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index 182d7b533..978838ab3 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -917,6 +917,7 @@ async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { now() - interval '19 hours', 10000, true, true, true, true, true, false, false, 'queue_storage', 'queue_storage_target' ); + ANALYZE awa.ready_entries; ANALYZE awa.lease_claims; ANALYZE awa.attempt_state;" )) From a6e901aae2069b586df62057a1d7dcfd6f527ede Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 16:58:26 +1300 Subject: [PATCH 21/22] test(rescue): scale the ring wait with the row-local backlog After 1.6 million claims drain, promotion admits their retries for a while and the rings took about two minutes to move past the pinned slots, beyond the fixed two-minute wait. --- awa/tests/bulk_rescue_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awa/tests/bulk_rescue_test.rs b/awa/tests/bulk_rescue_test.rs index 978838ab3..2fc103c02 100644 --- a/awa/tests/bulk_rescue_test.rs +++ b/awa/tests/bulk_rescue_test.rs @@ -1025,7 +1025,7 @@ async fn rescue_drains_abandoned_row_local_claims_and_unpins_both_rings() { abandoned as f64 / (drained - first_rescue).as_secs_f64().max(f64::EPSILON) ); let unpinned = wait_until( - scaled_timeout(Duration::from_secs(120)), + scaled_timeout(Duration::from_secs(120 + abandoned as u64 / 10_000)), "both rings to rotate past the slots the backlog pinned", || async { ring_generation(&pool, "queue").await >= queue_generation + queue_slots From 4e489e61ac7a71ee7ca219a4e119f9b4fa980740 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 9 Nov 2026 16:58:26 +1300 Subject: [PATCH 22/22] docs: describe bounded rescue drains for 0.6.16 The unreleased changelog gains the drain, its controls and metrics, and the heartbeat fixes this branch carries; the entries it held for fixes since released in 0.6.14 and 0.6.15 are gone with the merge. The configuration guide gives measured rates and costs for a row-local backlog, the troubleshooting entry names the queue-ring symptom and the budget cap, and the 0.6.15 to 0.6.16 upgrade guide has a self-contained section on what changes for a fleet with a backlog. --- CHANGELOG.md | 3 +++ docs/configuration.md | 2 ++ docs/troubleshooting.md | 3 ++- docs/upgrade-0.6.15-to-0.6.16.md | 13 +++++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6fbeb9c0..6746b58ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,13 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Added +- `ClientBuilder::rescue_pass_budget` and `rescue_pass_max_rows` (Python `rescue_pass_budget_ms` and `rescue_pass_max_rows`) control how long and how far one queue-storage rescue pass drains. `awa.maintenance.rescue.pass_size` (by `awa.rescue.kind` and `awa.rescue.pass_end`) and `awa.maintenance.rescue.oldest_attempt_age` show a drain in progress. - Durable operator halt/resume for `maintenance.prune`, in drain mode. Compatible leaders observe it on their existing session; dispatch, cron, promotion, rescue and enqueue continue. Status lists live runtimes without supported, recent reports and cannot report the fleet halted while any remain. Older binaries ignore the control. See [runtime controls](docs/runtime-controls.md). ### Fixed +- Queue-storage rescue drains a large backlog of abandoned claims in bounded time. A killed runtime leaves every attempt it claimed open, and until they close, queue prune skips the queue slots they reference (`queue.unclosed_claim_refs`) and claim prune skips their claim slots, so neither ring can wrap. Rescue closed at most 500 attempts per kind per 30-second interval: about 33 a second, so 200,000 abandoned row-local claims took about 100 minutes. A rescue pass now takes 500-attempt batches, each in its own transaction, until a batch comes back short, `rescue_pass_budget` (default 1 s, at most half the rescue interval) is spent, or it has rescued `rescue_pass_max_rows` (default 5,000), and while a backlog remains the next pass runs a second later. Rescue's cursor scans also seek to their cursor through the index instead of rereading every claim already swept, so a batch costs the same at the end of a drain as at the start, and they run without JIT, whose compilation took seconds per statement. With the defaults, 1.6 million abandoned row-local claims past their deadlines drained in about 17 minutes (1,600 a second), and both rings moved past the pinned slots about two minutes later. Deadline and callback rescue act on their expired deadlines; stale-heartbeat rescue takes more than one batch per pass only once runtime snapshots show the claims' owners gone, about 3 to 3.5 minutes after a runtime stops reporting with default settings. Retried attempts keep their backoffs, so their due times can cluster. Zero for either control keeps one batch per interval. See [draining a backlog of abandoned claims](docs/configuration.md#rescue-backlog). +- Heartbeats no longer queue behind a completion. Compact receipt claims are heartbeated by their claim batch and item index, an attempt whose closer holds its lock is skipped rather than waited for, and lease and progress writes skip rows a finalizer has locked and retry them on the next pass. One stalled completion could otherwise hold up a whole heartbeat pass until other attempts went stale and were rescued. A finished attempt stops flushing progress once its handler returns. - `admin::get_job` and the other queue-storage point reads of one job no longer return `JobNotFound` for a failed job while queue prune moves its row out of the slot being pruned. Since 0.6.14 the read used one snapshot, and a prune that committed after that snapshot was taken left the old slot empty and the moved row invisible to it. Prune moves each retained failed row once per ring cycle, and on a new schema prunes slot 0 right after the first rotation, so a read polling for `failed` could hit it within a second of a worker starting. A read that finds nothing now reads once more in a new snapshot. ## [0.6.15] — 2026-11-09 diff --git a/docs/configuration.md b/docs/configuration.md index 906a02939..b6e8938a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -485,6 +485,8 @@ Each stale-heartbeat, deadline or callback-timeout rescue therefore runs as a pa | `ClientBuilder::rescue_pass_budget` (Rust) / `rescue_pass_budget_ms` (Python kwarg) | `1s` | How long one pass keeps starting batches. Capped at half the rescue interval. `Duration::ZERO` / `0` keeps every pass to one batch. | | `ClientBuilder::rescue_pass_max_rows` (Rust) / `rescue_pass_max_rows` (Python kwarg) | `5000` | How many attempts one pass rescues before it stops. Bounds the write burst (closures, retry rows, WAL, replica and CDC traffic), with up to one final batch beyond the cap. `0` keeps one batch per ordinary interval without early re-arms. | +In practice the database sets the rate, not the cap. On PostgreSQL 17, a batch of 500 row-local claims (claims with a deadline) spends about 0.1–0.2 s in its rescue statements and writes about 0.3 MB of WAL for its closures, so a default pass takes several batches before its second of rest. Rescue kinds run one after another on the leader, not in parallel. Measured with default intervals, 1.6 million abandoned row-local claims past their deadlines drained in about 17 minutes (1,600 a second), against about 33 a second (500 per kind per interval) without passes. Raising `rescue_pass_budget` towards its cap and `rescue_pass_max_rows` keeps rescue running for more of each second, but each pass holds the leader for its whole budget, so promotion, rotation and prune wait as long. After a large drain the rings take a minute or two to move past the slots the backlog pinned, while promotion admits the rescued retries. + How a pass chooses and paces its work: - **Oldest claims first.** Batches visit claim slots from the oldest to the current one, because the ring cannot rotate back into the oldest slot until its claims are closed. Every fourth batch visits the other slots first, so a long drain of one slot does not starve the rest. A slot a batch found nothing in is skipped for the rest of the pass. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 22b7ef643..a5111f409 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -325,6 +325,7 @@ If a job really needs `20m`, set a longer deadline for that queue. Symptoms on queue storage, usually after a high-throughput runtime was OOM-killed or lost without a graceful shutdown: - `awa.maintenance.rotate.attempts` for the claim ring reports `skipped_busy` with blocker `claim.rows`, and `awa.ring.generation` for the claim ring stops advancing +- queue prune skips the slot those claims reference with reason `queue.unclosed_claim_refs`, and the queue ring stops wrapping too - the current claim slot keeps growing - jobs the dead runtime held wait well past `heartbeat_staleness` before they run again @@ -343,7 +344,7 @@ ORDER BY last_seen_at DESC; A snapshot that is no longer fresh but is younger than `heartbeat_staleness` holds acceleration back, as does a fresh snapshot with `heartbeat_alive = false`. The unconfirmed phase requires `3 × max(runtime_snapshot_interval, 1s) < heartbeat_staleness` for every runtime; otherwise a stalled live runtime can pass directly from fresh to gone ([owner-evidence limits](configuration.md#rescue-backlog)). The drain also needs `heartbeat_rescue_interval` below half of `heartbeat_staleness`. With the defaults, acceleration usually starts about 3–3.5 minutes after the last runtime snapshot, and a leadership change restarts the evidence window. Deadline rescue is not held back this way, so queues with a `deadline_duration` drain through it regardless. -To drain faster, raise `rescue_pass_max_rows` and `rescue_pass_budget`, keeping in mind that a pass holds the maintenance leader for its budget and can exceed `rescue_pass_max_rows` by one final batch of closures and retry rows. Retry due times can cluster; this cap does not bound promotion throughput. +To drain faster, raise `rescue_pass_budget` (at most half the rescue interval, so 15 s by default) and `rescue_pass_max_rows`, keeping in mind that a pass holds the maintenance leader for its budget and can exceed `rescue_pass_max_rows` by one final batch of closures and retry rows. The [measured rates](configuration.md#rescue-backlog) show what to expect. Retry due times can cluster; this cap does not bound promotion throughput. ## Dead Tuples Growing In Queue Storage diff --git a/docs/upgrade-0.6.15-to-0.6.16.md b/docs/upgrade-0.6.15-to-0.6.16.md index 31c40d89b..96c96d7ef 100644 --- a/docs/upgrade-0.6.15-to-0.6.16.md +++ b/docs/upgrade-0.6.15-to-0.6.16.md @@ -59,6 +59,19 @@ Potentially unsettled reclamation or admin metadata refresh sets `settlement_unk A `halted` status is not proof that PostgreSQL released every lock or that stale hosts are fenced. Runtime credentials can forge observations or delete runtime registrations, so confirmation also depends on trusting those credentials. The operator boundary protects the requested halt, not the reported proof. Inspect backend activity and cancel or fence lingering sessions if lock relief is required. See [runtime controls](runtime-controls.md#read-status) for the status fields and states and [runtime control privileges](security.md#runtime-control-privileges) for the trust boundary. +## Rescue drains abandoned claims in passes + +This part needs no migration and applies once the maintenance leader runs 0.6.16. Queue-storage rescue no longer stops at 500 attempts per kind per interval. Each pass takes 500-attempt batches, each in its own short transaction, for up to `rescue_pass_budget` (default 1 s, at most half the rescue interval) or `rescue_pass_max_rows` (default 5,000) rescued attempts, and while a backlog remains the next pass runs a second later. A runtime killed while holding claims therefore stops pinning the queue and claim rings for hours: with default settings, 1.6 million abandoned row-local claims drained in about 17 minutes, where 0.6.15 needed about 13 hours. + +What changes on a fleet with a backlog: + +- Claims past their deadline drain from the first deadline pass after the leader starts. Stale-heartbeat passes, the only route for claims without a deadline (`deadline_duration` zero), take more than one batch only once runtime snapshots have shown the dead runtime gone for a full staleness window, about 3–3.5 minutes with default settings. +- Rescued jobs return to `retryable` with their ordinary backoff, so they become due in bursts and the workers that run them see the backlog arrive within minutes rather than over hours. On DLQ-enabled queues, exhausted attempts move to the DLQ just as quickly. +- Closures, retry rows and their WAL arrive at the drain rate, about 0.6 KB of closure WAL per claim plus the retry rows, which replicas and CDC consumers see as a burst. +- Each pass holds the maintenance leader for up to its budget plus one batch, so promotion, rotation and prune run between passes. + +`rescue_pass_budget` and `rescue_pass_max_rows` (Python `rescue_pass_budget_ms` and `rescue_pass_max_rows`) tune the drain; zero for either keeps the 0.6.15 cadence of one batch per interval. Watch it with `awa.maintenance.rescue.pass_size` and `awa.maintenance.rescue.oldest_attempt_age`. See [draining a backlog of abandoned claims](configuration.md#rescue-backlog). + ## Roll back Before rolling back to 0.6.15, fence processes that could prune or explicitly accept losing the halt guarantee: 0.6.15 ignores the control, even when a halt remains stored. Leave the patch installed; its new tables and functions are safe for older binaries, and replay does not resume the fleet.