fix(worker): keep planner statistics fresh on tables autovacuum never analyses (0.6)

CHANGELOG.md

@@ -6,6 +6,50 @@ Notable changes between releases. Detailed migration notes for storage transitio
 
 ### Added
 
+- **The maintenance leader keeps planner statistics fresh on the queue-storage
+  tables autovacuum never analyses.** Autovacuum does not analyse partitioned
+  tables, so the thirteen ring parents (`ready_entries`, `done_entries`,
+  `leases`, ...) had no inherited statistics, and the two small head tables
+  (`queue_claim_heads`, `queue_enqueue_heads`) never reach the autoanalyze
+  threshold, so the planner priced lane joins against them at its defaults
+  while they drove joins over hundreds of thousands of retained rows. The
+  leader now runs `ANALYZE` on them when it is elected and then every five
+  minutes. `ClientBuilder::statistics_refresh_interval` (Python
+  `statistics_refresh_interval_ms`) changes the cadence; zero disables it. It
+  analyses one relation per transaction with a 2 s `lock_timeout` (a call that
+  times out waiting for a lock is retried next interval and recorded as
+  `skipped_lock`) and never makes prune wait: a prune attempt that
+  lands while ANALYZE samples the oldest slot's tables returns `blocked` and
+  retries on its normal backoff. Results are in
+  `awa.maintenance.analyze.attempts` and `awa.maintenance.analyze.duration`
+  (scope `heads`, `ring_parent` or `pass`; outcome `ok`, `skipped_privilege`,
+  `skipped_lock`, `unavailable` or `failed`).
+- **Schema patch `analyze_queue_storage_relation` lets a runtime role that does
+  not own the tables refresh statistics.** `ANALYZE` skips a table its caller
+  may not analyse, and the documented runtime grants do not include
+  `MAINTAIN`, so the refresh runs through the `SECURITY DEFINER` function
+  `awa.analyze_queue_storage_relation(schema, relation)`, owned by the role
+  that applied the patch. It analyses only a head table or partitioned table
+  of a queue-storage schema owned by the owner of the `awa` schema. It is not
+  executable by `PUBLIC`, because the lock it takes lasts until the caller's
+  transaction ends and would block ring prune: the patch grants `EXECUTE` to
+  the roles that already hold INSERT, UPDATE and DELETE on the head tables
+  (the runtime grantees), and the function requires the calling session user to
+  hold them. A role created later needs the documented
+  `GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa`. The 0.6
+  series cannot add a migration version, so it ships like
+  `wait_free_dirty_marks`: **run `awa migrate` as the schema owner after
+  upgrading** (it applies the patch on a v040 schema; `awa migrate --sql`
+  and `--extract-to` emit it as the repeatable script
+  `R__analyze_queue_storage_relation`). The same
+  function is awa 0.7 migration v054, which replaces it cleanly. A binary
+  running before the patch, or against a function whose owner may not analyse
+  the tables, logs one warning naming what is missing and skips the refresh
+  without affecting job processing. Earlier binaries never call the function.
+  The patch also repairs an older helper definition, PUBLIC EXECUTE or missing
+  grants for existing runtime roles; a missing runtime privilege skips a pass
+  before calling the helper.
+
 - **The queue-storage prune `lock_timeout` is configurable, and prune
   attempts are counted by outcome for every ring.** Prune (queue, lease and
   claim rings) and terminal-delta rollup take their exclusive locks under a
@@ -30,6 +74,8 @@ Notable changes between releases. Detailed migration notes for storage transitio
 
 ### Fixed
 
+- Statistics-refresh permission warnings distinguish caller grants from
+  helper/table ownership and ANALYZE privileges.
 - **Ring prune and terminal-count rollup report `Blocked` when any of their
   statements times out on a lock.** Prune sets a short `lock_timeout` (10 ms
   by default) before it takes `ACCESS EXCLUSIVE` on a slot's partitions, but

awa-metrics/src/lib.rs

@@ -71,6 +71,8 @@ pub mod names {
     pub const MAINTENANCE_ROTATE_SKIPPED_ROWS: &str = "awa.maintenance.rotate.skipped_rows";
     pub const MAINTENANCE_PRUNE_ATTEMPTS: &str = "awa.maintenance.prune.attempts";
     pub const MAINTENANCE_PRUNE_SKIPPED_ROWS: &str = "awa.maintenance.prune.skipped_rows";
+    pub const MAINTENANCE_ANALYZE_ATTEMPTS: &str = "awa.maintenance.analyze.attempts";
+    pub const MAINTENANCE_ANALYZE_DURATION: &str = "awa.maintenance.analyze.duration";
     pub const STORAGE_TRANSITION_READY: &str = "awa.storage.transition_ready";
     pub const STORAGE_CANONICAL_LIVE_BACKLOG: &str = "awa.storage.canonical_live_backlog";
     pub const STORAGE_LIVE_RUNTIME_CAPABILITY: &str = "awa.storage.live_runtime_capability";
@@ -253,6 +255,13 @@ pub struct AwaMetrics {
     /// rather than one per tick. Fleets alert on this rather than
     /// scraping the matching `tracing::warn!` line. Issue #242.
     pub maintenance_branch_overrun_total: Counter<u64>,
+    /// Planner-statistics `ANALYZE` calls by scope (`heads`, `ring_parent`, `pass`)
+    /// and outcome (`ok`, `skipped_privilege`, `skipped_lock`,
+    /// `unavailable`, `failed`). One increment per relation per refresh pass.
+    pub maintenance_analyze_attempts: Counter<u64>,
+    /// Wall-clock duration of one planner-statistics `ANALYZE` call, by scope
+    /// and outcome.
+    pub maintenance_analyze_duration_seconds: Histogram<f64>,
     /// Current ring `current_slot` per ring, sampled from each rotate call.
     /// The slot number itself isn't meaningful but the rate of advance is —
     /// dashboards plot `rate(slot_changes)` to see whether rotation is
@@ -551,6 +560,21 @@ impl AwaMetrics {
                 )
                 .with_unit("{episode}")
                 .build(),
+            maintenance_analyze_attempts: meter
+                .u64_counter(names::MAINTENANCE_ANALYZE_ATTEMPTS)
+                .with_description(
+                    "Planner-statistics ANALYZE calls by scope (heads/ring_parent) and outcome",
+                )
+                .with_unit("{attempt}")
+                .build(),
+            maintenance_analyze_duration_seconds: meter
+                .f64_histogram(names::MAINTENANCE_ANALYZE_DURATION)
+                .with_description(
+                    "Duration of one planner-statistics ANALYZE call by scope and outcome",
+                )
+                .with_unit("s")
+                .with_boundaries(WAIT_DURATION_BUCKETS_SECONDS.to_vec())
+                .build(),
             ring_current_slot: meter
                 .i64_gauge(names::RING_CURRENT_SLOT)
                 .with_description("Current slot index per ring (queue/lease/claim)")
@@ -1123,6 +1147,24 @@ impl AwaMetrics {
         self.maintenance_branch_overrun_total.add(1, &attrs);
     }
 
+    /// Record one planner-statistics `ANALYZE` call. `scope` is `"heads"`,
+    /// `"ring_parent"` or `"pass"` (a pass-level probe or failure); `outcome` is `"ok"`, `"skipped_privilege"`,
+    /// `"skipped_lock"`, `"unavailable"` or `"failed"`.
+    pub fn record_statistics_analyze(
+        &self,
+        scope: &'static str,
+        outcome: &'static str,
+        duration: Duration,
+    ) {
+        let attrs = [
+            opentelemetry::KeyValue::new("awa.analyze.scope", scope),
+            opentelemetry::KeyValue::new("awa.analyze.outcome", outcome),
+        ];
+        self.maintenance_analyze_attempts.add(1, &attrs);
+        self.maintenance_analyze_duration_seconds
+            .record(duration.as_secs_f64(), &attrs);
+    }
+
     /// Record a maintenance tick that skipped prune because the ring's
     /// backoff after consecutive `skipped_active` or `blocked` results had not
     /// yet elapsed.
@@ -1376,7 +1418,7 @@ mod tests {
             .build();
         let metrics = AwaMetrics::new(&provider.meter("awa-metrics-test"));
 
-        let seconds: [(&str, &Histogram<f64>, &[f64]); 8] = [
+        let seconds: [(&str, &Histogram<f64>, &[f64]); 9] = [
             (
                 names::ENQUEUE_DURATION,
                 &metrics.enqueue_duration_seconds,
@@ -1392,6 +1434,11 @@ mod tests {
                 &metrics.maintenance_branch_duration_seconds,
                 &WAIT_DURATION_BUCKETS_SECONDS,
             ),
+            (
+                names::MAINTENANCE_ANALYZE_DURATION,
+                &metrics.maintenance_analyze_duration_seconds,
+                &WAIT_DURATION_BUCKETS_SECONDS,
+            ),
             (
                 names::DISPATCH_WAKE_TO_CLAIM_DURATION,
                 &metrics.dispatch_wake_to_claim_seconds,

awa-model/migrations/patches/analyze_queue_storage_relation.sql

@@ -0,0 +1,193 @@
+-- Schema patch analyze_queue_storage_relation: planner statistics helper for
+-- the queue-storage tables autovacuum never analyses (#631).
+-- Counterpart of awa 0.7 migration v054, minus the schema_version row; v054
+-- replaces this body with one that checks the schema through
+-- awa.queue_storage_schemas(), which 0.6 does not have.
+--
+-- 0.6 delivery: the 0.6 series cannot add a migration version (0.7 owns v041
+-- and above), so this ships as an idempotent schema patch. `awa migrate`
+-- applies it when the schema is exactly at v040 and the probe in
+-- migrations.rs reports it missing; `awa migrate --sql` prints it as a
+-- repeatable R__ script for external runners.
+--
+-- Autovacuum does not analyse partitioned tables, so the ring parents
+-- (ready_entries, done_entries, leases, ...) have no inherited statistics. It
+-- also never reaches the autoanalyze threshold on the two small head tables
+-- (queue_claim_heads, queue_enqueue_heads), so the planner prices lane joins
+-- against them at its defaults (320 rows, n_distinct 200) while they drive
+-- joins over hundreds of thousands of retained rows.
+--
+-- The maintenance leader refreshes both on an interval. Runtimes usually
+-- connect as a role that does not own the tables, and ANALYZE skips with a
+-- warning a table its caller may not analyse, so the refresh runs through
+-- this SECURITY DEFINER function owned by the migration role, which owns the
+-- tables.
+--
+-- The function analyses one relation per call and the caller runs each call as
+-- its own transaction, so the locks ANALYZE takes on a partitioned table's
+-- children are held for that table rather than for the whole refresh.
+--
+-- Locking on a partitioned table, per the ANALYZE documentation: SKIP_LOCKED
+-- applies when ANALYZE starts work on a relation, so a child that ring prune
+-- holds ACCESS EXCLUSIVE is skipped during the per-child analysis, but
+-- sampling the children for the parent's statistics can still wait for it.
+-- That wait is bounded by the function's lock_timeout and fails the call with
+-- SQLSTATE 55P03, which the caller records as skipped_lock. SKIP_LOCKED also
+-- skips every partition silently when the parent itself is locked, so the
+-- function takes the parent's SHARE UPDATE EXCLUSIVE lock first and lets that
+-- wait fail instead.
+--
+-- Definer boundary: only a head table or ring parent of a queue-storage schema
+-- (the AWA-specific claim_ready_runtime sentinel) that is owned by the owner of
+-- the awa schema is analysed, from quoted identifiers only. ANALYZE writes
+-- statistics and nothing else.
+-- Only the fixed list of ring parents and the two head tables is accepted;
+-- a migration that adds a partitioned table to the substrate must extend it.
+--
+-- Who may call it: the lock above lasts until the caller's transaction ends
+-- and blocks ring prune of that table, so EXECUTE is not granted to PUBLIC. It
+-- goes to the roles that already hold INSERT, UPDATE and DELETE on the head
+-- tables of the awa schema (the runtime grantees), and the function itself
+-- requires the calling session user to hold those privileges on the target
+-- schema's queue_claim_heads. A role created later needs the documented
+-- `GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa` (or default privileges).
+--
+-- Returns 'analyzed'; 'skipped_privilege' when the function owner may not
+-- analyse the relation; or 'missing' when the relation does not exist.
+--
+-- Releases before the patch never call it; a binary that runs before the patch
+-- probes for the function, logs once that it is missing and skips the refresh.
+-- Creating or replacing it takes no table lock and completes in milliseconds;
+-- calling it takes SHARE UPDATE EXCLUSIVE on the relation until the caller's
+-- transaction ends.
+
+CREATE OR REPLACE FUNCTION awa.analyze_queue_storage_relation(
+    p_schema   TEXT,
+    p_relation TEXT
+)
+RETURNS TEXT
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog, pg_temp
+SET lock_timeout = '2s'
+AS $$
+DECLARE
+    v_heads       REGCLASS;
+    v_relid       OID;
+    v_relkind     "char";
+    v_relowner    OID;
+    v_may_analyze BOOLEAN;
+BEGIN
+    -- awa_statistics_helper_v1: bump when the definition or grants change.
+    IF to_regprocedure(format(
+        '%I.claim_ready_runtime(text,bigint,double precision,double precision)',
+        p_schema
+    )) IS NULL THEN
+        RAISE EXCEPTION '% is not a queue-storage schema', p_schema
+            USING ERRCODE = '22023';
+    END IF;
+
+    IF p_relation <> ALL (ARRAY[
+            'queue_claim_heads',
+            'queue_enqueue_heads',
+            'done_entries',
+            'lease_claim_batches',
+            'lease_claim_closure_batches',
+            'lease_claim_closures',
+            'lease_claims',
+            'leases',
+            'queue_terminal_count_deltas',
+            'ready_claim_attempt_batches',
+            'ready_entries',
+            'ready_segments',
+            'ready_tombstones',
+            'receipt_completion_batches',
+            'receipt_completion_tombstones'
+        ]) THEN
+        RAISE EXCEPTION '%.% is not a ring parent or head table of a queue-storage schema',
+            p_schema, p_relation
+            USING ERRCODE = '22023';
+    END IF;
+
+    v_heads := to_regclass(format('%I.queue_claim_heads', p_schema));
+    IF v_heads IS NULL
+       OR NOT has_table_privilege(session_user, v_heads, 'INSERT')
+       OR NOT has_table_privilege(session_user, v_heads, 'UPDATE')
+       OR NOT has_table_privilege(session_user, v_heads, 'DELETE') THEN
+        RAISE EXCEPTION 'permission denied: % may not maintain queue storage in schema %',
+            session_user, p_schema
+            USING ERRCODE = '42501';
+    END IF;
+
+    v_relid := to_regclass(format('%I.%I', p_schema, p_relation));
+    IF v_relid IS NULL THEN
+        RETURN 'missing';
+    END IF;
+
+    SELECT class.relkind, class.relowner INTO v_relkind, v_relowner
+    FROM pg_class AS class WHERE class.oid = v_relid;
+
+    IF v_relowner IS DISTINCT FROM (
+        SELECT namespace.nspowner FROM pg_namespace AS namespace WHERE namespace.nspname = 'awa'
+    ) THEN
+        RAISE EXCEPTION '%.% is not owned by the awa schema owner', p_schema, p_relation
+            USING ERRCODE = '42501';
+    END IF;
+
+    -- ANALYZE skips, with a warning rather than an error, a table the caller
+    -- may not analyse: the owner, a MAINTAIN holder (PostgreSQL 17+), a
+    -- member of pg_maintain, or the database owner may analyse it.
+    SELECT
+        pg_has_role(current_user, v_relowner, 'USAGE')
+        OR pg_has_role(
+            current_user,
+            (SELECT database.datdba FROM pg_database AS database
+             WHERE database.datname = current_database()),
+            'USAGE'
+        )
+        OR CASE
+            WHEN current_setting('server_version_num')::int >= 170000
+                THEN has_table_privilege(v_relid, 'MAINTAIN')
+            ELSE FALSE
+        END
+    INTO v_may_analyze;
+
+    IF NOT v_may_analyze THEN
+        RETURN 'skipped_privilege';
+    END IF;
+
+    IF v_relkind = 'p' THEN
+        EXECUTE format(
+            'LOCK TABLE ONLY %I.%I IN SHARE UPDATE EXCLUSIVE MODE', p_schema, p_relation
+        );
+        EXECUTE format('ANALYZE (SKIP_LOCKED) %I.%I', p_schema, p_relation);
+    ELSE
+        EXECUTE format('ANALYZE %I.%I', p_schema, p_relation);
+    END IF;
+    RETURN 'analyzed';
+END
+$$;
+
+REVOKE EXECUTE ON FUNCTION awa.analyze_queue_storage_relation(TEXT, TEXT) FROM PUBLIC;
+
+DO $grant$
+DECLARE
+    v_grantee TEXT;
+BEGIN
+    FOR v_grantee IN
+        SELECT pg_get_userbyid(privilege.grantee)
+        FROM pg_class AS heads
+        CROSS JOIN LATERAL aclexplode(heads.relacl) AS privilege
+        WHERE heads.oid = to_regclass('awa.queue_claim_heads')
+          AND privilege.grantee <> 0
+          AND privilege.privilege_type IN ('INSERT', 'UPDATE', 'DELETE')
+        GROUP BY privilege.grantee
+        HAVING count(DISTINCT privilege.privilege_type) = 3
+    LOOP
+        EXECUTE format(
+            'GRANT EXECUTE ON FUNCTION awa.analyze_queue_storage_relation(text, text) TO %I',
+            v_grantee
+        );
+    END LOOP;
+END
+$grant$;