[codex] Fix recovery parsing and queued-run observability

deploy/helm/paperclip/templates/prometheusrule.yaml

@@ -317,15 +317,33 @@ spec:
           # indefinitely while each individual row is young. This gauge
           # carries the server-computed age of the oldest queued row PER
           # AGENT (refreshed on every /metrics scrape from a live
-          # MIN(created_at) aggregate), so `for:` here is only riding out
+          # MIN(coalesce(queued_at, created_at)) aggregate), so `for:` here is only riding out
           # scrape flapping, not standing in for the real age.
-          expr: max(paperclip_queued_run_oldest_age_seconds) by (agent_id) > {{ .Values.prometheusRule.queuedRunStrandedAgeSeconds }}
+          # The freshness gate is load-bearing: a failed database refresh
+          # leaves the previous gauge value in memory, so stale data must not
+          # qualify a strand alert. Gate each API replica against its own
+          # refresh result before taking the fleet-wide per-agent max; an
+          # `on()` join would let one healthy replica bless another replica's
+          # stale age snapshot.
+          expr: max by (agent_id) (paperclip_queued_run_oldest_age_seconds and on(instance) (paperclip_queued_run_age_metrics_refresh_success == 1)) > {{ .Values.prometheusRule.queuedRunStrandedAgeSeconds }}
           for: {{ .Values.prometheusRule.queuedRunStrandedFor }}
           labels:
             severity: warning
           annotations:
             summary: "Paperclip agent {{`{{ $labels.agent_id }}`}} has a queued run stranded for longer than {{ .Values.prometheusRule.queuedRunStrandedAgeSeconds }}s"
-            description: "Agent {{`{{ $labels.agent_id }}`}}'s oldest `queued` heartbeat run has been waiting for {{`{{ $value }}`}}s (BLO-21116). The issue it targets looks `in_progress`/assigned with an active run and generates no other alert, but nothing is executing -- and left alone this also manufactures a false productivity-review escalation reading the queue time as unattended active duration. Check whether the agent is at `maxConcurrentRuns` (capacity starvation -- other runs should still be cycling) or fully idle (a scheduler-tick or dispatch-path fault). Query `select id, agent_id, status, created_at, queued_at, now() - coalesce(queued_at, created_at) as age, context_snapshot ->> 'issueId' as issue_id from heartbeat_runs where status = 'queued' and agent_id = '{{`{{ $labels.agent_id }}`}}' order by coalesce(queued_at, created_at) asc limit 10` to find the specific rows -- ordered and aged by the same coalesce(queued_at, created_at) the gauge itself uses, so a scheduled-retry promotion cannot appear hours older here than the $value that fired. See the runbook for the starvation-vs-dropped triage."
+            description: "Agent {{`{{ $labels.agent_id }}`}}'s oldest `queued` heartbeat run has been waiting for {{`{{ $value }}`}}s since queue entry (BLO-21116). The age is only trusted while paperclip_queued_run_age_metrics_refresh_success is 1. The issue it targets looks `in_progress`/assigned with an active run, but nothing is executing. Query coalesce(queued_at, created_at), not created_at alone: retries promoted from scheduled_retry intentionally reset their dispatch-wait clock. See the runbook for saturation-versus-dispatch triage."
+            runbook_url: {{ .Values.prometheusRule.queuedRunStrandedRunbookUrl | quote }}
+        - alert: PaperclipQueuedRunAgeMetricsRefreshFailed
+          # This is deliberately separate from the strand alert. A failed
+          # refresh means the age gauge is not fresh, not that the queue is
+          # empty; page on it so the freshness gate cannot hide a real strand.
+          expr: paperclip_queued_run_age_metrics_refresh_success == 0
+          for: {{ .Values.prometheusRule.queuedRunAgeMetricsRefreshFailureFor }}
+          labels:
+            severity: warning
+          annotations:
+            summary: "Paperclip queued-run age metric refresh is failing"
+            description: "The most recent /metrics queued-run-age database refresh failed (BLO-21116), so queued-run ages may be stale and PaperclipQueuedRunStranded is intentionally gated off. Check Paperclip logs and database connectivity; do not interpret a stale zero as an empty queue."
             runbook_url: {{ .Values.prometheusRule.queuedRunStrandedRunbookUrl | quote }}
     {{- with .Values.prometheusRule.additionalGroups }}
     {{- toYaml . | nindent 4 }}

deploy/helm/paperclip/tests/prometheus-rule.test.mjs

@@ -307,24 +307,13 @@ test("PaperclipPrReviewWakeTerminalFailed is pr_review-scoped, gauge-keyed, and
   );
 });
 
-test("PaperclipQueuedRunStranded is agent-keyed, gauge-thresholded, and links its runbook (BLO-21116)", () => {
-  const rendered = execFileSync(
-    "helm",
-    [
-      "template",
-      "paperclip",
-      "deploy/helm/paperclip",
-      "--namespace",
-      "paperclip",
-      "-f",
-      "deploy/helm/paperclip/values.blockcast.yaml",
-      "--show-only",
-      "templates/prometheusrule.yaml",
-      "--set",
-      "prometheusRule.enabled=true",
-    ],
-    { cwd: repoRoot, encoding: "utf8" },
-  );
+test("PaperclipQueuedRunStranded is agent-keyed, freshness-gated, and fires before 30m (BLO-21116)", () => {
+  const rendered = renderChart([
+    "--show-only",
+    "templates/prometheusrule.yaml",
+    "--set",
+    "prometheusRule.enabled=true",
+  ]);
 
   assert.match(rendered, /alert: PaperclipQueuedRunStranded/);
   const [, expr] = rendered.match(
@@ -340,9 +329,8 @@ test("PaperclipQueuedRunStranded is agent-keyed, gauge-thresholded, and links it
   // is young.
   assert.match(
     expr,
-    /^max\(paperclip_queued_run_oldest_age_seconds\) by \(agent_id\) > (\d+)$/,
-    "queued-run-stranded alert must threshold the per-agent age gauge, "
-      + "not a summed count under a long `for:`",
+    /^max by \(agent_id\) \(paperclip_queued_run_oldest_age_seconds and on\(instance\) \(paperclip_queued_run_age_metrics_refresh_success == 1\)\) > (\d+)$/,
+    "queued-run-stranded alert must gate each replica's age before taking the per-agent max",
   );
 
   const [, ageThreshold] = expr.match(/> (\d+)$/) ?? [];
@@ -390,3 +378,22 @@ test("PaperclipQueuedRunStranded is agent-keyed, gauge-thresholded, and links it
   );
 });
 
+test("PaperclipQueuedRunAgeMetricsRefreshFailed exposes a stale snapshot instead of hiding it", () => {
+  const rendered = renderChart([
+    "--show-only",
+    "templates/prometheusrule.yaml",
+    "--set",
+    "prometheusRule.enabled=true",
+  ]);
+
+  assert.match(
+    rendered,
+    /alert: PaperclipQueuedRunAgeMetricsRefreshFailed[\s\S]*?\n\s+expr: paperclip_queued_run_age_metrics_refresh_success == 0\n/,
+    "a failed queued-run-age refresh must have its own alert",
+  );
+  assert.match(
+    rendered,
+    /alert: PaperclipQueuedRunAgeMetricsRefreshFailed[\s\S]*?runbook_url: "[^"]*runbooks\/queued-run-stranded\.md"/,
+    "the freshness failure alert must route responders to the queued-run runbook",
+  );
+});

deploy/helm/paperclip/values.yaml

@@ -495,7 +495,9 @@ prometheusRule:
   queuedRunStrandedAgeSeconds: 1440
   # -- Scrape-flap tolerance only (BLO-21116). The ageing lives in queuedRunStrandedAgeSeconds above; it stacks on top of that threshold rather than testing independently, which is why the threshold is 1440 and not 1800.
   queuedRunStrandedFor: 5m
-  # -- Runbook for the queued-run-stranded alert, linked from its annotation.
+  # -- Alert only after sustained failed refreshes; a one-off failed scrape is logged and retried on the next scrape.
+  queuedRunAgeMetricsRefreshFailureFor: 5m
+  # -- Runbook for the queued-run-stranded and metric-freshness alerts, linked from their annotations.
   queuedRunStrandedRunbookUrl: "https://github.com/Blockcast/paperclip/blob/master/runbooks/queued-run-stranded.md"
   # -- Prometheus regex of suppression causes that mean an outage rather than a deliberate decline (BLO-18859). Expected policy declines (company.inactive, heartbeat.cooldown.active, heartbeat.disabled, heartbeat.wakeOnDemand.disabled, budget.blocked, agent.not_invokable, heartbeat.worktree_execution_cutoff, issue_tree_hold_active) are excluded so they never page. `other` is included: it is the bucket for a skip reason missing from KNOWN_GITHUB_SUPPRESSION_CAUSES, and an untriaged cause is not a known-good decline.
   githubDeliverySuppressionOutageCauses: "heartbeat.scheduling_suppressed|dispatch_rejected|other"

packages/db/src/heartbeat-runs-queued-age-index-migration.test.ts

@@ -0,0 +1,100 @@
+/**
+ * BLO-21116 review follow-up: migration 0217 must not build the queued-age
+ * index inline on a populated heartbeat_runs table.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import { afterEach, describe, expect, it } from "vitest";
+import postgres from "postgres";
+import { applyPendingMigrations, inspectMigrations } from "./client.js";
+import {
+  getEmbeddedPostgresTestSupport,
+  startEmbeddedPostgresTestDatabase,
+} from "./test-embedded-postgres.js";
+
+const MIGRATION_FILE = "0217_heartbeat_runs_queued_age_idx.sql";
+const INDEX_NAME = "heartbeat_runs_queued_age_idx";
+const INDEX_DEFINITION =
+  `ON heartbeat_runs USING btree (agent_id, (coalesce(queued_at, created_at))) `
+  + `WHERE status = 'queued'`;
+const cleanups: Array<() => Promise<void>> = [];
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+async function migrationHash() {
+  const content = await fs.promises.readFile(
+    new URL(`./migrations/${MIGRATION_FILE}`, import.meta.url),
+    "utf8",
+  );
+  return createHash("sha256").update(content).digest("hex");
+}
+
+afterEach(async () => {
+  while (cleanups.length > 0) await cleanups.pop()?.();
+}, 60_000);
+
+describeEmbeddedPostgres("heartbeat-run queued-age index migration", () => {
+  it("requires online precreation for a populated heartbeat_runs table", async () => {
+    const database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-queued-age-index-");
+    cleanups.push(database.cleanup);
+    const sql = postgres(database.connectionString, { max: 1, onnotice: () => {} });
+    cleanups.push(async () => sql.end());
+
+    await sql.unsafe(`
+      SET session_replication_role = replica;
+      INSERT INTO heartbeat_runs (company_id, agent_id, status, context_snapshot)
+      VALUES ('11111111-1111-4111-8111-111111111111', '22222222-2222-4222-8222-222222222222', 'queued', '{}'::jsonb);
+      SET session_replication_role = origin;
+      DROP INDEX ${INDEX_NAME};
+    `);
+    await sql`
+      DELETE FROM "drizzle"."__drizzle_migrations"
+      WHERE "hash" = ${await migrationHash()}
+    `;
+
+    await expect(applyPendingMigrations(database.connectionString)).rejects.toMatchObject({
+      message: "migration 0217 requires online queued-age index precreation",
+      hint: expect.stringContaining(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${INDEX_NAME}`),
+    });
+    expect(await inspectMigrations(database.connectionString)).toMatchObject({
+      status: "needsMigrations",
+      pendingMigrations: [MIGRATION_FILE],
+    });
+
+    await sql.unsafe(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${INDEX_NAME} ${INDEX_DEFINITION}`);
+    await applyPendingMigrations(database.connectionString);
+    expect((await inspectMigrations(database.connectionString)).status).toBe("upToDate");
+
+    const indexes = await sql<{ indexdef: string }[]>`
+      SELECT indexdef
+      FROM pg_indexes
+      WHERE schemaname = 'public' AND indexname = ${INDEX_NAME}
+    `;
+    expect(indexes).toHaveLength(1);
+    expect(indexes[0]?.indexdef).toContain("COALESCE(queued_at, created_at)");
+    expect(indexes[0]?.indexdef).toContain("WHERE (status = 'queued'::text)");
+  }, 60_000);
+
+  it("rejects an invalid same-name precreated index", async () => {
+    const database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-queued-age-invalid-");
+    cleanups.push(database.cleanup);
+    const sql = postgres(database.connectionString, { max: 1, onnotice: () => {} });
+    cleanups.push(async () => sql.end());
+
+    await sql`
+      DELETE FROM "drizzle"."__drizzle_migrations"
+      WHERE "hash" = ${await migrationHash()}
+    `;
+    await sql.unsafe(`
+      DROP INDEX ${INDEX_NAME};
+      CREATE INDEX ${INDEX_NAME}
+      ON heartbeat_runs USING btree (agent_id, created_at)
+      WHERE status = 'queued'
+    `);
+
+    await expect(applyPendingMigrations(database.connectionString)).rejects.toMatchObject({
+      message: "migration 0217 found an invalid or incorrectly defined queued-age index",
+      hint: expect.stringContaining(`DROP INDEX CONCURRENTLY IF EXISTS ${INDEX_NAME}`),
+    });
+  }, 60_000);
+});

packages/db/src/migrations/0217_heartbeat_runs_queued_age_idx.sql

@@ -0,0 +1,72 @@
+-- BLO-21116: make the scrape-time queued-run age aggregate use a queue-only
+-- expression index. The queued_at column and its rollout backfill already live
+-- in migration 0215 on current master.
+--
+-- Drizzle migrations run in a transaction, so CREATE INDEX CONCURRENTLY is not
+-- available here. A plain index build would take a SHARE lock over the hot
+-- heartbeat_runs table. Populated databases must precreate this index online;
+-- empty bootstrap databases can build it inline.
+-- paperclip:migration-safety-ignore large-create-index-not-concurrently: the populated-table path fails closed and supplies the concurrent command.
+DO $$
+DECLARE
+  normalized_predicate text;
+BEGIN
+  IF to_regclass('public.heartbeat_runs_queued_age_idx') IS NOT NULL THEN
+    SELECT trim(regexp_replace(
+             coalesce(pg_get_expr(index_metadata.indpred, index_metadata.indrelid, TRUE), ''),
+             '\s+', ' ', 'g'))
+      INTO normalized_predicate
+      FROM pg_index AS index_metadata
+     WHERE index_metadata.indexrelid = to_regclass('public.heartbeat_runs_queued_age_idx');
+
+    IF NOT EXISTS (
+      SELECT 1
+      FROM pg_index AS index_metadata
+      JOIN pg_class AS index_relation
+        ON index_relation.oid = index_metadata.indexrelid
+      JOIN pg_am AS access_method
+        ON access_method.oid = index_relation.relam
+      WHERE index_metadata.indexrelid = to_regclass('public.heartbeat_runs_queued_age_idx')
+        AND index_metadata.indrelid = 'public.heartbeat_runs'::regclass
+        AND index_metadata.indisvalid
+        AND access_method.amname = 'btree'
+        AND index_metadata.indnkeyatts = 2
+        AND index_metadata.indnatts = 2
+        AND ARRAY(
+          SELECT pg_get_indexdef(index_metadata.indexrelid, key_position, TRUE)
+          FROM generate_series(1, index_metadata.indnkeyatts) AS key_position
+          ORDER BY key_position
+        ) = ARRAY['agent_id', 'COALESCE(queued_at, created_at)']
+        AND index_metadata.indoption = '0 0'::int2vector
+        AND normalized_predicate = 'status = ''queued''::text'
+    )
+    THEN
+      RAISE EXCEPTION USING
+        MESSAGE = 'migration 0217 found an invalid or incorrectly defined queued-age index',
+        HINT = 'Run DROP INDEX CONCURRENTLY IF EXISTS heartbeat_runs_queued_age_idx; then CREATE INDEX CONCURRENTLY heartbeat_runs_queued_age_idx ON heartbeat_runs USING btree (agent_id, (coalesce(queued_at, created_at))) WHERE status = ''queued''; then retry migrations.';
+    END IF;
+  ELSE
+    IF EXISTS (SELECT 1 FROM "heartbeat_runs" LIMIT 1) THEN
+      RAISE EXCEPTION USING
+        MESSAGE = 'migration 0217 requires online queued-age index precreation',
+        HINT = 'Run CREATE INDEX CONCURRENTLY IF NOT EXISTS heartbeat_runs_queued_age_idx ON heartbeat_runs USING btree (agent_id, (coalesce(queued_at, created_at))) WHERE status = ''queued''; then retry migrations.';
+    END IF;
+
+    -- Close the gap between the empty-table check and CREATE INDEX without
+    -- taking this lock on a populated production table.
+    LOCK TABLE "heartbeat_runs" IN SHARE MODE;
+    IF EXISTS (SELECT 1 FROM "heartbeat_runs" LIMIT 1) THEN
+      RAISE EXCEPTION USING
+        MESSAGE = 'migration 0217 requires online queued-age index precreation',
+        HINT = 'Run CREATE INDEX CONCURRENTLY IF NOT EXISTS heartbeat_runs_queued_age_idx ON heartbeat_runs USING btree (agent_id, (coalesce(queued_at, created_at))) WHERE status = ''queued''; then retry migrations.';
+    END IF;
+
+    CREATE INDEX "heartbeat_runs_queued_age_idx"
+      ON "heartbeat_runs" USING btree (
+        "agent_id",
+        (coalesce("queued_at", "created_at"))
+      )
+      WHERE "status" = 'queued';
+  END IF;
+END
+$$;

packages/db/src/migrations/meta/_journal.json

@@ -1464,6 +1464,13 @@
       "when": 1786360500000,
       "tag": "0216_issue_work_products_external_identity",
       "breakpoints": true
+    },
+    {
+      "idx": 217,
+      "version": "7",
+      "when": 1786682095165,
+      "tag": "0217_heartbeat_runs_queued_age_idx",
+      "breakpoints": true
     }
   ]
 }

packages/db/src/schema/heartbeat_runs.ts

@@ -124,5 +124,8 @@ export const heartbeatRuns = pgTable(
       table.companyId,
       table.createdAt.desc(),
     ),
+    queuedAgeIdx: index("heartbeat_runs_queued_age_idx")
+      .on(table.agentId, sql`coalesce(${table.queuedAt}, ${table.createdAt})`)
+      .where(sql`${table.status} = 'queued'`),
   }),
 );

runbooks/README.md

@@ -18,6 +18,10 @@ platform cannot resolve automatically. Each runbook should be:
   which nothing re-drives: decide re-review vs accept without double-posting a
   review. Trigger: alert `PaperclipPrReviewWakeTerminalFailed`, or
   `paperclip_agent_wakeup_terminal_failed_unresolved{scope="pr_review"} > 0`.
+- [`queued-run-stranded.md`](queued-run-stranded.md) — a dispatchable
+  `heartbeat_runs.status='queued'` row is not advancing, or its age snapshot
+  cannot be refreshed safely. Trigger: `PaperclipQueuedRunStranded` or
+  `PaperclipQueuedRunAgeMetricsRefreshFailed`.
 - [`clear-polluted-ssh-workspace.md`](clear-polluted-ssh-workspace.md) —
   recover a stranded SSH-driven run whose workspace import is failing on a
   sibling task's leftover scratch state. Trigger: blocked issue auto-comment