fix(webapp): timestamp live metric responses

Author: carderne

apps/webapp/app/hooks/useMetricResourceQuery.ts

@@ -102,6 +102,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO
   );
   const [isLoading, setIsLoading] = useState(true);
   const [failed, setFailed] = useState(false);
+  const [responseReceivedAt, setResponseReceivedAt] = useState<number | null>(null);
   const abortRef = useRef<AbortController | null>(null);
   const loadedKeyRef = useRef<string | null>(null);
 
@@ -111,6 +112,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO
       loadedKeyRef.current = cacheKey;
       setRows(null);
       setFailed(false);
+      setResponseReceivedAt(null);
       setIsLoading(false);
       return;
     }
@@ -125,6 +127,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO
       loadedKeyRef.current = cacheKey;
       setRows(responseCache.get(cacheKey) ?? null);
       setFailed(false);
+      setResponseReceivedAt(null);
     }
     setIsLoading(true);
     fetch("/resources/metric", {
@@ -152,15 +155,18 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO
           cacheSet(cacheKey, data.data.rows);
           setRows(data.data.rows);
           setFailed(false);
+          setResponseReceivedAt(Date.now());
         } else {
           setFailed(true);
+          setResponseReceivedAt(null);
         }
         setIsLoading(false);
       })
       .catch((error) => {
         if (error instanceof DOMException && error.name === "AbortError") return;
         if (!controller.signal.aborted) {
           setFailed(true);
+          setResponseReceivedAt(null);
           setIsLoading(false);
         }
       });
@@ -191,5 +197,11 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO
     callback: load,
   });
 
-  return { rows: rows ?? [], isLoading, showLoading: isLoading && !rows, failed };
+  return {
+    rows: rows ?? [],
+    isLoading,
+    showLoading: isLoading && !rows,
+    failed,
+    responseReceivedAt,
+  };
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -420,25 +420,29 @@ function QueuesWithMetricsView() {
   // Empty rows (quiet env, or the very first fetch still in flight) fall back to the loader values,
   // so we never flash a stale 0. Fixed 15m window, env-wide (no queue filter), CH-only recurring
   // load; pauses while the tab is hidden (handled inside the hook).
-  const { rows: liveBlockRows } = useMetricResourceQuery(QUEUE_LIVE_BLOCKS_QUERY, {
-    organizationId: organization.id,
-    projectId: project.id,
-    environmentId: env.id,
-    timeRange: { period: QUEUE_LIVE_BLOCKS_PERIOD, from: null, to: null },
-    defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD,
-    fillGaps: false,
-    refreshIntervalMs: 15_000,
-  });
+  const { rows: liveBlockRows, responseReceivedAt } = useMetricResourceQuery(
+    QUEUE_LIVE_BLOCKS_QUERY,
+    {
+      organizationId: organization.id,
+      projectId: project.id,
+      environmentId: env.id,
+      timeRange: { period: QUEUE_LIVE_BLOCKS_PERIOD, from: null, to: null },
+      defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD,
+      fillGaps: false,
+      refreshIntervalMs: 15_000,
+    }
+  );
   const lastLiveBlockRow =
     liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null;
   // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on
   // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must
   // not override the loader's Redis-exact live values with a stale count.
   const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN;
   const freshLiveBlockRow =
+    responseReceivedAt !== null &&
     lastLiveBlockRow &&
     Number.isFinite(lastLiveBucketMs) &&
-    Date.now() - lastLiveBucketMs < LIVE_GAUGE_FRESH_MS
+    responseReceivedAt - lastLiveBucketMs < LIVE_GAUGE_FRESH_MS
       ? lastLiveBlockRow
       : null;
   const envQueuedLive = freshLiveBlockRow

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx

@@ -1121,7 +1121,7 @@ function QueueStats({
   // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first
   // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the
   // *Live values stay null, so the blocks show the loader values instead of flashing 0.
-  const { rows: liveRows } = useQueueMetric(
+  const { rows: liveRows, responseReceivedAt } = useQueueMetric(
     `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`,
     {
       ids,
@@ -1138,7 +1138,9 @@ function QueueStats({
   const latest = liveRows.length > 0 ? liveRows[liveRows.length - 1] : undefined;
   const latestBucketMs = latest ? clickhouseTimeToMs(latest.t) : NaN;
   const liveFresh =
-    Number.isFinite(latestBucketMs) && Date.now() - latestBucketMs < LIVE_GAUGE_FRESH_MS;
+    responseReceivedAt !== null &&
+    Number.isFinite(latestBucketMs) &&
+    responseReceivedAt - latestBucketMs < LIVE_GAUGE_FRESH_MS;
   const fresh = latest && liveFresh ? latest : undefined;
   const runningLive = fresh ? toNumber(fresh.running) : null;
   const queuedLive = fresh ? toNumber(fresh.queued) : null;

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -176,7 +176,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
         envParam,
         run: result.run,
       });
-      return typedjson({ type: "run" as const, run: result.run, queueMetrics });
+      return typedjson({
+        type: "run" as const,
+        run: result.run,
+        queueMetrics,
+        loadedAt: Date.now(),
+      });
     }
     return typedjson({ type: "span" as const, span: result.span });
   } catch (error) {
@@ -270,6 +275,7 @@ export function SpanView({
         <RunBody
           run={fetcher.data.run}
           queueMetrics={fetcher.data.queueMetrics}
+          loadedAt={fetcher.data.loadedAt}
           runParam={runParam}
           spanId={spanId}
           closePanel={closePanel}
@@ -395,12 +401,14 @@ function applySpanOverrides(span: Span, spanOverrides?: SpanOverride): Span {
 function RunBody({
   run,
   queueMetrics,
+  loadedAt,
   runParam,
   spanId,
   closePanel,
 }: {
   run: SpanRun;
   queueMetrics: RunQueueMetrics | null;
+  loadedAt: number;
   runParam: string;
   spanId: string;
   closePanel?: () => void;
@@ -1148,6 +1156,7 @@ function RunBody({
                   waiting={queueMetrics.waiting}
                   status={run.status}
                   createdAt={run.createdAt}
+                  loadedAt={loadedAt}
                   runFriendlyId={run.friendlyId}
                 />
               ) : null}
@@ -1283,6 +1292,7 @@ function WaitingInQueueBlock({
   waiting,
   status,
   createdAt,
+  loadedAt,
   runFriendlyId,
 }: {
   queueName: string;
@@ -1291,11 +1301,12 @@ function WaitingInQueueBlock({
   waiting: RunQueueWaiting;
   status: SpanRun["status"];
   createdAt: Date;
+  loadedAt: number;
   runFriendlyId: string;
 }) {
   // Latest gauges from ClickHouse (as on the queue page), polled so the blocks keep ticking. Trust
   // the newest bucket only while fresh; otherwise fall back to the loader's live values.
-  const { rows: liveRows } = useQueueMetric(
+  const { rows: liveRows, responseReceivedAt } = useQueueMetric(
     `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`,
     {
       ids: waiting.ids,
@@ -1307,8 +1318,9 @@ function WaitingInQueueBlock({
   );
   const latest = liveRows.length > 0 ? liveRows[liveRows.length - 1] : undefined;
   const latestBucketMs = latest ? clickhouseTimeToMs(latest.t) : NaN;
+  const now = responseReceivedAt ?? loadedAt;
   const fresh =
-    latest && Number.isFinite(latestBucketMs) && Date.now() - latestBucketMs < LIVE_GAUGE_FRESH_MS
+    latest && Number.isFinite(latestBucketMs) && now - latestBucketMs < LIVE_GAUGE_FRESH_MS
       ? latest
       : undefined;
 
@@ -1325,7 +1337,7 @@ function WaitingInQueueBlock({
   const showAtLimit = status === "PENDING" && atLimit && !paused;
   const pct =
     limit && limit > 0 ? Math.min(100, Math.round((runningAgainstLimit / limit) * 100)) : null;
-  const waitedMs = Math.max(0, Date.now() - new Date(createdAt).getTime());
+  const waitedMs = Math.max(0, now - new Date(createdAt).getTime());
 
   // Why the run is held, surfaced as a warning icon on the Status tile (queue-page style) rather
   // than a separate sentence.