From 130b81ddac26f6720c5a9480054da7dd89a79e61 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 20:29:04 +0000 Subject: [PATCH 01/12] Rebuild the admin overview around "is it keeping up?" A single snapshot cannot say whether a backlog is growing, and retained created_at cannot stand in for an arrival rate: on the queue-storage substrate compacted terminal rows fall back to finalized_at once their ready entry rotates. So `awa serve` now samples queue depth itself. Server (awa-ui): - QueueHistory keeps an in-memory window (default 60m at 15s) of per-queue available/running/lag, fed by the cached queue_overviews read and by a background sampler that holds only a weak reference and stops when the router is dropped. Served bucketed per minute at /api/stats/queue-history with nulls for gaps. - RouterConfig / router_with_config expose the setting; awa serve adds --queue-history-minutes (0 disables) and --queue-history-interval. Frontend: - Each queue is classified as not being worked, falling behind, long wait, paused, draining (with time to clear), steady or idle from its snapshot plus a least-squares fit over the last 5 minutes. - The overview leads with one verdict, the total waiting count and its last hour, then lists only queues needing attention. Failures are grouped by kind, queue and error; workers show only problems. - One dense QueueFlowTable serves Overview and Queues, switching to cards by container width rather than viewport. The Queues page gets back capacity/rate limit, DLQ policy and descriptor flags, and keeps search, sort and filter in the URL. - Built from the existing Card/Badge/Heading primitives; the page-specific CSS block is gone. Tests: Rust unit and DB-backed sampler tests, vitest coverage for the trend and verdict logic, synthetic browser scenarios rewritten around a stalled / falling-behind / draining / paused story, and the DB-backed Playwright specs updated. Removes the committed screenshots of the previous design. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016oDvumkeByijpFvqb85Qbj --- CHANGELOG.md | 13 + awa-cli/src/main.rs | 32 +- awa-ui/frontend/e2e/dashboard.spec.ts | 89 ++- awa-ui/frontend/e2e/navigation.spec.ts | 4 +- awa-ui/frontend/e2e/operations-fixture.ts | 96 ++- awa-ui/frontend/e2e/operations.spec.ts | 99 ++- awa-ui/frontend/e2e/queues.spec.ts | 35 +- .../frontend/src/__tests__/operations.test.ts | 272 ++++++++ awa-ui/frontend/src/app.css | 126 +--- .../frontend/src/components/HistorySpark.tsx | 91 +++ .../src/components/QueueFlowTable.tsx | 421 ++++++++++++ .../frontend/src/components/QueuePressure.tsx | 138 ---- .../src/components/SnapshotStatus.tsx | 9 +- awa-ui/frontend/src/lib/api.ts | 33 + awa-ui/frontend/src/lib/operations.ts | 438 +++++++++++- awa-ui/frontend/src/routes/index.tsx | 646 +++++++++++------- awa-ui/frontend/src/routes/queue-detail.tsx | 47 +- awa-ui/frontend/src/routes/queues.tsx | 245 ++++--- awa-ui/screenshots/overview-desktop.png | Bin 255990 -> 0 bytes awa-ui/screenshots/overview-mobile.png | Bin 260196 -> 0 bytes awa-ui/src/handlers/queues.rs | 14 +- awa-ui/src/handlers/stats.rs | 9 + awa-ui/src/history.rs | 398 +++++++++++ awa-ui/src/lib.rs | 69 +- awa-ui/src/state.rs | 13 + awa-ui/tests/admin_api_test.rs | 68 ++ docs/configuration.md | 17 + docs/ui-design.md | 30 +- skills/awa-operations/SKILL.md | 8 + 29 files changed, 2687 insertions(+), 773 deletions(-) create mode 100644 awa-ui/frontend/src/__tests__/operations.test.ts create mode 100644 awa-ui/frontend/src/components/HistorySpark.tsx create mode 100644 awa-ui/frontend/src/components/QueueFlowTable.tsx delete mode 100644 awa-ui/frontend/src/components/QueuePressure.tsx delete mode 100644 awa-ui/screenshots/overview-desktop.png delete mode 100644 awa-ui/screenshots/overview-mobile.png create mode 100644 awa-ui/src/history.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fe8d79a1..62c89d068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,19 @@ for each step. `awa.migration_lock_key()` expose the runner's advisory lock key. - Python prints a diagnostic to stderr if native shutdown has stalled for five seconds. +- **The admin overview answers "is it keeping up?" + ([#543](https://github.com/hardbyte/awa/pull/543)).** `awa serve` samples + queue depth every 15 seconds and keeps an hour in memory, served at + `/api/stats/queue-history`. The overview and Queues page classify each + queue as not being worked, falling behind, waiting long, paused, draining, + steady, or idle, list the ones needing attention first, and show an hour + of waiting-job history with a draining estimate. History starts when the + server starts and is per replica. Each sample is one queue overview read, + issued whether or not anyone has the UI open; tune it with + `--queue-history-minutes` and `--queue-history-interval`, or set + `--queue-history-minutes 0` to turn it off. Rust embedders get the same + default through every `awa_ui::router*` function, and can configure it + with `awa_ui::router_with_config`. ### Changed diff --git a/awa-cli/src/main.rs b/awa-cli/src/main.rs index f89003114..d7392cc2b 100644 --- a/awa-cli/src/main.rs +++ b/awa-cli/src/main.rs @@ -161,6 +161,20 @@ enum Commands { /// `--peer alloydb=https://awa-alloydb.internal`. #[arg(long = "peer", value_name = "NAME=URL")] peer: Vec, + /// Minutes of queue depth history kept in memory for the overview's + /// trends. 0 disables sampling. History starts when this process + /// starts and is not shared between replicas. + #[arg(long, default_value = "60", env = "AWA_QUEUE_HISTORY_MINUTES")] + queue_history_minutes: u64, + /// Seconds between queue depth samples. Each sample is one queue + /// overview read, issued whether or not anyone has the UI open. + #[arg( + long, + default_value = "15", + env = "AWA_QUEUE_HISTORY_INTERVAL", + value_parser = clap::value_parser!(u64).range(1..) + )] + queue_history_interval: u64, }, /// Callback receiver subcommands Callbacks { @@ -1195,6 +1209,8 @@ async fn run() -> Result<(), Box> { instance_name, instance_color, peer, + queue_history_minutes, + queue_history_interval, } => { // Validate identity flags before touching the database so a // typo'd color or peer spec fails fast. @@ -1232,12 +1248,18 @@ async fn run() -> Result<(), Box> { } else { awa_ui::state::ReadOnlyMode::Auto }; - let app = awa_ui::router_with_identity( + let app = awa_ui::router_with_config( pool, - cache_duration, - callback_hmac_secret, - read_only_mode, - instance, + awa_ui::RouterConfig { + cache_ttl: cache_duration, + callback_hmac_secret, + read_only_mode, + instance, + queue_history: awa_ui::history::QueueHistoryConfig { + window: Duration::from_secs(queue_history_minutes.saturating_mul(60)), + sample_interval: Duration::from_secs(queue_history_interval), + }, + }, ) .await?; let addr = format!("{host}:{port}"); diff --git a/awa-ui/frontend/e2e/dashboard.spec.ts b/awa-ui/frontend/e2e/dashboard.spec.ts index 7676db74c..a52c48787 100644 --- a/awa-ui/frontend/e2e/dashboard.spec.ts +++ b/awa-ui/frontend/e2e/dashboard.spec.ts @@ -1,76 +1,93 @@ -import { test, expect } from "@playwright/test"; +import { test, expect, type Page } from "@playwright/test"; + +/** The overview lists only queues needing attention until expanded; seeded + * queues may or may not qualify depending on their age, so expand first. */ +async function showAllQueues(page: Page) { + await expect(page.getByTestId("verdict")).not.toHaveText("Checking queues…"); + const expand = page.getByRole("button", { + name: /^Show \d+ queues? keeping up$/, + }); + if (await expand.isVisible()) await expand.click(); + return page.getByRole("table", { + name: /^(All queues by risk|Queues needing attention)$/, + }); +} test.describe("Operations overview", () => { - test("overview separates current work and last-hour completions", async ({ - page, - }) => { + test("answers whether the system is keeping up", async ({ page }) => { await page.goto("/"); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + await expect(page.getByText("Is it keeping up?")).toBeVisible(); + await expect(page.getByTestId("verdict")).not.toHaveText( + "Checking queues…", + ); + await expect(page.getByText("Waiting across all queues")).toBeVisible(); await expect( - page.getByRole("heading", { name: "Work in motion" }), - ).toBeVisible(); - await expect( - page.getByRole("heading", { name: "Current work" }), - ).toBeVisible(); - await expect( - page.getByText("Completed · last 60 minutes", { exact: true }), + page.getByText( + /Trends from samples every|Queue history starts with the first sample/, + ), ).toBeVisible(); - await expect(page.locator(".work-counter")).toHaveCount(6); }); - test("queue pressure includes available wait and state drilldowns", async ({ + test("queue table includes trend columns and state drilldowns", async ({ page, }) => { await page.goto("/"); - const table = page.getByRole("table", { name: "Queue pressure" }); + const table = await showAllQueues(page); await expect(table).toBeVisible(); for (const header of [ "Queue", - "Available", - "Oldest available", - "Running", - "Retryable / failed", + "Status", + "Waiting", + "Last hour", + "Trend", + "Oldest wait", ]) { await expect( table.getByRole("columnheader", { name: header, exact: true }), ).toBeVisible(); } - const row = table.getByRole("row", { name: /e2e_test/ }); + const row = table.getByRole("row", { name: /E2E Queue/ }); await row.locator('a[href*="state=available"]').click(); await expect(page).toHaveURL(/state=available/); await expect(page).toHaveURL(/e2e_test/); }); - for (const state of ["available", "retryable", "failed"]) { - test(`${state} counter links to its job state`, async ({ page }) => { + for (const [label, state] of [ + ["Retrying", "retryable"], + ["Failed", "failed"], + ["Running", "running"], + ] as const) { + test(`${label} counter links to its job state`, async ({ page }) => { await page.goto("/"); - await page.locator(`.work-counter[href*="state=${state}"]`).click(); + await page + .getByRole("navigation", { name: "Job states" }) + .getByRole("link", { name: new RegExp(`^${label}`) }) + .click(); await expect(page).toHaveURL(new RegExp(`state=${state}`)); await expect(page.getByRole("heading", { name: "Jobs" })).toBeVisible(); }); } - test("worker summary reports health or explicit empty state", async ({ + test("worker summary reports problems or an explicit healthy state", async ({ page, }) => { await page.goto("/"); - const section = page.getByRole("region", { name: "Worker health" }); + await expect(page.getByRole("link", { name: "All workers" })).toBeVisible(); await expect( - section.getByRole("link", { name: "All workers" }), + page + .getByText( + /Workers needing attention|reporting instances are healthy|No runtime snapshots/, + ) + .or(page.getByRole("list", { name: "Workers needing attention" })), ).toBeVisible(); - await expect(section).toContainText( - /healthy.*live.*stale.*leader|No runtime snapshots/, - ); }); - test("failed job drilldown opens the retained failure", async ({ page }) => { + test("failure group opens the latest retained failure", async ({ page }) => { await page.goto("/"); - const section = page.getByRole("region", { - name: "Failed jobs to inspect", - }); - await expect( - section.getByText(/Latest 5 retained failed jobs/), - ).toBeVisible(); - await section.locator(".failure-preview-row").first().click(); + const groups = page.getByRole("list", { name: "Failure groups" }); + await expect(groups).toBeVisible(); + await groups.getByRole("link").first().click(); await expect(page).toHaveURL(/\/jobs\/\d+/); await expect(page.getByRole("heading", { name: /Job #\d+/ })).toBeVisible(); }); diff --git a/awa-ui/frontend/e2e/navigation.spec.ts b/awa-ui/frontend/e2e/navigation.spec.ts index 753b5da45..01ecc19e5 100644 --- a/awa-ui/frontend/e2e/navigation.spec.ts +++ b/awa-ui/frontend/e2e/navigation.spec.ts @@ -40,7 +40,7 @@ test.describe("Shell navigation", () => { await nav.getByRole("link", { name: "Overview" }).click(); await expect(page).toHaveURL(/\/$/); await expect( - page.getByRole("heading", { name: "Work in motion" }) + page.getByRole("heading", { name: "Overview" }) ).toBeVisible(); }); @@ -55,7 +55,7 @@ test.describe("Shell navigation", () => { await logoLink.click(); await expect(page).toHaveURL(/\/$/); await expect( - page.getByRole("heading", { name: "Work in motion" }) + page.getByRole("heading", { name: "Overview" }) ).toBeVisible(); }); diff --git a/awa-ui/frontend/e2e/operations-fixture.ts b/awa-ui/frontend/e2e/operations-fixture.ts index 32b95966d..70d730fed 100644 --- a/awa-ui/frontend/e2e/operations-fixture.ts +++ b/awa-ui/frontend/e2e/operations-fixture.ts @@ -28,14 +28,17 @@ export async function mockOperations( descriptor_stale: false, descriptor_mismatch: false, total_queued: 0, - available: index === 0 ? 184200 : 2100 + index * 17, - running: 40 + index, + // Story: email-delivery started falling behind 20 minutes ago, + // image-processing has no worker claiming it, search-index is draining a + // burst, and invoice-export is paused with work waiting. + available: index === 0 ? 184200 : index === 4 ? 12168 : 2100 + index * 17, + running: index === 3 ? 0 : 40 + index, retryable: index === 0 ? 1840 : 10, failed: index === 0 ? 236 : 0, scheduled: 850 + index, waiting_external: index === 2 ? 14400 : 0, completed_last_hour: 128400 + index * 211, - lag_seconds: index === 0 ? 458 : index === 1 ? 135 : 2.4, + lag_seconds: index === 0 ? 458 : index === 1 ? 135 : index === 3 ? 95 : 2.4, paused: index === 1, })).map((q) => ({ ...q, @@ -75,20 +78,69 @@ export async function mockOperations( shutting_down: false, leader: index === 2, global_max_workers: 64, - queues: queues - .slice(0, 6) - .map((q) => ({ - queue: q.queue, - in_flight: 8, - overflow_held: 0, - config: config as RuntimeInstance["queues"][number]["config"], - })), + queues: queues.slice(0, 6).map((q) => ({ + queue: q.queue, + in_flight: 8, + overflow_held: 0, + config: config as RuntimeInstance["queues"][number]["config"], + })), }), ); + // Bucketed history the way the server reports it: an hour of one-minute + // buckets, a server that started 50 minutes ago, and a three-minute gap + // where sampling failed. + const BUCKETS = 60; + const depth = (index: number, minutesAgo: number): number => { + const current = queues[index]!.available; + if (index === 0) + return minutesAgo <= 20 + ? current - minutesAgo * 2500 + : 134200 + (minutesAgo % 3) * 300; + if (index === 1) + return minutesAgo <= 12 ? current : 40 + (minutesAgo % 4) * 5; + if (index === 3) + return minutesAgo <= 2 + ? current - minutesAgo * 900 + : 30 + (minutesAgo % 2) * 10; + if (index === 4) + return minutesAgo <= 15 ? current + minutesAgo * 2000 : 4000; + return current + ((minutesAgo * 7 + index) % 5) * 4; + }; + const queueHistory = { + enabled: true, + window_secs: 3600, + sample_interval_secs: 15, + bucket_secs: 60, + observed_since: ago(50 * 60), + buckets: Array.from({ length: BUCKETS }, (_, bucket) => + ago((BUCKETS - 1 - bucket) * 60), + ), + queues: Object.fromEntries( + queues.map((q, index) => { + const sampled = (bucket: number) => + bucket >= 10 && ![33, 34, 35].includes(bucket); + const minutesAgo = (bucket: number) => BUCKETS - 1 - bucket; + return [ + q.queue, + { + available: Array.from({ length: BUCKETS }, (_, bucket) => + sampled(bucket) ? depth(index, minutesAgo(bucket)) : null, + ), + running: Array.from({ length: BUCKETS }, (_, bucket) => + sampled(bucket) ? q.running : null, + ), + lag_seconds: Array.from({ length: BUCKETS }, (_, bucket) => + sampled(bucket) ? q.lag_seconds : null, + ), + }, + ]; + }), + ), + }; const jobs: JobRow[] = Array.from({ length: 60 }, (_, index) => ({ id: 984213 - index, - queue: "email-delivery", - kind: "SendTransactionalEmail", + queue: index % 7 === 3 ? "image-processing" : "email-delivery", + kind: index % 7 === 3 ? "ResizeImage" : "SendTransactionalEmail", args: { template: "receipt", synthetic: true }, state: "failed", priority: 2, @@ -103,7 +155,10 @@ export async function mockOperations( created_at: ago(1000), errors: [ { - error: "Provider request timed out after 30 seconds", + error: + index % 7 === 3 + ? `Out of memory decoding upload ${7_310_000 + index}` + : "Provider request timed out after 30 seconds", attempt: 5, at: ago(8), }, @@ -160,7 +215,14 @@ export async function mockOperations( await new Promise((resolve) => setTimeout(resolve, 2500)); if ( failReads && - ["/stats", "/queues", "/runtime", "/jobs", "/dlq/depth"].includes(path) + [ + "/stats", + "/queues", + "/stats/queue-history", + "/runtime", + "/jobs", + "/dlq/depth", + ].includes(path) ) return route.fulfill({ status: 503, @@ -181,6 +243,10 @@ export async function mockOperations( completed: 14820000, }; else if (path === "/queues") data = empty ? [] : queues; + else if (path === "/stats/queue-history") + data = empty + ? { ...queueHistory, observed_since: null, buckets: [], queues: {} } + : queueHistory; else if (path === "/queues/runtime") data = empty ? [] diff --git a/awa-ui/frontend/e2e/operations.spec.ts b/awa-ui/frontend/e2e/operations.spec.ts index feb1ac992..f53e581b4 100644 --- a/awa-ui/frontend/e2e/operations.spec.ts +++ b/awa-ui/frontend/e2e/operations.spec.ts @@ -11,22 +11,38 @@ test("busy overview communicates current work and opens targeted drilldowns", as page.on("pageerror", (error) => errors.push(error.message)); await page.setViewportSize({ width: 1440, height: 1050 }); await page.goto("/"); - await expect( - page.getByRole("heading", { name: "Work in motion" }), - ).toBeVisible(); - await expect( - page.locator(".work-counter").filter({ hasText: "Available" }), - ).toContainText("237,192"); - await expect( - page.locator(".work-counter").filter({ hasText: "Retryable" }), - ).toContainText("2,070"); - await expect(page.locator(".worker-preview-row")).toHaveCount(4); - await expect(page.locator(".worker-preview-row").first()).toContainText( - "health unknown", - ); - await expect(page.locator(".pressure-table tbody tr").first()).toContainText( - "7m 38s", + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", ); + await expect(page.getByText(/1 queue falling behind/)).toBeVisible(); + const attention = page + .getByRole("table", { name: "Queues needing attention" }) + .locator("tbody tr"); + await expect(attention).toHaveCount(3); + await expect(attention.nth(0)).toContainText("image-processing"); + await expect(attention.nth(0)).toContainText("Not being worked"); + await expect(attention.nth(1)).toContainText("email-delivery"); + await expect(attention.nth(1)).toContainText("Falling behind"); + await expect(attention.nth(1)).toContainText("+2,500/min"); + await expect(attention.nth(1)).toContainText("7m 38s"); + await expect(attention.nth(2)).toContainText("Paused"); + await page.getByRole("button", { name: "Show 21 queues keeping up" }).click(); + const everything = page + .getByRole("table", { name: "All queues by risk" }) + .locator("tbody tr"); + await expect(everything).toHaveCount(24); + await expect(everything.nth(3)).toContainText("search-index"); + await expect(everything.nth(3)).toContainText("Draining"); + await expect( + page.getByRole("navigation", { name: "Job states" }), + ).toContainText("Retrying2,070"); + const workers = page.getByRole("list", { name: "Workers needing attention" }); + await expect(workers.getByRole("listitem")).toHaveCount(2); + await expect(workers).toContainText("health unknown"); + const failures = page.getByRole("list", { name: "Failure groups" }); + await expect(failures.getByRole("listitem")).toHaveCount(2); + await expect(failures.getByRole("listitem").first()).toContainText("×43"); await page.screenshot({ path: `${artifacts}/overview-desktop.png`, fullPage: true, @@ -43,9 +59,7 @@ test("busy overview communicates current work and opens targeted drilldowns", as await page.getByRole("link", { name: "worker-eu-01" }).click(); await expect(page).toHaveURL(/runtime\/00000000/); await page.goBack(); - await expect( - page.getByRole("heading", { name: "Work in motion" }), - ).toBeVisible(); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); } expect(fixture.mutations).toEqual([]); expect(errors).toEqual([]); @@ -57,7 +71,9 @@ test("queue filters and core controls use only the synthetic API", async ({ const fixture = await mockOperations(page); await page.goto("/queues"); await page.getByRole("searchbox", { name: "Find a queue" }).fill("invoice"); - await expect(page.locator(".pressure-table tbody tr")).toHaveCount(1); + const rows = page.getByRole("table", { name: "Queues" }).locator("tbody tr"); + await expect(rows).toHaveCount(1); + await expect(rows).toContainText("max 64"); await page.getByRole("button", { name: "Resume", exact: true }).click(); await expect( page.getByRole("button", { name: "Pause", exact: true }), @@ -74,16 +90,24 @@ test("queue filters and core controls use only the synthetic API", async ({ expect(fixture.mutations).toHaveLength(2); await page.getByRole("button", { name: "Drain…" }).click(); await page.getByRole("button", { name: "Drain queue", exact: true }).click(); - await expect(page.locator(".pressure-table tbody tr")).toContainText( - "0 scheduled", - ); + await expect(rows.locator("td").nth(1)).toHaveText("0"); + await expect(page).toHaveURL(/q=invoice/); expect(fixture.mutations).toContain("POST /queues/invoice-export/drain"); await page.getByRole("searchbox").fill("not-a-real-queue"); await expect(page.getByText("No queues match these filters.")).toBeVisible(); await page.getByRole("searchbox").fill(""); - await page.getByRole("combobox", { name: "Sort by" }).selectOption("failed"); - await expect(page.locator(".pressure-table tbody tr").first()).toContainText( - "email-delivery", + await page + .getByRole("combobox", { name: "Sort queues" }) + .selectOption("failed"); + await expect(rows.first()).toContainText("email-delivery"); + // The drain above emptied the paused queue, so only the stalled and the + // falling-behind queue still need attention. + await page.getByText("Needs attention only").click(); + await expect(rows).toHaveCount(2); + await page.reload(); + await expect(rows).toHaveCount(2); + await expect(page.getByRole("combobox", { name: "Sort queues" })).toHaveValue( + "failed", ); await page.setViewportSize({ width: 1440, height: 1050 }); await page.screenshot({ @@ -108,7 +132,9 @@ test("mobile navigation, controls, job drilldown and layout at narrow sizes", as await mockOperations(page); await page.setViewportSize({ width: 390, height: 844 }); await page.goto("/"); - await expect(page.locator(".work-counter").first()).toContainText("237,192"); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); await page.screenshot({ path: `${artifacts}/overview-mobile.png`, fullPage: true, @@ -124,9 +150,7 @@ test("mobile navigation, controls, job drilldown and layout at narrow sizes", as await expect(page.getByRole("dialog")).toHaveCount(0); await page.getByRole("button", { name: "Toggle Sidebar" }).click(); await page.getByRole("link", { name: "Overview", exact: true }).click(); - await expect( - page.getByRole("heading", { name: "Work in motion" }), - ).toBeVisible(); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); await expect(page.getByRole("dialog")).toHaveCount(0); } await page.goto("/queues"); @@ -162,7 +186,7 @@ test("mobile navigation, controls, job drilldown and layout at narrow sizes", as await page.goto(route); const heading = route === "/" - ? "Work in motion" + ? "Overview" : route === "/queues" ? "Queues" : route === "/jobs" @@ -192,8 +216,8 @@ test("mobile navigation, controls, job drilldown and layout at narrow sizes", as page.getByRole("link", { name: "email-delivery", exact: true }), ).toBeVisible(); if (route === "/") - await expect(page.locator(".work-counter").first()).toContainText( - "237,192", + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", ); await page.evaluate(() => document.fonts.ready); if ( @@ -233,7 +257,9 @@ for (const scenario of ["empty", "error", "loading"] as const) { ).toBeVisible(); if (scenario === "error") { await expect(page.getByRole("alert")).toBeVisible({ timeout: 15000 }); - await expect(page.locator(".work-counter").first()).toContainText("—"); + await expect( + page.getByRole("navigation", { name: "Job states" }), + ).toContainText("Running—"); } if (scenario === "loading") await expect(page.getByText("Loading snapshots…")).toBeVisible(); @@ -249,13 +275,14 @@ test("failed refresh retains known values and labels them out of date", async ({ }) => { const fixture = await mockOperations(page); await page.goto("/"); - await expect(page.locator(".work-counter").first()).toContainText("237,192"); + const verdict = page.getByTestId("verdict"); + await expect(verdict).toHaveText("1 queue not being worked"); fixture.setReadFailure(true); await page.getByRole("button", { name: "Refresh data", exact: true }).click(); await expect(page.getByText(/Refresh failed ·/)).toBeVisible({ timeout: 15000, }); - await expect(page.locator(".work-counter").first()).toContainText("237,192"); + await expect(verdict).toHaveText("1 queue not being worked"); await page.screenshot({ path: `${artifacts}/stale-desktop.png`, fullPage: true, @@ -310,7 +337,7 @@ test("mobile drawer closes promptly with both motion preferences", async ({ ); await expect( page.getByRole("heading", { - name: name === "Overview" ? "Work in motion" : name, + name, exact: true, }), ).toBeVisible(); diff --git a/awa-ui/frontend/e2e/queues.spec.ts b/awa-ui/frontend/e2e/queues.spec.ts index a5c8be72b..3020e275c 100644 --- a/awa-ui/frontend/e2e/queues.spec.ts +++ b/awa-ui/frontend/e2e/queues.spec.ts @@ -26,16 +26,19 @@ test.describe("Queues page", () => { await loadQueuesPage(page); // Desktop table (visible at default viewport) - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect(queueTable).toBeVisible(); for (const header of [ "Queue", - "Available", - "Oldest available", + "Status", + "Waiting", + "Trend", + "Oldest wait", "Running", - "Retryable / failed", - "Completed last hour", + "Capacity", + "Done/min", + "Retrying / failed", "Controls", ]) { await expect( @@ -51,20 +54,20 @@ test.describe("Queues page", () => { ).toBeVisible(); }); - test("queue table shows descriptor description", async ({ page }) => { + test("queue table carries the descriptor description", async ({ page }) => { await loadQueuesPage(page); - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect( - queueTable.getByText("End-to-end queue used for UI coverage"), - ).toBeVisible(); + queueTable.getByRole("link", { name: "E2E Queue" }), + ).toHaveAttribute("title", "End-to-end queue used for UI coverage"); }); test("queue table surfaces available wait and controls", async ({ page }) => { await loadQueuesPage(page); - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect( - queueTable.getByRole("columnheader", { name: "Oldest available" }), + queueTable.getByRole("columnheader", { name: "Oldest wait" }), ).toBeVisible(); await expect( queueTable.getByRole("columnheader", { name: "Controls" }), @@ -135,7 +138,7 @@ test.describe("Queues page", () => { await page.goto("/queues"); - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect( queueTable.getByRole("rowheader", { name: /DLQ Off Queue/ }), ).toBeVisible(); @@ -147,7 +150,7 @@ test.describe("Queues page", () => { test("click queue navigates to queue detail", async ({ page }) => { await loadQueuesPage(page); - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); const queueLink = queueTable.getByRole("link", { name: "E2E Queue" }); await expect(queueLink).toBeVisible(); @@ -313,18 +316,18 @@ test.describe("Queues page", () => { }); await page.goto("/queues"); - const queueTable = page.getByRole("table", { name: "Queue pressure" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect( queueTable.getByText("Descriptor stale", { exact: true }), ).toBeVisible(); await expect( - queueTable.getByText("Descriptor drift across live runtimes"), + queueTable.getByText("Descriptor drift", { exact: true }), ).toBeVisible(); }); function queueRow(page: Page) { return page - .getByRole("table", { name: "Queue pressure" }) + .getByRole("table", { name: "Queues" }) .getByRole("row", { name: /E2E Queue/ }); } diff --git a/awa-ui/frontend/src/__tests__/operations.test.ts b/awa-ui/frontend/src/__tests__/operations.test.ts new file mode 100644 index 000000000..b1f08d275 --- /dev/null +++ b/awa-ui/frontend/src/__tests__/operations.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect } from "vitest"; +import type { JobRow, QueueHistory, QueueSeries, QueueStats } from "@/lib/api"; +import { + assessQueue, + assessQueues, + formatDuration, + formatRate, + groupFailures, + overallVerdict, + totalBacklog, + trailingTrend, +} from "@/lib/operations"; + +function queue(overrides: Partial = {}): QueueStats { + return { + queue: "email", + display_name: null, + description: null, + owner: null, + docs_url: null, + tags: [], + extra: {}, + descriptor_last_seen_at: null, + descriptor_stale: false, + descriptor_mismatch: false, + total_queued: 0, + scheduled: 0, + available: 1000, + retryable: 0, + running: 10, + failed: 0, + waiting_external: 0, + completed_last_hour: 600, + lag_seconds: 5, + paused: false, + ...overrides, + }; +} + +function series(available: (number | null)[]): QueueSeries { + return { + available, + running: available.map(() => 10), + lag_seconds: available.map(() => 5), + }; +} + +function history( + queues: Record, + enabled = true, +): QueueHistory { + const length = Object.values(queues)[0]?.length ?? 0; + return { + enabled, + window_secs: 3600, + sample_interval_secs: 15, + bucket_secs: 60, + observed_since: "2026-10-25T00:00:00Z", + buckets: Array.from( + { length }, + (_, index) => `2026-10-25T00:${String(index).padStart(2, "0")}:00Z`, + ), + queues: Object.fromEntries( + Object.entries(queues).map(([name, values]) => [name, series(values)]), + ), + }; +} + +describe("trailingTrend", () => { + it("needs three samples spanning two minutes", () => { + expect(trailingTrend([null, null, 10, 20], 60)).toBeNull(); + expect(trailingTrend([10, 20, 30], 30)).toBeNull(); + expect(trailingTrend([10, 20, 30], 60)).not.toBeNull(); + }); + + it("fits the slope per minute over the trailing window only", () => { + // An old spike outside the 5 minute window must not affect the fit. + const trend = trailingTrend([9000, 0, 100, 200, 300, 400, 500], 60)!; + expect(trend.perMinute).toBeCloseTo(100); + expect(trend.minutes).toBe(5); + expect(trend.change).toBeCloseTo(500); + }); + + it("skips gaps instead of treating them as zero", () => { + const trend = trailingTrend([100, null, 300, null, 500, 600], 60)!; + expect(trend.perMinute).toBeCloseTo(100); + }); +}); + +describe("assessQueue", () => { + it("flags waiting work with nothing running as stalled before anything else", () => { + const result = assessQueue( + queue({ running: 0, lag_seconds: 90 }), + series([100, 200, 300, 400, 500, 600]), + 60, + ); + expect(result.health).toBe("stalled"); + expect(result.needsAttention).toBe(true); + }); + + it("does not call a fresh queue stalled before the wait threshold", () => { + const result = assessQueue( + queue({ running: 0, lag_seconds: 10 }), + undefined, + 60, + ); + expect(result.health).toBe("unknown"); + }); + + it("reports a materially growing backlog as falling behind", () => { + const result = assessQueue( + queue({ available: 1500 }), + series([1000, 1100, 1200, 1300, 1400, 1500]), + 60, + ); + expect(result.health).toBe("behind"); + expect(result.detail).toContain("+100/min"); + }); + + it("treats small wobble on a large backlog as steady", () => { + const result = assessQueue( + queue({ available: 184_000 }), + series([184_000, 184_400, 183_900, 184_300, 184_100, 184_000]), + 60, + ); + expect(result.health).toBe("steady"); + expect(result.needsAttention).toBe(false); + }); + + it("estimates time to clear a draining backlog", () => { + const result = assessQueue( + queue({ available: 600 }), + series([1600, 1400, 1200, 1000, 800, 600]), + 60, + ); + expect(result.health).toBe("draining"); + expect(result.clearSeconds).toBeCloseTo(180); + expect(result.detail).toBe("Clears in about 3m 0s at this pace"); + }); + + it("flags a long wait even when depth holds", () => { + const result = assessQueue( + queue({ lag_seconds: 900 }), + series([1000, 1000, 1000, 1000]), + 60, + ); + expect(result.health).toBe("slow"); + }); + + it("only asks for attention on paused queues holding work", () => { + expect( + assessQueue(queue({ paused: true }), undefined, 60).needsAttention, + ).toBe(true); + const empty = assessQueue( + queue({ paused: true, available: 0 }), + undefined, + 60, + ); + expect(empty.needsAttention).toBe(false); + expect(empty.rank).toBeGreaterThan( + assessQueue(queue(), undefined, 60).rank, + ); + }); + + it("says why there is no trend when history is disabled", () => { + expect(assessQueue(queue(), undefined, 60, false).detail).toMatch( + /disabled/, + ); + }); +}); + +describe("assessQueues and overallVerdict", () => { + it("orders by risk, then backlog", () => { + const rows = assessQueues( + [ + queue({ queue: "steady", available: 9000 }), + queue({ queue: "stalled", running: 0, lag_seconds: 120 }), + queue({ queue: "idle", available: 0 }), + queue({ queue: "behind", available: 500 }), + ], + history({ + steady: [9000, 9000, 9000, 9000], + stalled: [1000, 1000, 1000, 1000], + idle: [0, 0, 0, 0], + behind: [100, 200, 300, 400, 500], + }), + ); + expect(rows.map((row) => row.queue.queue)).toEqual([ + "stalled", + "behind", + "steady", + "idle", + ]); + const verdict = overallVerdict(rows, true); + expect(verdict.tone).toBe("danger"); + expect(verdict.title).toBe("1 queue not being worked"); + expect(verdict.detail).toContain("1 queue falling behind"); + }); + + it("is honest when there is not enough history to judge", () => { + const rows = assessQueues([queue()], null); + expect(overallVerdict(rows, false).title).toBe("Not enough history yet"); + }); + + it("reports keeping up when nothing is wrong", () => { + const rows = assessQueues( + [queue({ available: 0 })], + history({ email: [0, 0, 0] }), + ); + expect(overallVerdict(rows, true)).toMatchObject({ + tone: "success", + title: "Keeping up", + }); + }); +}); + +describe("totalBacklog", () => { + it("sums per bucket and keeps buckets with no samples null", () => { + expect( + totalBacklog(history({ a: [null, 1, 2], b: [null, null, 3] })), + ).toEqual([null, 1, 5]); + }); +}); + +describe("groupFailures", () => { + function failed( + id: number, + message: string, + finalized: string, + kind = "SendEmail", + ): JobRow { + return { + id, + kind, + queue: "email", + errors: [{ error: message }], + finalized_at: finalized, + } as unknown as JobRow; + } + + it("groups by kind, queue and error, masking per-job identifiers", () => { + const groups = groupFailures([ + failed(1, "timed out calling order 18273645", "2026-10-25T00:00:01Z"), + failed(2, "timed out calling order 99887766", "2026-10-25T00:00:03Z"), + failed( + 3, + "timed out calling order 12345678", + "2026-10-25T00:00:02Z", + "Other", + ), + ]); + expect(groups).toHaveLength(2); + expect(groups[0]).toMatchObject({ kind: "SendEmail", count: 2 }); + expect(groups[0]!.latest.id).toBe(2); + expect(groups[0]!.message).toBe("timed out calling order 99887766"); + }); +}); + +describe("formatting", () => { + it("formats durations consistently across units", () => { + expect(formatDuration(2.4)).toBe("2s"); + expect(formatDuration(458)).toBe("7m 38s"); + expect(formatDuration(3720)).toBe("1h 2m"); + expect(formatDuration(null)).toBe("—"); + }); + + it("formats signed rates", () => { + expect(formatRate(1234, true)).toBe("+1,234/min"); + expect(formatRate(-0.5, true)).toBe("−0.5/min"); + expect(formatRate(52_000)).toBe("52K/min"); + }); +}); diff --git a/awa-ui/frontend/src/app.css b/awa-ui/frontend/src/app.css index 28486f5f6..0e4a424c0 100644 --- a/awa-ui/frontend/src/app.css +++ b/awa-ui/frontend/src/app.css @@ -308,130 +308,9 @@ If you want to use the typography plugin (prose), uncomment the line below. transform: translateZ(0); } -/* Operator views */ -.operations-overview { display: grid; gap: 24px; min-width: 0; } -.page-intro { display: flex; justify-content: space-between; align-items: center; gap: 16px; } -.page-intro h1 { font-size: clamp(26px, 3vw, 34px); line-height: 1.2; font-weight: 650; letter-spacing: -.045em; margin: 6px 0 8px; } -.page-intro p:not(.eyebrow) { color: var(--muted-fg); font-size: 14px; } -.eyebrow { font-size: 11px; text-transform: uppercase; letter-spacing: .12em; font-weight: 650; color: var(--muted-fg); } -.operator-link { color: var(--primary); font-size: 13px; font-weight: 600; white-space: nowrap; display: inline-flex; align-items: center; min-height: 44px; } -.snapshot-status { display: flex; gap: 8px; align-items: center; color: var(--muted-fg); font-size: 12px; margin-top: -10px; } -.status-dot { flex: 0 0 6px; height: 6px; border-radius: 100%; background: currentColor; } -.snapshot-warning { color: var(--danger-subtle-fg); } -.source-error { padding: 14px 18px; border: 1px solid var(--danger); background: var(--danger-subtle); color: var(--danger-subtle-fg); border-radius: 8px; font-size: 14px; } -.flow-summary { display: grid; grid-template-columns: 1.3fr 1fr; border: 1px solid var(--border); border-radius: 12px; overflow: hidden; } -.completion-summary { padding: 24px; background: var(--primary-subtle); } -.completion-summary .eyebrow { color: var(--primary-subtle-fg); } -.completion-value { display: block; font-size: clamp(32px, 4vw, 44px); font-weight: 600; letter-spacing: -.045em; font-variant-numeric: tabular-nums; margin: 8px 0; line-height: 1.2; } -.completion-value span { font-size: 15px; letter-spacing: 0; font-weight: 400; color: var(--muted-fg); } -.completion-summary > p:last-child { font-size: 12px; color: var(--muted-fg); max-width: 400px; line-height: 1.6; } -.attention-summary { padding: 20px 24px; background: var(--bg); } -.attention-summary > p { margin-bottom: 8px; } -.attention-summary > a { display: grid; grid-template-columns: minmax(42px, auto) 1fr 16px; align-items: center; gap: 12px; min-height: 40px; font-size: 13px; } -.attention-summary > a:hover { color: var(--primary); } -.attention-summary strong { font-variant-numeric: tabular-nums; font-size: 18px; font-weight: 600; } -.attention-summary > a > span:last-child { color: var(--muted-fg); } -.section-label { display: flex; justify-content: space-between; gap: 12px; margin-bottom: 12px; align-items: baseline; } -.section-label h2, .panel-heading h2 { font-weight: 600; font-size: 16px; letter-spacing: -.02em; } -.section-label > span { font-size: 12px; color: var(--muted-fg); } -.work-counters { display: grid; grid-template-columns: repeat(6, minmax(0, 1fr)); border: 1px solid var(--border); border-radius: 10px; overflow: hidden; background: var(--bg); } -.work-counter { padding: 16px; display: flex; flex-direction: column; gap: 7px; border-right: 1px solid var(--border); min-width: 0; } -.work-counter:last-child { border: 0; } -.work-counter:hover { background: var(--muted); } -.work-counter > span { font-size: 12px; font-weight: 550; } -.work-counter strong { font-size: clamp(18px, 1.8vw, 26px); font-weight: 600; letter-spacing: -.04em; font-variant-numeric: tabular-nums; } -.work-counter small { color: var(--muted-fg); font-size: 11px; line-height: 1.5; } -.counter-failed strong, .failure-value { color: var(--danger-subtle-fg); } -.counter-retryable strong, .retry-value { color: var(--warning-subtle-fg); } -.operations-panel { min-width: 0; border: 1px solid var(--border); border-radius: 10px; background: var(--bg); overflow: hidden; } -.panel-heading { padding: 18px 20px; display: flex; align-items: center; justify-content: space-between; gap: 12px; border-bottom: 1px solid var(--border); } -.panel-heading p { font-size: 12px; color: var(--muted-fg); margin-top: 5px; line-height: 1.5; } -.panel-footnote { padding: 12px 20px; border-top: 1px solid var(--border); font-size: 11px; line-height: 1.6; color: var(--muted-fg); background: var(--muted); } -.panel-empty { padding: 32px 20px; color: var(--muted-fg); font-size: 14px; } -.queue-board { overflow-x: auto; } -.pressure-table { width: 100%; border-collapse: collapse; text-align: right; font-size: 13px; } -.pressure-table thead { background: var(--muted); } -.pressure-table thead th { font-size: 11px; color: var(--muted-fg); font-weight: 500; white-space: nowrap; } -.pressure-table thead th span { display: block; } -.pressure-table th, .pressure-table td { padding: 14px 16px; font-variant-numeric: tabular-nums; vertical-align: middle; } -.pressure-table th:first-child { text-align: left; } -.pressure-table tbody tr + tr { border-top: 1px solid var(--border); } -.pressure-table tbody tr:hover { background: var(--muted); } -.pressure-table td { white-space: nowrap; } -.pressure-table td a { display: inline-flex; align-items: center; min-height: 32px; } -.pressure-table a:hover { text-decoration: underline; text-underline-offset: 3px; } -.queue-identity { min-width: 200px; max-width: 330px; font-weight: 500; overflow-wrap: anywhere; } -.queue-identity > a { color: var(--primary); display: inline-flex; align-items: center; min-height: 32px; } -.queue-identity code { display: block; color: var(--muted-fg); font-size: 11px; } -.queue-context { display: flex; align-items: center; flex-wrap: wrap; gap: 6px; color: var(--muted-fg); font-size: 11px; font-weight: 400; line-height: 1.6; } -.signal { border-radius: 4px; font-size: 10px; font-weight: 600; padding: 1px 5px; } -.signal-warning { background: var(--warning-subtle); color: var(--warning-subtle-fg); } -.wait-value { font-weight: 600; } -.queue-action-buttons { display: flex; justify-content: flex-end; gap: 4px; } -.overview-details { display: grid; grid-template-columns: 1fr 1fr; gap: 24px; align-items: start; } -.worker-preview-row { display: flex; align-items: center; justify-content: space-between; padding: 16px 20px; gap: 12px; font-size: 12px; } -.worker-preview-row + .worker-preview-row, .failure-preview-row + .failure-preview-row { border-top: 1px solid var(--border); } -.worker-preview-row:hover, .failure-preview-row:hover { background: var(--muted); } -.worker-preview-row strong { display: block; font-size: 13px; font-weight: 550; margin-bottom: 4px; } -.worker-preview-row > span { text-align: right; color: var(--muted-fg); white-space: nowrap; } -.worker-preview-row > div { min-width: 0; overflow-wrap: anywhere; } -.worker-role { display: block; font-size: 11px; margin-top: 4px; } -.concern-text { color: var(--danger-subtle-fg); } -.healthy-text { color: var(--primary-subtle-fg); } -.failure-preview-row { display: block; padding: 14px 20px; font-size: 12px; overflow-wrap: anywhere; } -.failure-preview-row strong { font-size: 13px; font-weight: 550; } -.failure-preview-row div > span { display: block; color: var(--muted-fg); font-size: 11px; margin-top: 3px; } -.failure-preview-row p { color: var(--danger-subtle-fg); margin-top: 7px; } -.failure-preview-row > span { display: block; font-size: 11px; color: var(--muted-fg); margin-top: 7px; } -.queue-filters { display: flex; flex-wrap: wrap; align-items: end; gap: 16px; } -.queue-filters label { display: grid; gap: 6px; font-size: 12px; font-weight: 500; } -.queue-filters input[type=search], .queue-filters select { border: 1px solid var(--border); background: var(--bg); color: var(--fg); border-radius: 7px; padding: 10px 12px; font-size: 14px; min-height: 44px; } -.queue-filters input[type=search] { min-width: 260px; } -.queue-filters .review-toggle { display: flex; align-items: center; gap: 8px; min-height: 44px; } -.queue-filters input[type=checkbox] { width: 18px; height: 18px; accent-color: var(--primary); } -.operations-overview a:focus-visible, .operations-overview input:focus-visible, .operations-overview select:focus-visible { outline: 2px solid var(--primary); outline-offset: 3px; } -@media (max-width: 1100px) { - .work-counters { grid-template-columns: repeat(3, minmax(0, 1fr)); } - .work-counter:nth-child(-n+3) { border-bottom: 1px solid var(--border); } - .work-counter:nth-child(3n) { border-right: 0; } - .overview-details { grid-template-columns: 1fr; } -} -@media (max-width: 1023px) { - .queue-board { overflow: visible; } - .pressure-table, .pressure-table tbody { display: block; } - .pressure-table thead { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); } - .pressure-table tbody tr { display: grid; grid-template-columns: 1fr 1fr; padding: 12px 16px; gap: 8px 12px; } - .pressure-table th, .pressure-table td { padding: 0; display: block; text-align: left; } - .pressure-table .queue-identity { grid-column: 1 / -1; max-width: none; min-width: 0; margin-bottom: 4px; } - .pressure-table .queue-identity > a { min-height: 44px; } - .pressure-table td::before { content: attr(data-label); display: block; font-size: 11px; font-weight: 400; color: var(--muted-fg); margin-bottom: 2px; } - .pressure-table td a { min-height: 44px; } - .pressure-table .queue-controls { grid-column: 1 / -1; } - .pressure-table .queue-controls::before { content: none; } - .queue-action-buttons { justify-content: flex-start; margin-top: 6px; } -} +/* Touch targets on phones: header controls and in-page buttons stay at + least 44px, and long unbroken identifiers wrap instead of widening the page. */ @media (max-width: 767px) { - .flow-summary { grid-template-columns: 1fr; } - .completion-summary { padding: 20px; } - .attention-summary { border-top: 1px solid var(--border); padding: 16px 20px; } - .attention-summary > a { min-height: 44px; } - .operations-overview { gap: 20px; } - .page-intro { align-items: start; } - .page-intro > .operator-link { display: none; } - .section-label { flex-direction: column; gap: 3px; } - .work-counters { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .work-counter { padding: 14px; } - .work-counter:nth-child(3n) { border-right: 1px solid var(--border); } - .work-counter:nth-child(2n) { border-right: 0; } - .work-counter:nth-child(-n+4) { border-bottom: 1px solid var(--border); } - .work-counter strong { font-size: 25px; } - .panel-heading { padding: 16px; align-items: start; } - .panel-footnote { padding: 12px 16px; } - .worker-preview-row, .failure-preview-row { padding: 14px 16px; } - .snapshot-cadence { display: block; } - .queue-filters { gap: 12px; } - .queue-filters > label:not(.review-toggle) { flex: 1 1 100%; } - .queue-filters input[type=search] { min-width: 0; width: 100%; } [data-slot=sidebar-trigger], header button, main button { min-height: 44px; min-width: 44px; } [data-slot=sidebar-item] { min-height: 44px; } main { overflow-wrap: anywhere; } @@ -440,4 +319,3 @@ If you want to use the typography plugin (prose), uncomment the line below. *, *::before, *::after { scroll-behavior: auto !important; animation: none !important; transition: none !important; } } -.dark .operator-link, .dark .queue-identity > a { color: var(--primary-subtle-fg); } diff --git a/awa-ui/frontend/src/components/HistorySpark.tsx b/awa-ui/frontend/src/components/HistorySpark.tsx new file mode 100644 index 000000000..f01ec1ab1 --- /dev/null +++ b/awa-ui/frontend/src/components/HistorySpark.tsx @@ -0,0 +1,91 @@ +/** + * Sparkline for sampled queue history. Unlike `Sparkline`, points keep their + * bucket position (so a gap in sampling shows as a gap, not a shorter line) + * and the y axis starts at zero: a large, steady backlog stays flat instead + * of min-max scaling turning a 1% wobble into a cliff. + */ +export function HistorySpark({ + data, + width = 120, + height = 24, + label, + className, +}: { + data: ReadonlyArray; + width?: number; + height?: number; + label: string; + className?: string; +}) { + const values = data.filter((value): value is number => value !== null); + if (values.length === 0) { + return ( + + ); + } + const max = Math.max(...values) || 1; + const step = data.length > 1 ? width / (data.length - 1) : 0; + const pad = 1.5; + const y = (value: number) => pad + (1 - value / max) * (height - pad * 2); + + const segments: string[][] = []; + let current: string[] = []; + data.forEach((value, index) => { + if (value === null) { + if (current.length) segments.push(current); + current = []; + return; + } + current.push(`${(index * step).toFixed(1)},${y(value).toFixed(1)}`); + }); + if (current.length) segments.push(current); + + return ( + + + {segments.map((points, index) => + points.length === 1 ? ( + + ) : ( + + ), + )} + + ); +} diff --git a/awa-ui/frontend/src/components/QueueFlowTable.tsx b/awa-ui/frontend/src/components/QueueFlowTable.tsx new file mode 100644 index 000000000..813c238e0 --- /dev/null +++ b/awa-ui/frontend/src/components/QueueFlowTable.tsx @@ -0,0 +1,421 @@ +import type { ReactNode } from "react"; +import { Link } from "@tanstack/react-router"; +import type { QueueRuntimeSummary, QueueStats } from "@/lib/api"; +import { Badge } from "@/components/ui/badge"; +import { HistorySpark } from "@/components/HistorySpark"; +import { + capacityLabel, + formatCount, + formatDuration, + formatRate, + type AssessedQueue, + type QueueHealth, +} from "@/lib/operations"; + +export const HEALTH_BADGE: Record< + QueueHealth, + "danger" | "warning" | "info" | "secondary" | "outline" +> = { + stalled: "danger", + behind: "warning", + slow: "warning", + paused: "secondary", + draining: "info", + unknown: "outline", + steady: "outline", + idle: "outline", +}; + +/** Text tint for the backlog sparkline, so the trend reads at a glance. */ +export const SPARK_TONE: Record = { + stalled: "text-danger", + behind: "text-warning", + slow: "text-warning", + paused: "text-muted-fg", + draining: "text-info", + unknown: "text-muted-fg", + steady: "text-muted-fg", + idle: "text-muted-fg", +}; + +function queueName(queue: QueueStats): string { + return queue.display_name?.trim() || queue.queue; +} + +function JobsLink({ + queue, + state, + value, + className = "", +}: { + queue: string; + state: string; + value: number; + className?: string; +}) { + return ( + + {value.toLocaleString()} + + ); +} + +function Capacity({ summary }: { summary: QueueRuntimeSummary | undefined }) { + if (summary && summary.live_instances === 0) { + return No live workers; + } + const title = summary + ? `${summary.healthy_instances}/${summary.live_instances} live instances healthy${summary.config_mismatch ? " · config differs between instances" : ""}` + : "No runtime has reported this queue"; + return ( + + {capacityLabel(summary)} + {summary?.config_mismatch && " · mismatch"} + {summary?.config?.dlq_enabled === false && ( + <> + {" · "} + DLQ off + + )} + + ); +} + +/** Descriptor problems are rare and worth seeing without opening the queue. */ +function DescriptorFlags({ queue }: { queue: QueueStats }) { + return ( + <> + {queue.descriptor_stale && ( + + Descriptor stale + + )} + {queue.descriptor_mismatch && ( + + Descriptor drift + + )} + + ); +} + +function TrendText({ row }: { row: AssessedQueue }) { + const { trend, clearSeconds, health } = row.assessment; + if (!trend) return —; + // Below the noise threshold the fitted slope is batching jitter; printing + // "-0.6/min" would invite reading meaning into it. + if (health === "steady" || health === "idle" || health === "paused") { + return flat; + } + return ( + + {formatRate(trend.perMinute, true)} + {clearSeconds !== null && ( + + {" "} + · ~{formatDuration(clearSeconds)} + + )} + + ); +} + +/** + * One line per queue, ordered by the caller. Switches to stacked cards when + * its own container is narrow (not the viewport: the docked sidebar takes a + * fixed share of a tablet screen). + */ +export function QueueFlowTable({ + rows, + controls, + runtime, + label, +}: { + rows: ReadonlyArray; + controls?: (queue: QueueStats) => ReactNode; + /** When given, adds a capacity column from live runtime config. */ + runtime?: ReadonlyMap; + label: string; +}) { + return ( +
+
+ + + + + + + + + + + {runtime && } + + + {controls && ( + + )} + + + + {rows.map((row) => { + const { queue, assessment } = row; + return ( + + + + + + + + + {runtime && ( + + )} + + + {controls && ( + + )} + + ); + })} + +
+ Queue + Status + Waiting + Last hour + Trend + + Oldest wait + + Running + Capacity + Done/min + + Retrying / failed + + Controls +
+ + {queueName(queue)} + + {queue.display_name?.trim() && ( + + {queue.queue} + + )} + + + + {assessment.label} + + + + + + + + + {queue.available > 0 ? ( + formatDuration(queue.lag_seconds) + ) : ( + — + )} + + + + + + {formatCount(Math.round(queue.completed_last_hour / 60))} + + + + + {controls(queue)}
+
+ +
    + {rows.map((row) => { + const { queue, assessment } = row; + return ( +
  • +
    +
    + + {queueName(queue)} + +

    {assessment.detail}

    + {(queue.descriptor_stale || queue.descriptor_mismatch) && ( +
    + +
    + )} +
    + + {assessment.label} + +
    +
    + + + + +
    +
    +
    +
    Waiting
    +
    + +
    +
    +
    +
    Oldest wait
    +
    + {queue.available > 0 + ? formatDuration(queue.lag_seconds) + : "—"} +
    +
    +
    +
    Running
    +
    + +
    +
    +
    +
    Done/min
    +
    + {formatCount(Math.round(queue.completed_last_hour / 60))} +
    +
    +
    +
    Retrying
    +
    + +
    +
    +
    +
    Failed
    +
    + +
    +
    + {runtime && ( +
    +
    Capacity
    +
    + +
    +
    + )} +
    + {controls &&
    {controls(queue)}
    } +
    • + ); + })} +
+
+ ); +} diff --git a/awa-ui/frontend/src/components/QueuePressure.tsx b/awa-ui/frontend/src/components/QueuePressure.tsx deleted file mode 100644 index f5a63fbde..000000000 --- a/awa-ui/frontend/src/components/QueuePressure.tsx +++ /dev/null @@ -1,138 +0,0 @@ -import type { ReactNode } from "react"; -import { Link } from "@tanstack/react-router"; -import type { QueueStats, QueueRuntimeSummary } from "@/lib/api"; -import { dlqPolicyLabel } from "@/components/RuntimeDisplay"; -import { availableWait } from "@/lib/operations"; - -export function QueuePressure({ - queues, - runtime, - actions, -}: { - queues: QueueStats[]; - runtime?: Map; - actions?: (queue: QueueStats) => ReactNode; -}) { - return ( -
- - - - - - - - - - {actions && } - - - - {queues.map((queue) => { - const workers = runtime?.get(queue.queue); - return ( - - - - - - - - {actions && ( - - )} - - ); - })} - -
QueueAvailableOldest availableRunningRetryable / failed - Completed last hour - Controls
- - {queue.display_name?.trim() || queue.queue} - - {queue.display_name?.trim() && {queue.queue}} -
- {queue.paused && ( - Paused - )} - - {queue.scheduled.toLocaleString()} scheduled ·{" "} - {queue.waiting_external.toLocaleString()} waiting external - -
- {runtime && queue.description && ( -

{queue.description}

- )} - {queue.descriptor_stale && ( -
- Descriptor stale -
- )} - {queue.descriptor_mismatch && ( -
- Descriptor drift across live runtimes -
- )} - {workers && ( -
- {workers.healthy_instances}/{workers.live_instances} live - instances healthy - {workers.config_mismatch && " · config mismatch"} -
- )} - {workers?.config && ( -
- {dlqPolicyLabel(workers.config)} -
- )} - 		- - {queue.available.toLocaleString()} - - - 0 ? "wait-value" : "text-muted-fg" - } - > - {availableWait(queue.lag_seconds)} - - - - {queue.running.toLocaleString()} - - - 0 ? "retry-value" : "text-muted-fg" - } - to="/jobs" - search={{ state: "retryable", q: `queue:${queue.queue}` }} - > - {queue.retryable.toLocaleString()} - - - 0 ? "failure-value" : "text-muted-fg" - } - to="/jobs" - search={{ state: "failed", q: `queue:${queue.queue}` }} - > - {queue.failed.toLocaleString()} - - - {queue.completed_last_hour.toLocaleString()} - - {actions(queue)} -
-
- ); -} diff --git a/awa-ui/frontend/src/components/SnapshotStatus.tsx b/awa-ui/frontend/src/components/SnapshotStatus.tsx index 38fec8cc2..ea8e8eebb 100644 --- a/awa-ui/frontend/src/components/SnapshotStatus.tsx +++ b/awa-ui/frontend/src/components/SnapshotStatus.tsx @@ -22,11 +22,14 @@ export function SnapshotStatus({ sources }: { sources: SnapshotSource[] }) { age !== null && age * 1000 > Math.max(30000, poll.interval * 2); return (
-