diff --git a/.gitignore b/.gitignore index 8123a213e..56fbf8e6e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ cover.out /*.png # Local agent and editor tooling indexes — machine-specific, never committed. +.claude/ .codegraph/ .serena/ .superpowers/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 000000000..4466753f5 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,20 @@ +# Default rules, plus known non-secrets: a fixed test-only auth code secret, +# a redaction fixture, a cache key name, and content hashes in fixtures. +[extend] +useDefault = true + +[[allowlists]] +description = "test-only fake secrets and cache key names" +regexes = [ + '''^0123456789abcdef0123456789abcdef$''', + '''^sk-live-abc123$''', + '''^edge_rollup_v2_rawmax$''', +] + +[[allowlists]] +description = "content hashes in fixtures and the retired Atlas checksum file" +paths = [ + '''^ui/host/e2e/data/mcp\.json$''', + '''^ui/host/tests/go-fragments/[^/]+\.json$''', + '''^internal/db/migrations/atlas\.sum$''', +] diff --git a/context7.json b/context7.json index 69a99477d..3f8e1df8f 100644 --- a/context7.json +++ b/context7.json @@ -5,10 +5,9 @@ "branch": "main", "folders": ["site/src/content/docs", "docs", "README.md"], "excludeFolders": [ - "docs/superpowers", + "docs/specs", "docs/media", - "docs/diagrams", - "docs/benchmarks" + "docs/diagrams" ], "rules": [ "Fanout is one Go process with no external dependencies to operate. There is no separate ingester, query service, metadata database, object store, or dashboard server to deploy.", diff --git a/docs/benchmarks/2026-10-agent-dashboards-m1.md b/docs/benchmarks/2026-10-agent-dashboards-m1.md deleted file mode 100644 index 139a6e0b8..000000000 --- a/docs/benchmarks/2026-10-agent-dashboards-m1.md +++ /dev/null @@ -1,127 +0,0 @@ -# Agent-built dashboards, Milestone 1 browser verification — 2026-10-05 - -This records a browser verification of Milestone 1 against a local Fanout instance on replayed demo data. It measures; it does not fix. Every value below was read from a real Chromium session driven through Playwright, the HTTP API (`curl` with the admin session cookie) and the control database (read-only). - -## Setup - -- **Build.** Branch `feat/agent-dashboards`, commit `c3353cce`, one local instance at `http://127.0.0.1:7520`. -- **Data.** Twenty-four hours of fanout-demo telemetry (the OpenTelemetry Demo services: spans, logs, metrics), time-shifted so it ends near the replay time. The newest data point is 2026-10-05T17:00:25Z, and the replay finished at about 17:25Z. The last 25 or more minutes therefore have no telemetry. That is a property of the setup, not a defect. The "Last hour" views show roughly 44 minutes of data, and rate-style stats are computed over that window. -- **Severity data.** The logs hold only `INFO` (12,624 in the last 24 hours at first check) and `UNSPECIFIED` (3,394) severities. There are no `ERROR` or `FATAL` logs. -- **Model.** The chat pane ran `claude-haiku-4-5`, the model in the local `.env` at the time. The model comparison below replaced it. -- **Browser.** Playwright Chromium, 1440 by 900 viewport, session cookie injected into the browser context. -- **Timing method.** Agent timings poll `GET /api/dashboards` every 2 seconds from the moment the prompt is sent, so they have 2-second resolution. Render timings use `performance.now()` from navigation start, polled on every animation frame. - -## Success criteria - -| Criterion | Target | Measured | Verdict | -|---|---|---|---| -| S1 Benchmark prompts produce a saved dashboard | 10 of 10 | Verbatim prompts: 2 of 5 (prompts 4 and 10). Prompts 1, 2 and 6 were answered in text with no dashboard. Re-run with a "Build me a dashboard" prefix: 3 of 3 saved. Prompts 3, 5, 7, 8 and 9 not run. | Not met on verbatim prompts | -| S2 Panels with rows or an explained empty state at save | 100% | 100% on the 5 dashboards run: 24 of 24 panels `ok`, 0 empty without a reason, 0 error. | Met on this sample | -| S3 Specs failing validation after the agent's own correction loop | 0 | 0 of 5 saved dashboards (the agent corrected an over-strict error filter in prompt 10 before saving). | Met on this sample | -| S4 Median prompt to saved dashboard | at most 45 s | 16 s median over 5 saved dashboards (12, 14, 16, 28, 32 s; maximum 32 s). | Met | -| S5 Follow-up edits change only the panels they name | 100% over 5 edits | 1 of 1 edit. Version 2 differs from version 1 by one added panel and nothing else. Five consecutive edits were not run. | Met on 1 edit | -| S6 First full render of a 12-panel, 24-hour dashboard | at most 1.5 s p95 | 397 ms p95 (361, 368, 371, 376, 382, 397 ms over 6 loads; median 374 ms). | Met | -| S7 Panel query latency, warm read cache | at most 500 ms p95 | 121 ms p95 server `elapsed_ms` over 60 panel queries (12 panels, loads 2 to 6); median 70 ms, maximum 135 ms. | Met | -| S8 Requests per dashboard refresh | 1 batch request, plus 1 for annotations; variable options once per range change | 1 `POST /api/panels/query` per manual refresh. 1 `POST /api/variables/resolve` plus 1 `POST /api/panels/query` per range change and per variable change. No annotation request was observed in M1. A dashboard with no variables makes 0 resolve requests on load. | Met, annotations not present | -| S9 Panel types | 15 | Not measured in M1 | n/a | - -S6 note: the 12-panel dashboard is built from the default spec's panel shapes: 4 stats, 4 time series, 2 bars and 2 tables, plus 2 more time series. Its range is 24 hours with auto-refresh off. "Rendered" means 6 chart images, 2 tables and the stat values are present with no loader. The single `panels/query` call finished at about 300 ms of that. - -## Default dashboard ("System overview") - -| Panel | State | -|---|---| -| Requests per second | data (4.2 to 4.4/s) | -| Error rate | data (2.4%, "Degraded") | -| p95 latency | data (42 ms, "Healthy") | -| Error logs | empty, with a stated reason: "No logs match upper(severity) IN ('ERROR', 'FATAL'). Without that filter, N do." Correct: the data holds no error logs. | -| Requests by service | data (stacked time series) | -| p95 latency by service | data | -| Error rate by service | data (bar) | -| Slowest endpoints | data (15 rows) | - -Controls checked: all worked. - -- Time picker: 11 presets plus an absolute range. Changing to 24 hours sent 1 resolve and 1 query. -- Refresh: 1 query. -- Auto-refresh radios: present (Off, 10s, 30s, 1m, 5m). -- Compare: turned on and off. Stats show "vs previous period" deltas. -- Variable `service`: selecting `checkout` re-resolved and re-queried, and the URL gained `var-service=checkout`. -- Panel menu: View, Inspect, Explain in chat, Copy link. -- Full-screen view: opens a dialog and sets `view=`. Escape closes it. -- Inspect drawer: tabs Data, Query, Spec, Timing, all populated. Timing reads "Ran in 28 ms, one point per 30s, 70 rows." -- Copy link: the clipboard held the dashboard URL with `view=errors`, and a "Link copied" notice showed. -- Edit mode at 1440 px: dragged "Requests per second" to the right of the first row, saved. `PUT` created version 2 ("Edited layout"). After a full reload the panel was still at the right of the first row. -- Light and dark themes both render with readable contrast. The browser console had no errors or warnings across the session. - -## Agent prompts - -All times are seconds from send to a saved dashboard. Status is `ok`, `empty` or `error` panels at save, read from `GET /api/dashboards/{id}` and `POST /api/panels/query`. - -| # | Prompt | Dashboard | Seconds | Panels (ok/empty/error) | Answers the question? | -|---|---|---|---|---|---| -| 1 | Give me a health overview of the shop. | none; answered in text | n/a | n/a | Text answer was accurate, but no dashboard. | -| 1b | Build me a dashboard: give me a health overview of the shop. | Shop Health Overview (7 panels) | 16 | 7/0/0 | Yes. Four stats, error-rate and latency timelines, and a per-service health table. | -| 2 | Why is checkout slow right now? | none; answered in text | n/a | n/a | Text answer found the 84 s Kafka "orders publish" span and said there was no telemetry in the last 15 minutes. | -| 2b | Build me a dashboard: why is checkout slow right now? | Checkout Performance Diagnosis (8 panels) | 32 | 8/0/0 | Partly. It isolates PlaceOrder and the "orders publish" span, but the "Checkout Health Status" stat shows p50 latency, and the three latency charts look identical at this scale. | -| 4 | Frontend latency by route with p50, p95 and p99. | Frontend latency by route (2 panels) | 12 | 2/0/0 | Partly. The table has p50, p95 and p99 per route. The time series shows p95 only. | -| 6 | Show the slowest PlaceOrder traces and what they logged. | none; answered in text | n/a | n/a | Text answer was good. | -| 6b | Build me a dashboard of the slowest PlaceOrder traces and what they logged. | Slowest PlaceOrder Traces (4 panels) | 28 | 4/0/0 | Partly. The trace table is right, but the log panels are totals for the hour, not tied to those traces. | -| 10 | Log volume by severity and service, with a stream of the errors. | Log Volume by Severity and Service (3 panels) | 14 | 3/0/0 | Partly. Volume by severity and by service is right. There is no error stream, because there are no error logs. The agent said so in its reply but left no explained-empty panel. | - -Follow-up edit, on Shop Health Overview: "Add a panel with error rate for payment." It took 6 seconds to save version 2 (author kind `agent`, message "Add payment service error rate panel"). Comparing the stored specs of versions 1 and 2 shows one added panel, `payment_error_rate`, and no other change. The panel returns data. - -## Screenshots - - -- `overview-light.png`, `overview-dark.png`: System overview. -- `compare24.png`: System overview, 24 hours, compare on. -- `service-checkout.png`: System overview with `service` set to checkout. -- `inspect.png`: inspect drawer. -- `fullscreen.png`: full-screen view of a panel. -- `edit.png`: edit mode. -- `dash-p1-light.png`, `dash-p2-light.png`, `dash-p4-light.png`, `dash-p6-light.png`, `dash-p10-light.png`: the agent-built dashboards, light theme. - -## Defects - -The fix-commit column is left for the controller. - -| ID | Severity | Title | Steps to reproduce | Expected | Actual | Fix commit | -|---|---|---|---|---|---|---| -| D1 | Medium | Dashboard-shaped prompts answered in text, no dashboard saved | In a new chat send "Give me a health overview of the shop.", "Why is checkout slow right now?" or "Show the slowest PlaceOrder traces and what they logged." | A saved dashboard, as S1 requires (10 of 10) | Text-only answers; no dashboard created. With "Build me a dashboard" prefixed, all three saved one. || 02c73ff5 (agent prompt) | -| D2 | Medium | A panel-level `unit` applies to every measure column | Open "Slowest PlaceOrder Traces". The spec sets `unit: ms` on a table with `duration_ms` and `count_distinct(service)`. | `service_count` shown as a plain count (1) | It shows "1.0ms". The column of per-trace durations is correct. || 9b19faca | -| D3 | Medium | Error-stream request dropped silently instead of an explained empty panel | Send "Log volume by severity and service, with a stream of the errors." on data with no ERROR logs. | A logs panel for errors with an explained empty state (S2 allows this) | No error panel at all; only mentioned in chat. || 02c73ff5 (agent prompt) | -| D4 | Low | Small rates display as "0/s" | Open System overview, set `service` to checkout. "Requests per second" and the table's `rate` column. | About 0.04 to 0.1/s | Stat shows "0/s" while the same service's chart shows 0.1/s. || 9b19faca | -| D5 | Low | Empty-state reason names the wrong filter and prints a literal `$service` | Set `service` to checkout. "Slowest endpoints" has the filter `http_route <> ''`, which gRPC-only checkout does not satisfy. | A reason that names the filter that removed the rows | "No spans match service = $service. Without that filter, 4612 do." The variable filter was not the cause. || 9b19faca | -| D6 | Low | Chart legend overlaps the top y-axis label | System overview: "Requests by service" and "p95 latency by service" at 1440 px, legend wrapped to two rows. | Legend and axis do not overlap | The second legend row overlaps the top axis tick ("10/s", "10m"). || 9b19faca | -| D7 | Low | Narrow text wrap and clipped tables | System overview "Slowest endpoints": `frontend` wraps as "fronten / d". "Slowest PlaceOrder Traces" table is cut off after 6 rows with a nested scrollbar. | Cells do not break words; panels size to content or scroll clearly | As described. A large blank area also sits under the single-row table in "Checkout Performance Diagnosis". || 9b19faca | -| D8 | Low | URL parameters carry JSON-quoted values | Toggle compare or open edit mode. | `compare=1`, `edit=1` | `compare=%221%22`, `edit=%221%22` || 9b19faca | -| D9 | Low | Agent output quality in the checkout dashboard | Open "Checkout Performance Diagnosis". | A health stat that shows health; three charts that show different things | "Checkout Health Status" displays p50 latency (15 s). The checkout, PlaceOrder and orders-publish charts look identical. || 02c73ff5 (agent prompt) | -| D10 | Low | No API to read a stored dashboard version | `GET /api/dashboards/{id}/versions` lists metadata only; there is no route for a version's spec. | A way to diff versions through the API | I diffed versions by reading the `dashboard_versions` table directly. || Deferred to milestone 3 (version history) | - -## Concerns - -- S1 depends on prompt wording. The agent chooses between a dashboard and a text answer, and the verbatim spec prompts for 1, 2 and 6 take the text path. -- Only 5 of 10 benchmark prompts and 1 of 5 follow-up edits were run, so S1 and S5 are sampled, not complete. The model name was not recorded. -- Timings and the render check run on a warm local machine with a single-user data set. S6 and S7 pass with wide margin, but they are not a load test. -- The new payment panel uses thresholds with status `ok` at 1 and `warn` at 5, and no server-kind filter, which is inconsistent with the default dashboard. I did not judge this beyond noting it. -- The "Last hour" and "right now" views end at 17:00Z because of the setup. The agent handled this sensibly in prompt 2 ("no active telemetry ... in the last 15 minutes"). - -## Model comparison - -After the browser run, the agent prompt was changed so dashboard-shaped questions build a dashboard (`02c73ff5`), and the OpenAI provider moved to the Responses API so current OpenAI reasoning models can call tools (`6a87d2bb`). Three models then ran the ten benchmark prompts, each on its own instance with a fresh control database and a copy-on-write clone of the same replayed telemetry. Two runs per model. - -| Model | S1 saved | S2 panels ok or explained | S3 invalid | S5 scoped edit | Median seconds to save | -|---|---|---|---|---|---| -| `claude-sonnet-5-5` | 10/10, 10/10 | 100%, 98.5% | 0 | 2/2 | 28, 21 | -| `claude-opus-5-5` | 10/10, 10/10 | 100%, 100% | 0 | 2/2 | 36, 39 | -| `gpt-6.1-sol` | 10/10, 10/10 | 100%, 98.2% | 0 | 2/2 | 47, 44 | - -Answer quality on the first run, scored 0 to 5 by two graders that were not contestants (`claude-fable-5-1` and `gpt-5.6-terra`) on four criteria: Sonnet 4.58, Opus 4.67. Sol was not graded. - -Held-out prompts: sixteen requests written by a separate agent that never saw the agent prompt or the benchmark failures, eleven that call for a dashboard and five that call for a short answer. They were not used for tuning. Sonnet's two runs: 11/11 and 10/11 dashboards, 5/5 and 5/5 answers without a dashboard, intent accuracy 1.00 and 0.94. The Opus held-out runs were stopped early for cost, and Sol's were not run. - -Decision: `claude-sonnet-5-5` is the default. It matches the others on every hard criterion, ties Opus on graded quality within the graders' spread, and saves dashboards about 40% sooner. The OpenAI default is `gpt-6.1-sol`. - -S1 on the benchmark prompts is now 10 of 10 for every model, against 2 of 5 in the browser run above, which used the old prompt and Haiku. The held-out set is the better guide to real wording; it should be rewritten before any further prompt tuning. - diff --git a/docs/benchmarks/2026-10-agent-dashboards-m2.md b/docs/benchmarks/2026-10-agent-dashboards-m2.md deleted file mode 100644 index 4bd2dcaa8..000000000 --- a/docs/benchmarks/2026-10-agent-dashboards-m2.md +++ /dev/null @@ -1,288 +0,0 @@ -# Agent dashboards Milestone 2 verification - -Build: 669bec26 (collector run on 2c37797e; 669bec26 changes only a manual-refresh queue and a test wait). Model: claude-sonnet-5-5. Source: Replayed fanout-demo format-2 telemetry converted to format 3 over OTLP by .superpowers/replay (newest input 2026-10-05T16:45:10.69Z, target 2026-10-06T22:40:35.61Z), served by a disposable local instance with no AI provider. Deploy markers use 7,201 synthetic cart spans with service.version 2.3.0 (attribute fanout.synthetic), 19:30:58Z-23:30:58Z. Loading, empty, error, partial and sparkline-gap states were produced by browser network interception, not demo data. Evaluations: claude-sonnet-5-5 on the same frozen replay; benchmark baseline and candidate are two-repeat means; holdout baseline is a two-repeat mean, holdout candidate is one run (the second failed to start in the harness). No judge pass.. -Status: PASS. - -This run was taken about a day after the replay's newest point, so its 24-hour window holds only part of the replayed telemetry. An earlier run inside a fresh window returned about 20,500 cells per refresh in 1.6–2.1 s, also with exactly one panels request and one annotations request. - -Replay: 2213782 spans, 860377 logs, 14302306 metric points; shift_ns=107724921629673. - -## Browser type coverage - -| Theme | Type | Visible | Aria | Inspect | Console errors | -|---|---|---|---|---|---| -| light | stat | true | true | true | 0 | -| light | gauge | true | true | true | 0 | -| light | timeseries | true | true | true | 0 | -| light | bar | true | true | true | 0 | -| light | table | true | true | true | 0 | -| light | text | true | true | true | 0 | -| light | heatmap | true | true | true | 0 | -| light | histogram | true | true | true | 0 | -| light | scatter | true | true | true | 0 | -| light | state_timeline | true | true | true | 0 | -| light | logs | true | true | true | 0 | -| light | log_patterns | true | true | true | 0 | -| light | traces | true | true | true | 0 | -| light | service_map | true | true | true | 0 | -| light | health | true | true | true | 0 | -| dark | stat | true | true | true | 0 | -| dark | gauge | true | true | true | 0 | -| dark | timeseries | true | true | true | 0 | -| dark | bar | true | true | true | 0 | -| dark | table | true | true | true | 0 | -| dark | text | true | true | true | 0 | -| dark | heatmap | true | true | true | 0 | -| dark | histogram | true | true | true | 0 | -| dark | scatter | true | true | true | 0 | -| dark | state_timeline | true | true | true | 0 | -| dark | logs | true | true | true | 0 | -| dark | log_patterns | true | true | true | 0 | -| dark | traces | true | true | true | 0 | -| dark | service_map | true | true | true | 0 | -| dark | health | true | true | true | 0 | - -## S8 and bounded refresh - -| Theme | Panels | Panel requests | Annotation requests | Cells | ms | -|---|---:|---:|---:|---:|---:| -| light | 23 | 1 | 1 | 6642 | 1219 | -| light | 23 | 1 | 1 | 6642 | 1201 | -| light | 23 | 1 | 1 | 6642 | 1268 | -| dark | 23 | 1 | 1 | 6642 | 1269 | -| dark | 23 | 1 | 1 | 6642 | 1240 | -| dark | 23 | 1 | 1 | 6642 | 1169 | - -## Interaction evidence - -- inspect: PASS; 30 theme/type Inspect checks matched frames or SQL and closed (text verified directly) -- loading: PASS; Loading panel visible before delayed response, absent after 1500ms response and render settle -- empty: PASS; empty asserted on 14 types/panels (calls,err_gauge,rate_ts,svc_bar,svc_table,lat_heat,lat_hist,ops_scatter,err_states,recent_logs,patterns,slow_traces,svc_map,svc_health); interception removed; observed real response and rendering recovered -- error: PASS; error asserted on 1 types/panels (rate_ts); interception removed; observed real response and rendering recovered -- partial: PASS; partial asserted on 1 types/panels (rate_ts); interception removed; observed real response and rendering recovered -- filter_chip: PASS; Real bar selected frontend; removal restored All and 10 visible services; selection request, removal cache_hit -- linked_crosshair: PASS; 4 required comparisons show exactly one ≤8px band within ±12px at relative x=.63; wide tooltip bands recorded and ignored; heatmap/timeline followers recorded; chart IDs ec_1791397700259→ec_1791397727777 -- brush_history: PASS; 40-move 200px brush pushed exactly one history entry; from/to appeared; Back restored http://127.0.0.1:7520/dashboards/01a11382-f0df-708c-bade-71d3fa50fc21?range=24h -- panel_time: PASS; Visible 1d badge; requested panel shift=1d; observed response windows differ exactly 86400000ms; all other windows match (1791311395896..1791397795896) -- drill_url: PASS; slow_traces: scrollY=3707 preserved on open/close; click→drill URL→reload retained→Back closed; recent_logs: scrollY=2979 preserved on open/close; click→drill URL→reload retained→Back closed -- waterfall_logs: PASS; Selected trace 020a920f65fbf268508dab44747cdbf9; 2 waterfall spans; correlated logs empty with trace-ID/window reason -- cancel_drill: PASS; Delayed first drill; closed before completion; second dialog matched its URL kind/panel; releasing first response never restored first drawer or target (MutationObserver + URL) -- deploy_markers: PASS; Controller visual inspection in the scripted screenshots and in a real Chrome session on build 2c37797e: the cart-scoped charts draw a dashed "cart 2.3.0" line at the 19:30Z first-seen instant plus the anomaly band; the no-marker twin draws neither; service=checkout shows no deploy line. The pixel heuristic stays inconclusive because the marker label lane compresses the plot; the version is a labelled synthetic injection. -- anomaly_areas: PASS; Cart error_rate_change 2026-10-06T22:29:00Z..2026-10-06T22:54:00Z produces 136px shaded band; checkout has no unrelated band; matched twin screenshots and saved diffs -- deploy_split: PASS; Cart visible frame has Before deploy,Since deploy series and nonblank bar screenshot (request); All has a plain bar and note: Service is All; showing the unsplit whole-window frame. (cache_hit) -- column_formats: PASS; service_link click sets picker/request; sparkline SVG, status text with non-colour cue, width bar, trace_link drill, normalized duration unit and log_template code/wildcards observed -- stat_sparkline: PASS; Calls total=232282 unchanged after one simulated null; 2 SVG subpaths prove gap; route removed and real data recovered - -## Agent evaluation - -Holdout is never used for tuning. Matching repeats use the same frozen telemetry and account isolation. - -| Set | Baseline S1 | Candidate S1 | Baseline intent | Candidate intent | Errors | Unchecked | -|---|---:|---:|---:|---:|---:|---:| -| benchmark | 1 | 1 | 1 | 1 | 0 | 0 | -| holdout | 0.9091 | 1 | 0.9375 | 1 | 0 | 0 | - -## S13 M2 capability preview comparison - -| Capability | Preview evidence | M2 evidence | Gap | -|---|---|---|---| -| heatmap | Latency distribution heatmap: log buckets labelled in ms (<25 ms to >=3.2 s), sequential blue ramp | lat_heat: log2 buckets labelled with units (0-1 ms to 2.2-4.4 min), p99-capped continuous colour scale with its legend, deploy and anomaly markers, drill to exemplar traces; light and dark screenshots | none | -| log_patterns | Error and warning patterns: severity, template with wildcards, service, count, trend | patterns: body_template with placeholders, count bar and per-template trend, which is the spec definition (body_template counts with trend); the preview adds per-pattern severity and service columns, tracked for the milestone 4 parity review | none | -| deploy_markers | Dashed "deploy payments v2.14.0" line and a shaded anomaly region on time charts | Dashed "cart 2.3.0" line at the first-seen instant and the 22:29-22:54Z anomaly band on cart-scoped charts; absent for checkout; label clear of the zoom tools | none | -| drill_drawer | Click a chart for exemplar traces; click a row to open the trace | Drawer addressed by a drill= URL that survives reload and closes on Back; exemplars, shared waterfall and correlated logs; a cancelled drill never reappears | none | -| split_bars | Time in downstream calls: paired Before deploy and Since deploy bars | deploy_bar with service=cart shows Before deploy and Since deploy series; with All it shows the plain bar and a note | none | -| table_formats | Trace links, durations with units and inline bars, counts and error text | service_link, sparkline with value, status badge with percent and icon, bar, trace_link opening the drill, unit and log_template, all asserted in the DOM | none | - -## Screenshots - -- .superpowers/replay/m2-screenshots/light-six-series-legend.png -- .superpowers/replay/m2-screenshots/light-full.png -- .superpowers/replay/m2-screenshots/light-viewport-00.png -- .superpowers/replay/m2-screenshots/light-viewport-01.png -- .superpowers/replay/m2-screenshots/light-viewport-02.png -- .superpowers/replay/m2-screenshots/light-viewport-03.png -- .superpowers/replay/m2-screenshots/light-viewport-04.png -- .superpowers/replay/m2-screenshots/light-viewport-05.png -- .superpowers/replay/m2-screenshots/dark-six-series-legend.png -- .superpowers/replay/m2-screenshots/dark-full.png -- .superpowers/replay/m2-screenshots/dark-viewport-00.png -- .superpowers/replay/m2-screenshots/dark-viewport-01.png -- .superpowers/replay/m2-screenshots/dark-viewport-02.png -- .superpowers/replay/m2-screenshots/dark-viewport-03.png -- .superpowers/replay/m2-screenshots/dark-viewport-04.png -- .superpowers/replay/m2-screenshots/dark-viewport-05.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-p95_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-lat_heat-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_states-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-p95_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-p95_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-lat_heat-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-lat_heat-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_states-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-mounted-rate_ts-err_states-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-p95_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-lat_heat-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_states-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-p95_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-p95_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-lat_heat-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-lat_heat-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_states-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-rate_ts-err_states-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-p95_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_states-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-p95_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-p95_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_states-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-lat_heat-err_states-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-err_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-p95_ts-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-lat_heat-before.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-err_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-err_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-p95_ts-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-p95_ts-diff.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-lat_heat-after.png -- .superpowers/replay/m2-screenshots/dark-crosshair-remounted-err_states-lat_heat-diff.png -- .superpowers/replay/m2-screenshots/dark-deploy-cart-markers.png -- .superpowers/replay/m2-screenshots/dark-deploy-cart-plain.png -- .superpowers/replay/m2-screenshots/dark-deploy-cart-diff.png -- .superpowers/replay/m2-screenshots/dark-deploy-negative-checkout-markers.png -- .superpowers/replay/m2-screenshots/dark-deploy-negative-checkout-plain.png -- .superpowers/replay/m2-screenshots/dark-deploy-negative-checkout-diff.png -- .superpowers/replay/m2-screenshots/dark-deploy_markers-failure.png -- .superpowers/replay/m2-screenshots/dark-anomaly-cart-markers.png -- .superpowers/replay/m2-screenshots/dark-anomaly-cart-plain.png -- .superpowers/replay/m2-screenshots/dark-anomaly-cart-diff.png -- .superpowers/replay/m2-screenshots/dark-anomaly-negative-checkout-markers.png -- .superpowers/replay/m2-screenshots/dark-anomaly-negative-checkout-plain.png -- .superpowers/replay/m2-screenshots/dark-anomaly-negative-checkout-diff.png -- .superpowers/replay/m2-screenshots/dark-deploy-split-cart.png -- .superpowers/replay/m2-screenshots/dark-stat-null-gap.png - -## Remaining defects and gates - -All gates passed against measured evidence. -## Defects this verification found and fixed - -The unit and integration suites passed before this check. Running the -dashboards in a real browser on replayed demo data still found the -following, each fixed with a regression test in its owning suite -(commits 93cd7556 and ef61ab1e): - -- **Deploy history lost after compaction.** The version rollup skipped any - batch over 64,000 rows, which includes every compaction output, so its - versions were never read and every dashboard reported limited annotation - history. Oversized batches now get a pass of their own, oldest first, and - a batch that keeps failing backs off without blocking newer ones. -- **Panels stuck loading.** A panel scrolled into view while a batch was in - flight was never requested. A requested panel the server omits now shows - an error, and lazy batches keep the dashboard's annotations. -- **One clock per batch.** Panels resolved "now" separately; a 1-day shifted - panel was 27 ms off. The batch now resolves a single instant. -- **Misleading empty diagnosis.** A highlighted logs panel blamed the - highlight even when a filter caused the empty result. -- **Presentation:** status cells now carry the measure's unit; the traces - panel truncates trace IDs instead of overlapping columns and shows - Error/OK rather than OTLP enum names; sparkline cells show their value; - heatmap and histogram buckets carry units; the heatmap colour scale is - capped at the 99th percentile with a legend; single-series charts drop - their legend; day boundaries read as month and day; deploy labels stay - clear of the zoom tools; heatmap time labels no longer run together. - -Synthetic data: the demo has a single `service.version` per service, so -deploy markers and the deploy split were exercised with a labelled -synthetic version change for one service. Browser network interception -produced the loading, empty, error, partial and sparkline-gap states. - -## Final review - -A broad review and a focused security and concurrency review followed the -browser check. The focused review ran about 100 SQL-panel shapes against -the pinned engine and found no path to unredacted log bodies, no escape -from the approved relations, and no injection or owner-scope bypass; a -concurrent ingest, compaction and rollup probe converged with every -version present. The findings it and the broad review raised were fixed in -2c5a686c: grouped panels now accept one measure instead of silently -dropping the rest, empty variable options no longer crash a saved -selection, bar drills keep every grouping value and the deploy period, -share sparklines in limited tables use the whole scope, a failing version -rollup batch no longer holds back healthy ones, annotations read in one -transaction, and SQL panel CTEs may not shadow telemetry relations. - -Deferred: an explicit empty multi-value selection does not survive a shared -URL (rare; it shows no data either way), and schema-qualified log columns -fail with an error rather than binding (they fail closed). - -## Hands-on testing - -The dashboards were then used by hand in a real Chrome session, with the agent on -`claude-sonnet-5-5` and, separately, `gpt-6.1-sol`. From one sentence each provider -built a correct dashboard in 30–60 seconds; Sonnet used deploy markers, a -before-and-since split and a version table to show a cart release raising p95 from -2.4 ms to 8.4 ms and errors from 0% to 6%, and it edited the dashboard by -conversation in under 20 seconds. - -That session found what the scripted checks could not, all fixed (13aa6942, -2c37797e, 669bec26): the chat gave no link to the dashboard the agent saved and the -sidebar did not list it until a reload; opening a drill scrolled the dashboard to -the top; every chart painted a needless scrollbar on systems that always show -scrollbars; the deploy-split note showed a raw timestamp; the grid could first lay -out at a guessed width; series colours came from two hue families by name hash and -failed colour-vision checks, now a validated six-slot palette assigned in order with -at most six series per chart; and a refresh pressed during a partial lazy batch was -dropped. The scripted collector now also checks that chart bodies fit, that drills -keep the scroll position and that series colours are distinct. - -## Production-grade pass - -After hands-on use, every panel type was captured in both themes at 1100 px -and 1440 px on a fresh replay and reviewed against the capability preview. -Each defect found was fixed with a regression test, and the captures were -repeated until none remained: - -- **Gauge:** became a linear meter that stays readable in a small card. -- **Panel chrome:** a "viz · source" subtitle, Chart/Data/Spec views (folded into the panel menu on narrow cards) and interaction hints. -- **Time series:** - - labels at the line ends, and labelled thresholds; - - deploy and anomaly labels in a lane above the plot; - - legends that wrap rather than truncate. -- **Series selection:** - - six validated, colour-blind-safe series colours, assigned in order; - - series chosen worst-first by confidence (the Wilson lower bound for error rates; at least 20 samples for latency), with the rest folded into "Other (N)". -- **Heatmaps:** a log count scale on a seven-step ramp, with cells sized to the panel. -- **State timelines:** merged runs, every row labelled, and a state legend. -- **Service map:** rebuilt as a left-to-right layered dependency map. - - Service cards with health and key metrics. - - Directed edges weighted by traffic. - - Neighbourhood highlight and keyboard focus. - - The layout is cached on graph structure; refreshing a 400-service map takes about 3 ms of main-thread work. - - Page scrolling passes through. -- **Tables, logs and traces:** - - tabular numerals and placeholder chips; - - a soft highlight with its term shown; - - compact timestamps; - - log-pattern severity and service. -- **Correctness:** charts no longer animate through overlapping layouts. The grid measures before rendering, drills and Back keep the scroll position, a refresh pressed during lazy loading still runs, and an empty variable list no longer crashes a dashboard. -- **Security:** every telemetry-derived string in an HTML tooltip is escaped. - -Final checks on ef412b4b–d4f2f075: - -- `just check` passes. -- A page smoke run over six dashboards, both themes and both widths found no error boundary and no page errors. -- 21 of 22 scripted browser checks pass. The remaining Inspect check was verified by hand and by a reviewer's real-click sweep across all fifteen types. -- An agent benchmark with `claude-sonnet-5-5` on fresh data: - - all ten dashboards built; - - every panel returned rows or explained why it was empty; - - full intent accuracy; - - a median of 29.6 s; - - half as many validation retries as before. diff --git a/docs/benchmarks/2026-10-agent-dashboards-m3.md b/docs/benchmarks/2026-10-agent-dashboards-m3.md deleted file mode 100644 index 8406f2f03..000000000 --- a/docs/benchmarks/2026-10-agent-dashboards-m3.md +++ /dev/null @@ -1,181 +0,0 @@ -# Agent dashboards Milestone 3 — measurements - -## Build measurements (Task 1b) - -Task 1b measurements only. Controller light/dark live-chat, console/network and screenshot acceptance remains pending; automated fixtures are not browser evidence. - -| MCP app output | Raw bytes | gzip -9 -n bytes | -|---|---:|---:| -| `internal/mcp/apps/panels.html` | 2,342,591 | 772,682 | - -The five retired outputs totaled 7,076,818 raw bytes in the supplied baseline. The generic output is below the 2,500,000-byte target and reduces that aggregate by 4,734,227 bytes (66.90%). - -SPA embedded tree: 2,972,143 → 2,974,801 raw bytes. Route-related chunk measurements below include shared chunks; no SPA threshold was specified. - -| SPA chunk family | Before raw bytes | After raw bytes | -|---|---:|---:| -| `chat-*.js` | 17,950 | 19,565 | -| `chat.index-*.js` | 70 | 70 | -| `chat._threadId-*.js` | 70 | 70 | -| `dashboards.index-*.js` | 430 | 430 | -| `dashboards._dashboardId-*.js` | 517 | 517 | -| `page-*.js` | 1,118,815 | 1,118,815 | -| `settings-*.js` | 12,390 | 12,390 | -| `mcp-app-frame-*.js` | 242,399 | 243,366 | -| `index-*.js` | 705,786 | 705,786 | -| `preload-helper-*.js` | 16,963 | 17,039 | - -`page-*.js` remains the shared dashboard page chunk. The MCP host frame remains lazy; the app build rejects runtime imports of SPA routes/auth/API modules and emits only `panels.html` with inlined styles, fonts and Blob worker code. - -Verification: host Vitest and TypeScript gates, seeded MCP fragment calls, source boundary fixtures, app build graph/CSP fixtures, and temporary-output byte/path comparisons passed. Browser acceptance and the clean-source private audit binary run belong to the controller review gate. - -### Task 1b fix round 1 - -The rebuilt `panels.html` is **2,343,092 raw bytes / 773,074 gzip -9 -n bytes**. The SPA embedded tree is **2,978,557 raw bytes**. Both regenerated trees match a fresh staged build. The app remains below the 2,500,000-byte target. - -`BenchmarkBoundFragmentLarge`, one measured iteration over 40 × 5,000 cells of 346 characters (200,000 cells, about 69 MB), took **350,498,500 ns/op** (350.5 ms), with **1,363,672,496 B/op** total allocations and **1,203,179 allocs/op**. Total allocations are not peak memory. This is a local benchmark, not a browser or concurrency measurement. - -Automated size-reporter tests see no additional notification over a simulated three-second settled interval and test wrapper-only measurement. Actual Chromium convergence, node-label font size, graph width and reader scroll acceptance still require the controller's browser collector. - -### Task 1b fix round 2 - -The rebuilt `panels.html` is **2,333,439 raw bytes / 769,783 gzip -9 -n bytes**. The SPA embedded tree is **2,969,904 raw bytes**. Both trees match a fresh staged build; the app remains below the 2,500,000-byte target. - -The shared map now uses one deterministic Dagre LR layout with two-dimensional contain fitting. The 20-node fixture passes box, route and label geometry assertions at 780 × 460 and 1100 × 220. The reusable DOM collector is `ui/host/tests/service-map-collector.ts`; actual Chromium runs on chat and dashboards remain the controller's acceptance gate. - -### Task 1b fix round 3 - -The rebuilt `panels.html` is **2,334,158 raw bytes / 770,144 gzip -9 -n bytes**. The SPA embedded tree is **2,970,771 raw bytes**. Both trees match a fresh staged build; the app remains below the 2,500,000-byte target. - -Full 44 px cards and compact 20 px cards now share the same Dagre LR algorithm. Cached geometries let resize choose the card mode without a fresh layout. The restored dashboard fixture passes containment and measured name/font budgets at 1100 × 190 and 1100 × 220, with all twenty names visible at or above 11 px. Call the DOM collector with `{requireNames:true}` for the controller's dashboard browser re-check; automated fixtures do not establish live browser acceptance. - -## Authoring benchmark (Task 12) - -Status: **PASS** on the iteration run (all of S1–S5). The baseline failed S2 for an environment reason, explained below. This file records Task 12 of the milestone 3 plan. Task 14 adds the final frozen-candidate acceptance run and the browser matrix. - -### Method - -- **Runner:** `scripts/dashboard-eval/main.ts --set benchmark`, the promoted runner from Task 3. Each run is the ten benchmark prompts from the design spec (S1–S4), followed by five consecutive edits on the first saved board (S5): title, threshold, add, remove and unit. Every provider call is metered into one cost ledger shared across both runs, capped at $5 for the task. No paid judge. The sealed holdout set was not opened. -- **Model:** `claude-sonnet-5-5` (Anthropic), the default agent model. Every metered call matched the configured model. -- **Telemetry:** the immutable fanout-demo format-2 source (483 files; source hash `7131c02c93179d51…`). It was replayed over OTLP/HTTP into a fresh disposable instance, which stores format 3. - - Each run used one signed shift that places the newest replayed event at the replay start, so "right now" questions read fresh telemetry against the running clock. - - Traces (2,213,782 spans) and logs (860,377 records) were replayed by two parallel processes sharing that one shift. Metrics were excluded so the replay finishes inside the runner's five-minute freshness gate. None of the ten prompts needs metric points. - - Control state was fresh: no copied database and no secrets. -- **Instance:** a production build of the candidate, on a local native Fanout with a fresh data directory. - -### Results - -| Run | Candidate | S1 saved | S2 good / total | S3 invalid | S4 median (s) | S5 edits | Cost (USD) | -|---|---|---:|---:|---:|---:|---:|---:| -| Baseline | d5bdc79c | 10/10 | 56/59 | 0 | 15.5 | 5/5 | 1.388 | -| Iteration | fb00809e | 10/10 | 58/58 | 0 | 19.9 | 5/5 | 1.445 | - -Targets: S1 10 of 10, S2 100%, S3 0, S4 a median of 45 s or less, S5 100%. S4 measures from the prompt to the save that S1–S3 score. Total spend for the task: $2.83 of $5. - -#### Baseline per prompt - -| Prompt | Panels | Explained empty | Errors | First save (s) | Scored save (s) | Cost (USD) | -|---|---:|---:|---:|---:|---:|---:| -| frontend-latency | 3 | 0 | 0 | 14.0 | 14.0 | 0.165 | -| health-overview | 7 | 0 | 0 | 25.2 | 25.2 | 0.150 | -| checkout-slow | 9 | 0 | 0 | 25.6 | 25.6 | 0.154 | -| errors-by-service | 7 | 1 | 0 | 15.8 | 15.8 | 0.114 | -| hour-compare | 6 | 0 | 3 | 20.5 | 20.5 | 0.122 | -| placeorder-traces | 3 | 0 | 0 | 14.9 | 14.9 | 0.111 | -| kafka-throughput | 7 | 0 | 0 | 15.1 | 15.1 | 0.098 | -| db-cache | 6 | 0 | 0 | 14.4 | 14.4 | 0.109 | -| deploys | 5 | 0 | 0 | 20.1 | 20.1 | 0.117 | -| log-volume | 6 | 2 | 0 | 12.6 | 12.6 | 0.100 | - -#### Iteration per prompt - -| Prompt | Panels | Explained empty | Errors | First save (s) | Scored save (s) | Cost (USD) | -|---|---:|---:|---:|---:|---:|---:| -| frontend-latency | 3 | 0 | 0 | 12.1 | 12.1 | 0.162 | -| health-overview | 11 | 0 | 0 | 21.8 | 21.8 | 0.140 | -| checkout-slow | 8 | 0 | 0 | 15.6 | 15.6 | 0.109 | -| errors-by-service | 8 | 0 | 0 | 24.3 | 24.3 | 0.154 | -| hour-compare | 5 | 0 | 0 | 22.9 | 22.9 | 0.124 | -| placeorder-traces | 4 | 0 | 0 | 14.7 | 14.7 | 0.118 | -| kafka-throughput | 5 | 0 | 0 | 18.0 | 18.0 | 0.111 | -| db-cache | 5 | 0 | 0 | 46.4 | 46.4 | 0.181 | -| deploys | 5 | 0 | 0 | 41.1 | 41.1 | 0.118 | -| log-volume | 4 | 1 | 0 | 14.2 | 14.2 | 0.081 | - -"Explained empty" panels returned no rows and carry both the executor's diagnosis and an authored description. S2 counts them as good. Every explained empty panel in these runs asks for error-severity logs, and the replayed demo logs have none. - -### What changed between the runs - -1. **Environment: the DuckDB memory cap.** - - The baseline instance inherited a 4 GB `FANOUT_DUCKDB_MEMORY` cap from an earlier local setup. Bulk-replaying 40 hours of telemetry in about two minutes left a backlog of several thousand small batches. The service rollup that worked through that backlog repeatedly hit "Out of Memory Error (3.7 GiB/3.7 GiB used)", and so did three table panels the post-save check ran at the same moment. - - Those three errors are the baseline's entire S2 failure. The authored queries were valid. - - The iteration run uses Fanout's default sizing (unset; derived from detected memory) and saw no memory errors. -2. **Product: guidance against pinned identifiers (fb00809e).** - - In the baseline, the "slowest PlaceOrder traces" board saved a logs panel filtered by three trace IDs copied from one query result. Such a panel goes stale as soon as newer traces arrive. - - The analysis guidance now forbids filtering saved panels on trace, span or request IDs copied from a single result, and points to a traces panel with drill to its logs. `internal/agent/dashboard_guidance_test.go` pins the rule. - - The iteration run's board follows it: a traces panel with drill, a latency series, log patterns and service logs. - -No scorer, threshold or post-save check was changed. - -### Findings for milestone 4 - -- **Memory under an ingest backlog:** - - With a 4 GB DuckDB cap, the service rollup and concurrent panel queries fail with out-of-memory errors while a large backlog of small batches is pending, instead of spilling or degrading. - - The version rollup also hit its execution timeout. - - This belongs with the S6/S7 performance work. The post-save check records only `status: error` for such panels, without the executor message. -- **Run-to-run variation:** the S4 median moved from 15.5 s to 19.9 s between runs. The slowest iteration prompts (database and cache, recent deploys) took 41–46 s and made more tool calls. These are single runs, not repeated measurements. - -### Limitations - -- Each column is one run, not a mean. -- Metric points were not replayed. -- The edits target the first saved board (frontend latency by route). -- The sealed holdout set is untouched and was not used for tuning. Task 14 records it as unrun; see below. - -## Final acceptance (Task 14) - -Status: **PASS** for the authoring benchmark, the sealed holdout, the browser collector, the checked-in smoke test and the per-panel check. - -### Authoring benchmark on the frozen candidate - -| Run | Candidate | S1 saved | S2 good / total | S3 invalid | S4 median (s) | S5 edits | Cost (USD) | -|---|---|---:|---:|---:|---:|---:|---:| -| Final | eee385d8 | 10/10 | 67/67 | 0 | 18.7 | 5/5 | 1.449 | - -- The method is the same as Task 12: a production build, a fresh replay with one shared shift, default DuckDB sizing, and its own $5 cost ledger. -- The fix rounds after eee385d8 change rendering, the server's packing of new panels and query-cancellation logging. They do not change agent guidance, tools or panel execution results, so the run was not repeated. - -### Browser evidence - -- **Collector** (private, development build of a1395856, fresh replay with deploy markers, light and dark): 33 of 33 checks pass, and theme coverage passes 30 of 30 visualization and theme rows. -- **Smoke test** (`just e2e`, 1100 and 1440 px, light and dark): 24 of 24 on every fix round, including 0d8a5123. -- **Per-panel check** (0d8a5123, a 23-panel verification board and a 10-panel agent-built board, light and dark): - - all 33 panels render with no failed panels, stuck loaders, truncated titles or overflow outside the card; - - every row has one height and spans all twelve columns; - - there are no console errors. -- **Service map overflow:** at half width, 7 of 18 services fit at the readable floor. The cropped edge fades and Fit appears at first render. Fit shows all 18 services, and a second press returns to the readable view. A map that fits shows neither. - -### Panel app size - -`internal/mcp/apps/panels.html` is the only MCP app: 2,380,413 bytes raw and 784,374 bytes with `gzip -9 -n`. The five milestone 2 apps were 7,076,818 and 2,412,876 bytes, so the new app is 66% smaller raw and 67% smaller gzipped. - -### Holdout set - -The sealed holdout ran once on a production build of 04cd5c19, through the private intent-aware harness that scored the milestone 2 holdout. The prompt text was not read, and nothing was tuned on it. - -| Run | Correct outcome | Dashboards saved | Answered without a dashboard | Acceptable panels | S3 invalid | S4 median (s) | Run errors | -|---|---:|---:|---:|---:|---:|---:|---:| -| M2 candidate | 16/16 | 11/11 | 5/5 | 95/95 | 0 | 57.4 | 0 | -| M3 final | 16/16 | 11/11 | 5/5 | 88/88 | 0 | 33.4 | 0 | - -- The set has 16 prompts: 11 ask for a dashboard and 5 ask for a direct answer. -- The harness scores the agent's choice of outcome and the panel checks. It does not score whether a saved dashboard answers the question; that needs the paid two-model judge, which neither milestone ran. -- The harness has no cost ledger. At the benchmark's per-prompt cost, the run is estimated below $3. -- Three dashboards were saved on a second create after the first create returned an error; every saved panel passed. - -### Findings for milestone 4 (Task 14) - -- A long refresh error is truncated in the panel and its full text cannot be reached. -- The rollup and panel out-of-memory errors under a DuckDB memory cap during an ingest backlog, recorded in Task 12, remain open. While a bulk replay's backlog drains, detector annotation writes and the version rollup also time out once a minute; both stop when the backlog clears. -- Promote an intent-aware runner with cost metering and the two-model quality judge, so the holdout is scored like the benchmark. -- Saved dashboards keep their stored grid. Row snapping and twelve-column fill apply only when the server packs new panels. diff --git a/docs/benchmarks/2026-10-agent-dashboards-m4.md b/docs/benchmarks/2026-10-agent-dashboards-m4.md deleted file mode 100644 index 549bba09d..000000000 --- a/docs/benchmarks/2026-10-agent-dashboards-m4.md +++ /dev/null @@ -1,535 +0,0 @@ -# Agent dashboards Milestone 4 — evidence audit - -Task 1 audit, 2026-10-09. Source: `82ab72e9629475af52aadb337b63d91bc61138a9`, -branch `feat/agent-dashboards-m4`. Task 1 initialized the evidence for the M4 -candidate and made no product changes ($0). Task 11 replaced its PENDING -verdicts. A Task 11 rerun on the corrected candidate then replaced the first -run's verdicts with the measurements below. **Every criterion the rerun -measured passes:** S1–S5, S8, S9, S11, S12 and S13. S6 (local) and S10 keep -their first-run PASS, and S7 passes on the raw path. The S6 CI ceiling is 1,800 ms, -calibrated from three runner measurements. - -Requirements: [design spec](../superpowers/specs/2026-10-04-agent-dashboards-design.md) -and [M4 plan](../superpowers/plans/2026-10-09-agent-dashboards-m4.md), subject to -the controller's binding rulings. Historical baselines are -[M1](2026-10-agent-dashboards-m1.md), -[M2](2026-10-agent-dashboards-m2.md) and -[M3](2026-10-agent-dashboards-m3.md). -Their measurements, screenshots and PASS labels do not establish an M4 PASS. - -## S1–S13 final verification - -Task 11 rerun, 2026-10-10 UTC. - -- **Candidate:** HEAD `ae10f2f67969b0f6ded4a49ea3d799188f16ee31`. It carries the - first run's corrections D1, D2 and D4 and the card-corner fix. It differs from - `2d17645a` only in a panel benchmark test. -- **Binaries:** the production binary was built from the clean tree with its - committed browser assets. The collector used the - development audit build of the same tree. -- **Engine:** the pin is unchanged since the first run (`SELECT version()`: - `v2.0.0-alpha43763`). -- **Platform:** darwin/arm64, 14 logical CPUs, default DuckDB sizing (22,118 MB, - 14 connections). The machine was shared, with a load average of 7–20 during - the benchmark and collector runs. -- **Model and spend:** `claude-sonnet-5-5` only. Paid spend was **$1.905655** - on a new rerun ledger with a $2.80 cap. The first run spent $2.087733 on its - own ledger, so both runs together spent $3.993388. -- **Holdout and judge:** the holdout was not run. The judge was skipped - (`judge_keys_unconfigured`). -- **Product changes:** none in the rerun. Two corrections went to the browser - collector instead (see the first-run history below). - -Telemetry: - -- **Source:** the same immutable fanout-demo format-2 source (483 files, - 2,213,782 spans, 860,377 logs, 14,302,306 metric points). -- **Replay:** it was replayed again, with the same converter and transport - groups, into a fresh local instance with fresh control state. The first - run's copy was older than the evaluation runner's five-minute freshness gate - allows. -- **Shift:** one signed shift (+394,040,685,048,673 ns), shared by all signals - and twelve converter processes. The newest event landed 60 s after the replay - started, and the replay took 160.6 s. -- **Read back from the instance:** exact source counts. Event time runs from - 2026-10-08T13:23:21Z to 2026-10-10T06:12:31Z, with one namespace and 17 span - services. There are 342 metric names (gauge 95, histogram 35, sum 213), - including the first run's pool metrics. -- **Timing:** the benchmark started 1.8 s after the replay ended. -- **Synthetic deploy:** injected after the benchmark, as in the first run: - 7,201 `cart` server spans at version 2.3.0 over the last four hours, 432 of - them errors, marked `fanout.synthetic=deploy-verification`. -- **Edge rollup:** the backlog and lag were 0 eight minutes after the replay, - and the one-hour service map returned 20 services and 23 routes. - -| Criterion and literal target | Measured on the M4 candidate | Evidence | Verdict | -|---|---|---|---| -| **S1:** Benchmark prompts produce a saved dashboard, **10 of 10** | **10/10** saved, each as one immutable version 1. The runner recognised every save (exit 0, aggregate pass). | Ten-prompt benchmark with the five edits: `--set benchmark` with the plan's Task 11 Step 4 inputs and a $2.80 cap. | **PASS** | -| **S2:** Panels return rows, or carry an explained empty state, at save time, **100%** | **65/65** panels good at save time: 63 with rows and 2 authored explained-empty. | The runner's save-time executor checks. Both empty panels filter `severity_number >= 17`, which matches no demo logs, and each panel's description says so. | **PASS** | -| **S3:** Specs fail validation after the agent's own correction loop, **0** | **0** invalid. 10/10 final specs valid and checked. | Runner validation. | **PASS** | -| **S4:** Median prompt to saved dashboard, default model, **≤ 45 s** | Median first save **21.5 s** (range 10.7–54.9 s). | Runner save timing. The slowest prompt, `db-cache`, took 54.9 s. | **PASS** | -| **S5:** Only named panels change over **5 consecutive edits, 100%** | **5/5** exact: title, threshold, add, remove and unit (versions 2–6). Only the named panel changed. Layout changed only for add and remove. | Runner edit comparison. | **PASS** | -| **S6:** First full render, **12 panels / 24 hours, ≤ 1.5 s p95** | Not re-measured. First run, real data, local: light **634 ms**, dark **582 ms** p95 (24 fresh loads each). Controller's local e2e: 361 / 393 ms. CI: three ubuntu-24.04 runs gave a worse-theme p95 of 1,025, 1,410 and 1,239 ms; the enforced CI ceiling is 1,800 ms (max x 1.25, rounded up to 50 ms). | First run: the checked-in 12-panel fixture on the replay instance, timed with the performance spec's in-page `allPanelsPainted` callback. Each load made one panel batch with zero faults. Since then, the product changes are the empty-state wrap, card corners and edge rollup. | **PASS** (local, first run); CI: **controller** | -| **S7:** Warm-cache panel query latency, **≤ 500 ms p95** | Slowest shape **47.1 ms** p95 on the traces-and-logs replay; **33.1 ms** on the three-signal replay, including metric panels (gauge 33.1, counter 27.4, histogram 23.2 ms). 100 warm samples per shape, idle machine. | `BenchmarkPanelQueries24Hours` (readbench) on replayed demo data, 24 hours, default production sizing. See Task 6 below. | **PASS** | -| **S8:** **1 batch + 1 annotations request per refresh; variable options once per range change** | Collector: **6/6** refresh cycles on the 23-panel board (3 per theme). Each made exactly 1 panel batch and 1 annotations request. | `drill_url`: opening and closing drills, including Back after a reload, sent 0 panel requests. The controller's `just e2e` performance spec covers range-change counts. | **PASS** | -| **S9:** **15 visualization types** listed in the spec | Collector: **30/30** type/theme rows visible, labelled, inspectable and passing their rendering assertions. Page smoke: 56/56 across 14 dashboards at 1100 and 1440 px. | The service map showed 20 services and 23 routes in both themes. Its top and bottom overflow fades were drawn, and the whole-graph control removed the overflow. | **PASS** | -| **S10:** **WCAG AA in both themes; categorical palette passes CVD validation** | Not re-measured. First run: Task 8 guards (axe-core 4.13.0 scans of every dashboard and chat panel type in both themes, in the controller's passing `just e2e`) and the six-slot CVD unit test (Machado 2009, OKLab thresholds). | First-run screenshot review at 1100 and 1440 px found no contrast failure. The clipping it found (D2) is fixed. | **PASS** (first run) | -| **S11:** Shared link reproduces **time, variables, comparison and focused panel** | Collector: `panel_time`, `brush_history`, `empty_multiselect` and `fullscreen_panel` PASS. | `fullscreen_panel` checked the focused-panel dashboard URL, shared links, a missing panel and Esc/Close/Back in both themes. `empty_multiselect` reopened a copied URL with comparison and an explicit empty selection exactly. | **PASS** | -| **S12:** #232 **items 1, 9, 12, 14, 17, 24, 30 resolved** | 1: `stat_sparkline` PASS. 9: `empty` PASS on 14 types. 12: the missing-panel case in `fullscreen_panel` PASS. 14: `column_formats` and `legend_geometry` PASS. 17 and 24: `preview_geometry`, `resize_geometry` and `route_scroll` PASS. 30: refresh counts and `drill_url` PASS. | Collector: 33/33 checks, zero defects. | **PASS** | -| **S13:** **Every in-scope preview capability shipped; checkout-incident and payments-database boards rebuilt from prompts** | All eight in-scope capabilities PASS. Five come from the first run's prompt-built boards. Deploy markers, database statement detail and trace/log drill were re-measured on this candidate. | Side-by-side comparison below. | **PASS** | - -### S13 side-by-side comparison - -In the first run, the two exact preview prompts each ran once through the agent -chat endpoint. They were metered on that run's ledger and saved valid boards -(13 and 8 panels; 21/21 checks good). The rebuilt boards were captured at 1100 -and 1440 px in both themes, next to a network-free copy of the private preview. -The rerun spent nothing on parity. It re-measured the three capabilities that -had failed, using the collector and the product's `preview_panels` tool, and -re-rendered the first run's checkout board on this build. Capabilities are -compared, not values. The preview's error-budget board is out of scope (SLO). - -| Capability | Preview | Product evidence | Live evidence | Verdict | -|---|---|---|---|---| -| Heatmap | Log2 latency distribution with exemplar drill | Checkout (first run): log2 duration heatmap with trace drill | Heatmap type rows, both themes | PASS | -| Contextual log patterns | Error and warning patterns with trends | Checkout (first run): `body_template` patterns with log drill | `pattern_context` PASS on dashboard and chat, both themes | PASS | -| Deploy markers and anomaly bands | Deploy line and anomaly shading on latency and pool charts | Both rebuilt boards enable deploys and anomalies. The checkout board, re-rendered on this build, lists `cart` 2.3.0 and an anomaly band, and shows no marker because only `cart` deployed. | `deploy_markers` PASS: one aligned 1-px `cart 2.3.0` band, and no band at that time on checkout. `anomaly_areas` PASS. | PASS | -| Before/since bars | Downstream calls split at the deploy | Checkout (first run): operations bar with `split: deploy` | `deploy_split` PASS (cart Before/Since series) | PASS | -| Database and cache detail | Share of p99 by statement, duration against rows, statements in timeout traces | `preview_panels` (no model call) ran three structured `product-reviews` panels, all `ok`: p99 by `db.statement` (3.8 ms), share by statement (100%), and a table of statement, database, rows (6,421), share, average, p99 and maximum duration. They rendered as a board in both themes at 1100 and 1440 px. | `preview_panels` over the session MCP route on the replay | PASS | -| Trace and log drill | Exemplar traces, trace table, logs from a trace | Trace drill on the heatmap, state timeline and traces panel; log drill on patterns. The re-rendered checkout board shows all of them. | `drill_url`, `waterfall_logs` and `cancel_drill` PASS | PASS | -| Table formats | Trace link, duration bar, template chips | Bar and unit column formats; trace links and duration bars in the traces panel | `column_formats` PASS | PASS | -| Pool metrics | Pool in-use time series with a pool-max threshold, peak stat | Payments (first run): table of six real pool series and their peaks, with typed attributes | Verified pool names and attributes on both replays | PASS | - -### First run (history) - -The first run (HEAD `fed0a916`, production binary from `1fd54332`, $2.087733) -measured these verdicts: - -- S1–S5: BLOCKED. -- S9, S11, S12 and S13: FAIL. -- S6 (local), S8 and S10: PASS. - -Its findings and their outcomes: - -| ID | Finding | Outcome | -|---|---|---| -| D1 | Eval runner: `findSaved` required an explicit `error` flag, and the AG-UI Go message omits an empty one, so no save was recognised. | Fixed in `bbfa48dc`. The rerun scored S1–S5. | -| D2 | UI: a long unbroken empty-state diagnosis overflowed its card. | Fixed in `2d17645a`. | -| D3 | `drill_url`: Back after a drill reload re-requested all 23 panels. | Not a product defect. In 28 diagnostic trials on the unchanged UI, Back after a reload stayed in the reloaded document and sent 0 panel requests, including with 6 s stale data and under 6× CPU throttling. The first run's traffic (`panels:aborted, panels:200, annotations:200`, 23 ids, page at the top) matches a Back that lands in a new document. The browser automation launches Chromium with the back/forward cache disabled, so that document must load every panel. The collector now marks the reloaded document and classifies a Back into a new document as a fresh load, as it already did for the reload. | -| D4 | Edge rollup took over an hour to catch up after a bulk replay. | Fixed in `bbfa48dc`. Backlog 0 within eight minutes; map routes present. | -| H1, H2 | Collector: a stale thread-list mock, and a colour-string comparison. | Fixed in the collector. | -| H3 | Collector (found in the rerun): the service-map overflow assertions still required the M2 fade element and the "+N below" hint, which M3 replaced with `data-map-overflow` / `data-map-fade`. | The collector now reads the current markers and still fails an overflow without a drawn fade. | -| H4 | Collector (found in the rerun): the deploy-marker negative check counted raw pixel differences. The marker board's annotation label strip shortens its plot and rescales every series point against the twin (939 px). | The collector now applies the positive check's narrow-band rule at the `cart` deploy time (largest column 3 rows, threshold 22.4). It still fails when a marker is drawn there. | - -The first collector rerun, with only the D3 correction, passed 32/33 checks -and failed on H3 and H4. The run reported above includes all three collector -corrections. Each correction has node tests showing it still fails on the -defect it guards. - -### Limitations - -- S6 and S10 were not re-measured on this candidate. -- Parity data is the demo: - - `payment` has no database spans. - - `product-reviews` issues one distinct statement, so its share is 100%. - - The only deploy is the labelled synthetic `cart` injection, so the - checkout board cannot show a checkout deploy. -- Statement detail was verified with `preview_panels` and a hand-written - structured spec, not by rerunning the payments prompt. -- The rebuilt boards use no variables, click-to-filter or saved comparison, - which the preview shows. These are outside the brief's capability list. -- In the first run, anomalies were written for all namespaces (empty namespace), - so an annotation request scoped to `otel-demo` returned none. Unscoped panels - show them. -- The machine was shared. - -## S12 item provenance and limits - -Associations are checked against the design spec's cited #232 behaviors in -“What exists today”, Structured query, Layout, Execution and Browser. The -full external issue text is not present in the supplied spec; this audit -does not claim to have independently fetched it. Keep the original issue -item numbers and actual behavior test names. - -| #232 item | Spec behavior and implementation | Existing guard and narrower evidence | -|---|---|---| -| **1** | Stat tiles must use the whole window rather than a partial/last bucket. `internal/panel/exec.go:totals` computes window totals. | `internal/panel/exec_test.go:TestRunProducesFrames` asserts the native fixture's whole-window request total is 120. `frame_semantics_test.go:TestGaugeTotalsMatchStat` compares nonempty gauge/stat totals; equality alone is narrower than proving every measure's whole-window value. `table-formats.test.tsx`: `uses accessible gap-preserving stat trends while retaining whole-window totals and delta`; `viz-regressions.test.tsx`: `uses the previous window total for the correct delta` cover rendering supplied totals. | -| **9** | Generated panels must diagnose empty queries so the author can fix or explain them before saving. | `internal/panel/diagnose_regression_test.go:TestDiagnosisNamesOnlyEmptyingRouteFilter` names the causal route filter beyond the old cutoff; `TestDiagnosisBoundsReruns` limits native-engine reruns. These establish causal diagnosis and bounded work, not that a model always repairs a panel. Fresh save checks remain required. | -| **12** | An unknown dashboard ID produces not-found rather than the default board. | `ui/host/src/dashboards/page.test.tsx`: `answers a missing dashboard instead of showing another` uses a mocked read; it is not a new browser issue regression. | -| **14** | Preserve measure units and prevent mixed-unit/second-series scaling errors. | `internal/panel/unit_regression_test.go:TestCompileMeasureUnitsOverridePanelUnit` checks compiler/wire column units. `validate_test.go:TestValidateReportsPathsAndHints/mixed units` rejects different unit families on one structured chart axis. `panels.test.ts`: `uses each column unit for mixed measure axes, tooltips and bar labels` checks renderer formatting/axis assignment; `chart-units.test.ts` adds axis and label guards. These are specific compiler/renderer cases, not a proof of every authored chart. | -| **17** | Downsample 24-hour series using client width, with bounded rows/cells. | `internal/panel/timerange_test.go:TestAutoInterval`; `compile_test.go:TestCompiledQueriesRunOnTheEngine`; `heat_density_test.go:TestWidthAwareCellBuckets` executes 24-hour line/heat/state bucket decisions at two widths, but its telemetry fixture is small. `distribution_test.go:TestDistributionBudget`, `execution_limits_test.go:TestSeriesRowCap`, `frame_semantics_test.go:TestBudgetKeepsCompleteNewestBuckets` cover caps. These do not measure S6/S7 performance. | -| **24** | Pack free/new panels without wasted gaps; stored user grids take precedence. | `internal/dashboard/layout_test.go:TestPackFillsGapsWithoutStretching`, `TestPackedRowBandsSnapOnlyFreeNewPanels`, `TestPackedRowBandsFillOnlyFreeColumns`; `layout_constraints_test.go:TestAddPreservesSavedLayout`. Host `layout.test.ts`: `mirrors the Go height tables and row unit for every size and visualization`; `grid-row-bands.test.ts`: `preserves server-snapped heights and the next full-width band during vertical compaction`. Host layout-table/compactor tests are narrower than a real browser screenshot. Existing saved grids are not retroactively repacked. | -| **30** | Avoid duplicate fetches and refresh only visible panels while the tab is active; resolve dependent variables appropriately. | The S8 manual-refresh, in-flight, already-loaded-scroll and known-empty guards named above; `use-variables.test.ts`: `uses server precedence and preserves provided values until options load`; `page.test.tsx`: `waits for defaultless variable options and sends exactly one panel request with opts[0]`. Known-empty visibility and selection tests do not establish real background-tab network behavior or once-per-range option counts; Task 7 owns those checks. | - -## M3 retirement audit - -**Verified at Task 1:** no unexpected live retired HTTP/widget contract was -found. This is retirement verification, not completion of S13 parity. - -- The plan's literal search across `cmd`, `internal/api`, `internal/mcp`, - `ui/host/src`, `scripts/dashboard-eval` and generated public references - finds `/api/observability` only in - `internal/api/traces_test.go:TestTraceRouteRejectsInvalidScopesAndRetiredPaths`. - Its six occurrences are negative tests, all expecting **404**: - `/api/observability/trace`, `/overview`, `/performance`, `/topology`, - `/logs` and `/dependencies` under that prefix. No `widgets/` match exists. -- File inventory contains no old widget renderer tree. The current decoder - is `ui/panels/fragment.ts:panelFragment`, consumed by - `ui/host/src/mcp-app-content.ts` and the shared fragment view. It requires a - v1 dashboard fragment and matching results, with no old widget payload - decoder. `fragment-view.test.tsx` rejects old `{ data: {} }` payloads and - exercises `uses the authored multi-panel preset title without a legacy - title decoder`. -- `cmd/fanout/main.go` registers panel, annotation and current trace routes. - `internal/api/auth_middleware.go:classifyRoute` has no retired - observability path classification. `internal/api/traces.go` registers - `GET /api/traces/:id` with `telemetry:read`; it remains an active contract. -- `cmd/fanout-docgen/routes_test.go:TestCollectRoutesResolvesGroupPrefixes` - verifies the current trace capability, and - `TestCollectRoutesClassifiesEveryRegisteredRoute` checks route policy - completeness. Neither is a dedicated negative-retirement assertion; the - generated-reference literal audit and `docs-generate-check` corroborate - absence of retired public routes without editing generated pages. -- `internal/mcp/apps/panels.html` is the **only** app output. - `internal/mcp/apps.go:registerAppResources` registers only that resource; - `ui/host/vite.apps.config.ts` builds only `panels.html`; - `scripts/ui-compare.mjs` enforces the one-file app tree. No retired app - output or resource registration was found. -- Historical spec sections describe widgets and `/api/observability/*` as - the pre-panel baseline and retirement plan. They are historical - documentation, not registrations. Active MCP presets such as - `get_observability_overview` return current panel fragments through - `panels.html`. Active `internal/observability` health, map, trace and - dependency services remain needed and were not removed. - -The targeted retirement test was run with `-count=1` and exited **0**. -Documentation freshness and both dead-code gates are recorded below. - -## Dispatch verification - -Task 1 ran formatting and all eleven individual gates, with the pinned -DuckDB wrapper through `just test`, offline modules and the required private -Go cache/temp directories. Detailed command outputs and the task report -remain private dispatch artifacts. None of these gates substitutes for the pending browser, -provider, real-data performance, capped-backlog or two-board measurements. - -| Command | Exit code | Observed output summary | -|---|---:|---| -| `rtk just test ./internal/api -run TestTraceRouteRejectsInvalidScopesAndRetiredPaths -count=1` | 0 | API retirement guard passed; all six retired paths expect 404. | -| `rtk just fmt` | 0 | Formatting completed; no existing tracked file changed. | -| `rtk just fmt-check` | 0 | No formatting differences. | -| `rtk just ui-boundaries-check` | 0 | UI workspace boundaries passed. | -| `rtk just lint` | 0 | **0 issues**; sandbox warnings prevented persisting golangci analysis facts to its cache. No cache workaround was attempted. | -| `rtk just test ./internal/... ./cmd/...` | 0 | Go suites passed, including panel, API, dashboard, MCP, query and docgen. The opt-in `readbench` suite was not run. | -| `rtk proxy sh -c 'cd ui/host && bun run test && bun run lint'` | 0 | **92 test files / 1284 tests passed**; all three TypeScript checks passed. Both commands completed successfully. | -| `rtk just ui-deadcode` | 0 | Production Knip files/exports check passed. | -| `rtk just go-deadcode` | 0 | Only the existing allowlisted `OAuthTokenPair.String` security-formatting entry was reported. | -| `rtk just script-tests` | 0 | **116 tests passed, 0 failed**; mock CLI reported ten saves, five edits, **23 failure injections**, S1–S5 verified, **$0**. | -| `rtk just test-names-check` | 0 | Behavioral test names passed. | -| `rtk just dashboard-eval-generate-check` | 0 | `TestDashboardEvalGoldensCurrent` passed with `-count=1`. | -| `rtk just docs-generate-check` | 0 | **13 generated pages** match their documented types. | - -No new failing test or correction was needed: Task 1 authorizes an audit and -one evidence document, with no product edits. Existing behavior guards were -run as evidence of current coverage. No aggregate `just check`, UI rebuild, -site dependency install, browser run, provider call, sealed-input read, -release, deployment or git write was performed. - -## Task 2: capped small-batch backlog reproduction - -**Deadline/progress failure reproduced; native OOM inconclusive.** This is -diagnosis on the uncorrected candidate, not an S2/S6/S7 PASS. The new opt-in -`readbench` probes are `TestBacklogPressureRecordsConcurrentStatementFailures` -and `TestPanelsRemainUsableDuringBacklog`. The latter intentionally fails -until the diagnosed behavior is corrected. No Task 3 correction was applied. - -Each process seeds a fresh native format-3 repository before opening DuckDB: -4096 batches, 512 spans and 192 logs per batch, six services, v1/v2, distinct -IDs, 24-hour event-time spread and a 4.095-second ingest interval. Only one -batch is materialized at a time. The seed contains 8192 Parquet files and -2,883,584 events, about 183 MB compressed in the first control. This file-count -fixture does not replace the supplied representative replay. All generated -state and exact logs remain private under `.superpowers/`; the replay was -left untouched. - -The fixed configuration is **4GB, four threads, five read connections**, -30-day retention and one-second rollup ticks. Native `SELECT version()` -returned `v2.0.0-alpha43763` (source -`96063b9e39749cc0f087bb11501d7edbee527957`), on darwin/arm64 with 14 logical -CPUs. The engine reports 3.7 GiB for this cap. A 30-minute context starts -before seed; the test timeout is 35 minutes. Seed and workload are timed -separately. - -The authoritative panel workload uses four simultaneous complete dashboard -requests through the production executor: service-count table, span-count -stat, grouped p95 and log-count stat. Absolute 24-hour bounds bind the real -`queryrows.Window`; executor fanout and timeouts are unchanged. After a -30-second panels-only control, a two-minute combined phase runs real -`RunRollups`, including maintenance/read-cache refresh, 32 bounded live -publications, version refresh and anomaly writes. Half the live batches have -events twelve hours late. Workers are joined, then marker catch-up has a -three-minute bound. A test-only forwarding proxy records actual failed SQL, -including separate stat totals, without changing native execution. - -All attempts were retained. Reduced-fanout diagnostics and diagnostic exit -zero are not robustness results. - -| Attempt | Seed | Total test | Outcome | Exit | -|---|---:|---:|---|---:| -| 4096, reduced-fanout exploratory control | 113.066 s | 444.90 s | 11 failures; drain incomplete | 0 | -| 4096, query solo/loop control | 121.433 s | 270.07 s | Solo reads succeeded; loop version admission deadlines | 0 | -| 4096, initial full dashboard | 121.429 s | 454.21 s | 24 failures; drain incomplete; count assertion subsequently corrected | 1 | -| 1024, full dashboard | 25.092 s | 175.46 s | One anomaly admission deadline; exact counts and marker drain | 1 | -| 4096, before actual-SQL proxy | 112.035 s | 448.19 s | 112 failures; drain incomplete | 1 | -| 4096, final full dashboard | 103.222 s | 441.35 s | 85 unexpected failures; drain incomplete | 1 | - -The smallest **tested** failing workload is 1024 batches. All its completed -panel results succeeded, but an anomaly call waited **10.018 s** and returned -`anomaly_log/write_gate: context deadline exceeded`. Its raw, footer and -service totals agree on **540,672 spans / 202,752 logs**. No exhaustive -threshold search or 8192-batch run was needed to establish this deadline -defect; the OOM question remains open. - -In the final 4096 run, combined pressure lasted **120.129 s**, with 32 -successful publications, 64 directly processed version batches and one -successful anomaly write. Six version rows and one anomaly row were visible -before drain. The last active/cached/version-marker counts were -**3620/702/259**. Raw executor, active-footer and service counts nevertheless -agree exactly on **2,113,536 spans / 792,576 logs**. Exact event totals do not -prove complete cache acknowledgement or completed edge catch-up. - -The returned errors identify two kinds of wait: - -- Ten full anomaly admission timeouts returned - `anomaly_log/write_gate: context deadline exceeded`, approximately 10 s - each, before bounds/insert SQL. -- Five full version admission timeouts returned - `version_rollup/write_gate: context deadline exceeded`, approximately - 20 s each, before insertion. -- Panel calls returned `context deadline exceeded` and native - `INTERRUPT Error: Interrupted!` (DuckDB error type 29). Actual failed - statements were span/log totals, their bucketed trends, count by service - and grouped p95. All 87 SQL-proxy errors, including planned shutdown - interruptions, occurred at `QueryContext`. Planned phase-end cancellation - is excluded from the 85 unexpected failures. - -No workload `Out of Memory Error` was observed. Fixed `%w` labels were added -only after failing attribution tests, preserving native cause and context -identity. An injected OOM in those attribution tests is a mock cause and is -not workload evidence. Whole-window stat totals and actual-SQL attribution -also have passing native fixture tests. - -Service alone took **5.063 s**; edge alone **22.496 s**, even though these -parentless spans yield no edges. Through final drain, completed analytical -gate holds totaled **256.250 s / 6** for edge and **41.830 s / 6** for -service. Read-cache gate waits totaled only **0.000080 s / 22**. The final -read pool reached all five occupied slots and recorded 323 waits totaling -**939.573 s** across concurrent callers. - -Largest sampled tracked memory was **1,887,812,526 bytes** in the reduced -fanout control and **1,043,647,116 bytes** in the 1024 full workload. The -final 4096 workload's largest successful sample was **242,194,364 bytes**; -only four SQL snapshots succeeded because the read pool was busy. These -are sampled lower bounds, not measured full-workload peaks. All observed -temporary-file and temporary-storage totals were zero. The configured spill -directory exists, but positive spill was not demonstrated. RSS measurement -was **BLOCKED** by denied `ps` execution; peak RSS and untracked memory are -unavailable. Go heap/cumulative allocations and CPU time are reported -separately in the private evidence and are not substitutes for RSS. - -The Task 3 recommendation is to first reduce the existing -`maxEdgeSubWindowsPerPass` budget from eight to one, retaining its committed -cursor and watermark rules. This yields at an existing unit without a new -scheduler, cache, option or longer deadline. A minute-scale service ingest -clamp cannot split this fixture's 1–4-second burst. This is a candidate, -not a proven fix: cold panel deadlines and service holds also occur. Task 3 -must pass the unchanged full regression and eventual exact-count checks; -representative replay OOM and RSS remain controller measurements. - -Task 2 verification used the individual dispatch gates. Formatting, formatting -check, UI boundaries, normal Go tests, normal and `readbench` lint (**0 -issues**), both dead-code gates, test names, eval goldens and documentation -freshness all exited **0**. The native attribution/count suites also exited -**0**. The full gate remains incomplete: host tests exited **1** twice on -existing five-second ECharts rendering timeouts (one case, then three); -script tests exited **1** twice on the existing five-second test-name fixture -(115/116 passed). Separate host lint and the skipped mock eval CLI exited -**0**; the mock recorded ten saves, five edits and 23 failure injections for -**$0**. No unrelated tests or timeout settings were changed. Exact commands, -exits, timing, errors and frozen source hashes are in the private task report. - -## Task 3: one committed edge sub-window per pass - -**INCOMPLETE: five anomaly admission deadlines remain at 4096 batches.** The -controller-selected correction changes `maxEdgeSubWindowsPerPass` from eight -to one. Existing committed cursor and watermark rules remain intact. No -second mechanism, longer deadline, reduced panel fanout or larger memory cap -was added. - -`TestAnalyticalBacklogAllowsVersionAndAnomalyProgress` failed with the old -budget because the first pass consumed its whole fixture instead of yielding. -It now checks one committed sub-window, concurrent version/anomaly writes -before complete drain, independent completed-batch cache writes during an -analytical transaction, unchanged committed progress after cancellation, -exact eventual edge counts, compaction and active batch-ID acknowledgements. -`TestServiceRollupBacklogDrainsWithoutDroppingLateBuckets` checks every raw -namespace/minute/service span and log count against the rollup, with separate -equal-ingest and **3h40m ingest-backlog** cases, a late publication below the -guarded tip and idle plateau. Existing wide-window tests now drain bounded -passes before checking final counts. The cancellation case uses an already -canceled context; it does not claim interruption at a specific point after -DELETE. - -Both unchanged Task 2 probes ran once at each batch size, in separate fresh -processes, sequentially without concurrent gates. Their source hashes match -Task 2. Configuration, fixture, native engine, full executor fanout and phase -durations remain as recorded above. - -| Probe / batches | Seed | Process duration | Exit | Observed result | -|---|---:|---:|---:|---| -| Query / 1024 | 27.153 s | 154.366 s | 0 | No logged errors; diagnostic only. | -| Full panel / 1024 | 22.945 s | 177.795 s | 0 | Zero unexpected failures; exact counts and marker drain. | -| Query / 4096 | 104.303 s | 238.345 s | 0 | No logged errors; diagnostic only. | -| Full panel / 4096 | 111.286 s | 330.316 s | **1** | **Five anomaly admission timeouts**; exact counts and marker drain. | - -At 1024, raw executor, active footer and service counts agree on **540,672 -spans / 202,752 logs**; markers drain to **40/40/40** -active/cache/version-marked batches in **0.064 s**. At 4096, all counts agree -on **2,113,536 spans / 792,576 logs**; markers drain to **699/699/699** in -**65.368 s**, within the original three-minute bound. Both publish all 32 -live batches and expose six version rows and one anomaly row before drain. -Successful anomaly calls number 72 / 9 respectively. All completed panel -results are OK; planned phase-end canceled requests remain archived. - -The five live failures return exactly -`anomaly_log/write_gate: context deadline exceeded`, after -**10.004259, 10.000994, 10.002053, 10.002298 and 10.000538 s**. They fail -in `RecordAnomalies` at `writeGate.LockContext`, before bounds or insertion -SQL executes. There is no failed SQL statement for these admissions. -Additional phase-end gate deadlines and native interruptions are retained -separately; the unchanged regression excludes them from unexpected failures. -Exact timestamps, errors, original failed panel SQL and controller rerun -commands remain in the private Task 3 report/artifacts. - -The 4096 query control's edge solo pass takes **4.415 s**, versus Task 2's -**22.496 s** for eight sub-windows; service solo takes **5.877 s**. More -passes are needed. Full-probe tracked-memory snapshot maxima are -**96,831,702 / 1,413,598,568 bytes** for 1024 / 4096. These are sampled lower -bounds, not process peaks. Observed temporary storage is zero; no workload -OOM was observed. RSS/untracked memory remains **BLOCKED** by the unchanged -denied process inspection. After drain, read-pool aggregate waits total -**198.272 / 657.307 s**; 4096 completed edge/service gate holds total -**89.997 / 84.302 s** over 28 holds each. Gate sums do not identify another -correction. - -Combined completed-panel p95 values (table/span stat/grouped p95/log stat) -are **844/1567/907/1591 ms** at 1024 (312 samples per shape) and -**5619/7664/6080/7573 ms** at 4096 (70 per shape). These mixed cold/pressure -four-shape measurements do not establish S6 browser rendering or S7 warm -replay performance. The large probes' drain condition covers markers and -the service watermark; it does not assert full edge-watermark catch-up. -Exact edge convergence is covered by the new native fixture. Remaining -admission failures prevent a capped-backlog robustness PASS and keep Task 3 -**INCOMPLETE**. - -Task 3's focused native regression suite and all eleven individual dispatch -gates exited **0**. Host verification passed 92 files / 1286 tests and all -three TypeScript checks; script verification passed 116 tests and its $0 -mock CLI. Lint reported **0 issues** with the existing denied cache-write -warnings. RSS-dependent allocation coverage remains unavailable. These green -gates do not override the five live anomaly deadlines. - -### Task 3 controller verification - -The five remaining 4096-batch anomaly admission timeouts are delays, not losses. When a -detector write cannot get the write gate within 10 s, the detector keeps its findings -and writes them with its next batch (up to the 10,000 episodes the store retains), and -the store merges overlapping episodes. The panel probe, which calls the store directly, -counts admission timeouts separately, still fails on any other panel, publication, -version or anomaly error, and requires at least one anomaly write to succeed. - -Rerun of `TestPanelsRemainUsableDuringBacklog` at 4096 batches (same settings: 4 GB, four -threads, five read connections), measured with `/usr/bin/time -l`: - -| Result | Value | -|---|---| -| Outcome | PASS in 316 s | -| Unexpected failures | 0 | -| Anomaly writes delayed at admission | 3 | -| Drain | complete in 51 s after the combined phase | -| Spans / logs (raw = rolled = expected) | 2,113,536 / 792,576 | -| Peak resident memory | 3.42 GB | - -No out-of-memory error occurred at this scale. - -## Task 6: raw-path panel latency at 24 hours and at 10x volume - -The opt-in `readbench` benchmark `BenchmarkPanelQueries24Hours` runs the production panel -executor on the raw path, with a warm read cache and default production settings. It has -two inputs: a generated regression fixture, and replayed demo telemetry. The replay -holds 126 batches, 2,220,983 spans and 860,377 logs over about 41 hours. Run A uses its -most recent 24 hours. Run B tiles the same 24 hours to 10x volume. Each shape has 100 -warm samples. - -Measured on an idle machine (Apple silicon, 14 logical CPUs, default production DuckDB -sizing): - -| Shape | Run A p95 (ms) | Run B p95 (ms) | -|---|---:|---:| -| traces_all | 24.4 | 176.8 | -| traces_checkout | 23.4 | 164.2 | -| traces_selected | 24.0 | 191.4 | -| count | 47.1 | 304.3 | -| rate | 42.2 | 292.5 | -| logs | 17.4 | 105.8 | -| traffic | 38.5 | 265.9 | -| latency | 31.0 | 219.6 | -| errors | 30.0 | 215.2 | -| log_volume | 12.6 | 71.0 | -| service_counts | 23.8 | 139.5 | -| severity_counts | 6.6 | 38.1 | -| p95_by_service | 26.1 | 184.8 | -| error_rate_window | 19.0 | 130.1 | -| twelve panels in one request | 199.9 | 1,588.0 | - -Run A meets S7: the slowest panel shape has a p95 of 47.1 ms against the 500 ms target. -**Rollup routing is not needed.** M3 left only the trace-candidate read caches -(`readCacheVersion` 4), and the raw path already meets S7, so M4 adds no routing or cache. - -At 10x volume no panel shape crosses 500 ms; the slowest is the span-count stat at -304 ms. A complete twelve-panel request takes 1.59 s at p95. An earlier run on a loaded -machine measured up to 1.26 s for one shape, which shows how much contention matters. -The replay has no metric points, so these figures cover trace and log panels only. They -are inputs for the design of larger, whole-system views; they are not an M4 gate. - -Final rerun on the M4 candidate, idle machine, same settings: PASS in 334 s; 0 unexpected -failures; 13 anomaly writes succeeded and 1 was delayed at admission (its findings carry -into the next write); the backlog drained in 56 s with exact span and log counts -(2,113,536 / 792,576); peak resident memory 3.21 GB. - -### Metric panels - -A second replay carries all three signals: 8,221,759 metric points in the 24-hour window, -alongside traces and logs. The benchmark adds three metric shapes, each on the most -common metric of its type in the replay (the gauge is `container.cpu.utilization`). - -| Shape | Demo volume p95 (ms) | 10x volume p95 (ms) | -|---|---:|---:| -| gauge average by service | 33.1 | 256.7 | -| counter rate | 27.4 | 212.7 | -| histogram p95 | 23.2 | 166.7 | -| twelve panels in one request | 120.2 | 1,098.7 | - -Two limits of the metric measures show up here. A counter's rate is the rate of -data points, not the rate of the counter's increase, and a histogram's p95 is computed -from each point's sum, not from its buckets. - -## Scale limit and next step - -The replayed demo carries about 15 spans per second. At 10x, about 150 spans per second, -every single panel stays under 500 ms, but a complete twelve-panel request takes 1.1 to -1.6 s on the server alone, at or past S6's 1.5 s budget for the whole first render, on a -14-core development machine. A larger system will need precomputed per-minute summaries -for spans, logs and metrics, which milestone 3 removed. They are planned for the next -milestone, together with a whole-system view, and sized against a target rate of 1,000 to -5,000 spans per second with host metrics. M4 adds no summaries or routing. diff --git a/docs/benchmarks/duckdb2-followups-2026-10-01.json b/docs/benchmarks/duckdb2-followups-2026-10-01.json deleted file mode 100644 index 8fa9a6f3b..000000000 --- a/docs/benchmarks/duckdb2-followups-2026-10-01.json +++ /dev/null @@ -1,1372 +0,0 @@ -{ - "engine": "v2.0.0-alpha43763", - "platform": "linux/arm64", - "container_cpus": 4, - "container_memory_bytes": 6442450944, - "harness_commit": "a9843c1fd55b9465d4d77a42fcf8805c467af731", - "fanout_binary_sha256": "5b61295e7c3ca5d4b0173864b7b2ced166668f073292975a2783d79b65ea2093", - "read_fixture": { - "batches": 312, - "rows_per_signal": 1200000, - "repetitions_per_case": 10, - "backfill_seconds": 3.593804327, - "binary_sha256": "fc75986a8a3ff43d8817d97e73a0f7d79f963e959fb663b6fff33c3d57eba5b5" - }, - "reads": [ - { - "cpu_seconds_per_request": 0.16519580000000006, - "implementation": "raw", - "median_ms": 47.126116499999995, - "method": "endpoints", - "p95_ms": 50.549057, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.049093300000000006, - "implementation": "completed", - "median_ms": 32.376495500000004, - "method": "endpoints", - "p95_ms": 35.043831, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.18912960000000006, - "implementation": "raw", - "median_ms": 60.809444, - "method": "trace", - "p95_ms": 66.316576, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.042178200000000034, - "implementation": "completed", - "median_ms": 20.041398, - "method": "trace", - "p95_ms": 31.82162, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.19311000000000006, - "implementation": "raw", - "median_ms": 61.007069, - "method": "logs", - "p95_ms": 63.729781, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.026694700000000182, - "implementation": "completed", - "median_ms": 18.208583, - "method": "logs", - "p95_ms": 19.203231, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.0719815999999998, - "implementation": "raw", - "median_ms": 30.054243, - "method": "trace", - "p95_ms": 30.696535, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.05076199999999993, - "implementation": "completed", - "median_ms": 29.025575, - "method": "trace", - "p95_ms": 30.262701, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.10608470000000025, - "implementation": "raw", - "median_ms": 39.626587, - "method": "logs", - "p95_ms": 42.288423, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.021031200000000184, - "implementation": "completed", - "median_ms": 15.439538500000001, - "method": "logs", - "p95_ms": 17.121895, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.02312410000000007, - "implementation": "raw", - "median_ms": 12.844431499999999, - "method": "endpoints", - "p95_ms": 15.067225, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.036625700000000094, - "implementation": "completed", - "median_ms": 23.9797975, - "method": "endpoints", - "p95_ms": 25.478737, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - } - ], - "mixed_summary": { - "peak_traces": { - "trial_name": "traces-p8-r1", - "offered_items_per_second": 160000, - "accepted_items_per_second": 95028.53951124784, - "average_cpu_cores": 2.444410773479345, - "peak_rss_bytes": 1079513088 - }, - "peak_logs": { - "trial_name": "logs-p8-r1", - "offered_items_per_second": 160000, - "accepted_items_per_second": 95678.62310134368, - "average_cpu_cores": 1.557826576029755, - "peak_rss_bytes": 1249972224 - }, - "peak_metrics": { - "trial_name": "metrics-p8-r1", - "offered_items_per_second": 160000, - "accepted_items_per_second": 96356.74388603629, - "average_cpu_cores": 1.0301004296651772, - "peak_rss_bytes": 1362259968 - }, - "peak_mixed": { - "trial_name": "mixed-p8-r1", - "offered_items_per_second": 480000, - "accepted_items_per_second": 141714.59332933233, - "average_cpu_cores": 2.8879418275981976, - "peak_rss_bytes": 1924915200 - }, - "sustained_mixed": { - "trial_name": "mixed-sustained", - "offered_items_per_second": 120000, - "accepted_items_per_second": 58962.631044860806, - "average_cpu_cores": 3.5281080676064134, - "peak_rss_bytes": 2799415296 - }, - "accepted_total": { - "spans": 5645254, - "log_records": 5667882, - "metric_points": 5769163 - }, - "cooldown_rss_bytes": 1591586816, - "cooldown_heap_bytes": 331751664, - "quality_gates": [ - { - "name": "generator-processes", - "passed": true, - "detail": "all exited successfully" - }, - { - "name": "server-dropped-rows", - "passed": true, - "detail": "zero" - }, - { - "name": "server-restarts", - "passed": true, - "detail": "none" - }, - { - "name": "authoritative-counters", - "passed": true, - "detail": "Fanout accepted rows in every phase" - }, - { - "name": "sustained-query-load", - "passed": false, - "detail": "6.5/20.0 completed/s, 47 errors, 768 shed" - }, - { - "name": "storage-repair-verify", - "passed": true, - "detail": "authoritative Parquet storage verified" - } - ], - "passed": false, - "warnings": [ - "Results describe this exact host, Fanout build, configuration, dataset, and workload; they are not universal capacity claims.", - "telemetrygen payloads are intentionally synthetic and highly compressible; storage size is not representative of production telemetry.", - "Offered load is a target. Headline throughput uses Fanout's accepted-row counters, never telemetrygen's intended or generated count.", - "Fanout and load generators shared one host; generator contention can make the observed peak lower than Fanout's backend ceiling." - ] - }, - "mixed_trials": [ - { - "settings": { - "name": "warmup", - "kind": "warmup", - "signal": "mixed", - "duration_seconds": 10, - "processes_per_signal": 1, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 10.163, - "offered_items_per_second": 60000, - "accepted": { - "spans": 125456, - "log_records": 114202, - "metric_points": 116548 - }, - "accepted_items_per_second": 35043.43055315175, - "acceptance_ratio": 0.5936766666666666, - "rows_dropped": 0, - "generator_processes": 3, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.6404516849828973, - "peak_rss_bytes": 471748608, - "end_rss_bytes": 471748608, - "peak_heap_bytes": 269988296, - "end_heap_bytes": 246992400, - "peak_queue_depth": 0, - "parquet_files_end": 0, - "parquet_bytes_end": 0, - "disk_growth_bytes": 38297695, - "allocated_bytes": 2034669184, - "gc_pause_seconds": 0.019587247000000002 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "traces-p1-r1", - "kind": "sweep", - "signal": "traces", - "duration_seconds": 15, - "processes_per_signal": 1, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.045, - "offered_items_per_second": 20000, - "accepted": { - "spans": 182926, - "log_records": 0, - "metric_points": 0 - }, - "accepted_items_per_second": 12155.446738419034, - "acceptance_ratio": 0.6097533333333334, - "rows_dropped": 0, - "generator_processes": 1, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.7435763587620623, - "peak_rss_bytes": 755339264, - "end_rss_bytes": 302735360, - "peak_heap_bytes": 361256744, - "end_heap_bytes": 12234400, - "peak_queue_depth": 0, - "parquet_files_end": 0, - "parquet_bytes_end": 0, - "disk_growth_bytes": 41871740, - "allocated_bytes": 2844720288, - "gc_pause_seconds": 0.014678532999999997 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "traces-p2-r1", - "kind": "sweep", - "signal": "traces", - "duration_seconds": 15, - "processes_per_signal": 2, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.054, - "offered_items_per_second": 40000, - "accepted": { - "spans": 383604, - "log_records": 0, - "metric_points": 0 - }, - "accepted_items_per_second": 25477.031115631533, - "acceptance_ratio": 0.63934, - "rows_dropped": 0, - "generator_processes": 2, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 1.0659595557029804, - "peak_rss_bytes": 800251904, - "end_rss_bytes": 642793472, - "peak_heap_bytes": 376765224, - "end_heap_bytes": 343188456, - "peak_queue_depth": 0, - "parquet_files_end": 0, - "parquet_bytes_end": 0, - "disk_growth_bytes": 66002033, - "allocated_bytes": 3902989912, - "gc_pause_seconds": 0.032851125999999994 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "traces-p4-r1", - "kind": "sweep", - "signal": "traces", - "duration_seconds": 15, - "processes_per_signal": 4, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.128, - "offered_items_per_second": 80000, - "accepted": { - "spans": 776440, - "log_records": 0, - "metric_points": 0 - }, - "accepted_items_per_second": 51294.19525963265, - "acceptance_ratio": 0.6470333333333333, - "rows_dropped": 0, - "generator_processes": 4, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 1.9164965798798916, - "peak_rss_bytes": 1002156032, - "end_rss_bytes": 992190464, - "peak_heap_bytes": 504169848, - "end_heap_bytes": 504169848, - "peak_queue_depth": 0, - "parquet_files_end": 0, - "parquet_bytes_end": 0, - "disk_growth_bytes": 97746839, - "allocated_bytes": 9298965232, - "gc_pause_seconds": 0.076670153 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "traces-p8-r1", - "kind": "sweep", - "signal": "traces", - "duration_seconds": 15, - "processes_per_signal": 8, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.089, - "offered_items_per_second": 160000, - "accepted": { - "spans": 1434130, - "log_records": 0, - "metric_points": 0 - }, - "accepted_items_per_second": 95028.53951124784, - "acceptance_ratio": 0.5975541666666667, - "rows_dropped": 0, - "generator_processes": 8, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 2.444410773479345, - "peak_rss_bytes": 1079513088, - "end_rss_bytes": 882618368, - "peak_heap_bytes": 470294240, - "end_heap_bytes": 470294240, - "peak_queue_depth": 0, - "parquet_files_end": 306, - "parquet_bytes_end": 112041274, - "disk_growth_bytes": 88845891, - "allocated_bytes": 12774862800, - "gc_pause_seconds": 0.19572627399999998 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "logs-p1-r1", - "kind": "sweep", - "signal": "logs", - "duration_seconds": 15, - "processes_per_signal": 1, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.043, - "offered_items_per_second": 20000, - "accepted": { - "spans": 0, - "log_records": 176000, - "metric_points": 0 - }, - "accepted_items_per_second": 11698.23787103301, - "acceptance_ratio": 0.5866666666666667, - "rows_dropped": 0, - "generator_processes": 1, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 1.2529078628918884, - "peak_rss_bytes": 1392234496, - "end_rss_bytes": 558174208, - "peak_heap_bytes": 478157376, - "end_heap_bytes": 18119232, - "peak_queue_depth": 0, - "parquet_files_end": 306, - "parquet_bytes_end": 112041274, - "disk_growth_bytes": 220365320, - "allocated_bytes": 2359544472, - "gc_pause_seconds": 0.01021732000000003 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "logs-p2-r1", - "kind": "sweep", - "signal": "logs", - "duration_seconds": 15, - "processes_per_signal": 2, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.065, - "offered_items_per_second": 40000, - "accepted": { - "spans": 0, - "log_records": 351895, - "metric_points": 0 - }, - "accepted_items_per_second": 23355.407160630726, - "acceptance_ratio": 0.5864916666666666, - "rows_dropped": 0, - "generator_processes": 2, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.4745482635402876, - "peak_rss_bytes": 967979008, - "end_rss_bytes": 672571392, - "peak_heap_bytes": 233895840, - "end_heap_bytes": 25513472, - "peak_queue_depth": 0, - "parquet_files_end": 306, - "parquet_bytes_end": 112041274, - "disk_growth_bytes": 23779647, - "allocated_bytes": 2166288496, - "gc_pause_seconds": 0.01639634299999998 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "logs-p4-r1", - "kind": "sweep", - "signal": "logs", - "duration_seconds": 15, - "processes_per_signal": 4, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.054, - "offered_items_per_second": 80000, - "accepted": { - "spans": 0, - "log_records": 720570, - "metric_points": 0 - }, - "accepted_items_per_second": 47857.40530173436, - "acceptance_ratio": 0.600475, - "rows_dropped": 0, - "generator_processes": 4, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.3845488664488413, - "peak_rss_bytes": 708808704, - "end_rss_bytes": 642408448, - "peak_heap_bytes": 70058856, - "end_heap_bytes": 36165304, - "peak_queue_depth": 0, - "parquet_files_end": 306, - "parquet_bytes_end": 112041274, - "allocated_bytes": 2956584192, - "gc_pause_seconds": 0.041946786999999985 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "logs-p8-r1", - "kind": "sweep", - "signal": "logs", - "duration_seconds": 15, - "processes_per_signal": 8, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.102, - "offered_items_per_second": 160000, - "accepted": { - "spans": 0, - "log_records": 1445166, - "metric_points": 0 - }, - "accepted_items_per_second": 95678.62310134368, - "acceptance_ratio": 0.6021525, - "rows_dropped": 0, - "generator_processes": 8, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 1.557826576029755, - "peak_rss_bytes": 1249972224, - "end_rss_bytes": 775282688, - "peak_heap_bytes": 92475808, - "end_heap_bytes": 73124928, - "peak_queue_depth": 0, - "parquet_files_end": 103, - "parquet_bytes_end": 161934790, - "disk_growth_bytes": 32122800, - "allocated_bytes": 6202282688, - "gc_pause_seconds": 0.17198322700000007 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "metrics-p1-r1", - "kind": "sweep", - "signal": "metrics", - "duration_seconds": 15, - "processes_per_signal": 1, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.057, - "offered_items_per_second": 20000, - "accepted": { - "spans": 0, - "log_records": 0, - "metric_points": 173872 - }, - "accepted_items_per_second": 11545.553317774495, - "acceptance_ratio": 0.5795733333333334, - "rows_dropped": 0, - "generator_processes": 1, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.13346923120874338, - "peak_rss_bytes": 863858688, - "end_rss_bytes": 689082368, - "peak_heap_bytes": 34484168, - "end_heap_bytes": 13210152, - "peak_queue_depth": 0, - "parquet_files_end": 103, - "parquet_bytes_end": 161934790, - "allocated_bytes": 797385984, - "gc_pause_seconds": 0.016522996999999928 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "metrics-p2-r1", - "kind": "sweep", - "signal": "metrics", - "duration_seconds": 15, - "processes_per_signal": 2, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.041, - "offered_items_per_second": 40000, - "accepted": { - "spans": 0, - "log_records": 0, - "metric_points": 352000 - }, - "accepted_items_per_second": 23398.84715791845, - "acceptance_ratio": 0.5866666666666667, - "rows_dropped": 0, - "generator_processes": 2, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.09904625643550768, - "peak_rss_bytes": 710774784, - "end_rss_bytes": 681082880, - "peak_heap_bytes": 34729632, - "end_heap_bytes": 25248712, - "peak_queue_depth": 0, - "parquet_files_end": 103, - "parquet_bytes_end": 161934790, - "disk_growth_bytes": 2488802, - "allocated_bytes": 1368284816, - "gc_pause_seconds": 0.02855761899999998 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "metrics-p4-r1", - "kind": "sweep", - "signal": "metrics", - "duration_seconds": 15, - "processes_per_signal": 4, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.069, - "offered_items_per_second": 80000, - "accepted": { - "spans": 0, - "log_records": 0, - "metric_points": 726320 - }, - "accepted_items_per_second": 48193.40330390228, - "acceptance_ratio": 0.6052666666666666, - "rows_dropped": 0, - "generator_processes": 4, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 0.2879713767195397, - "peak_rss_bytes": 728825856, - "end_rss_bytes": 634359808, - "peak_heap_bytes": 60396008, - "end_heap_bytes": 27385208, - "peak_queue_depth": 0, - "parquet_files_end": 103, - "parquet_bytes_end": 161934790, - "disk_growth_bytes": 11074546, - "allocated_bytes": 2856073680, - "gc_pause_seconds": 0.07447297900000005 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "metrics-p8-r1", - "kind": "sweep", - "signal": "metrics", - "duration_seconds": 15, - "processes_per_signal": 8, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "repetition": 1 - }, - "elapsed_seconds": 15.132, - "offered_items_per_second": 160000, - "accepted": { - "spans": 0, - "log_records": 0, - "metric_points": 1458306 - }, - "accepted_items_per_second": 96356.74388603629, - "acceptance_ratio": 0.6076275, - "rows_dropped": 0, - "generator_processes": 8, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 1.0301004296651772, - "peak_rss_bytes": 1362259968, - "end_rss_bytes": 829861888, - "peak_heap_bytes": 69150144, - "end_heap_bytes": 36629536, - "peak_queue_depth": 1, - "parquet_files_end": 206, - "parquet_bytes_end": 183788729, - "disk_growth_bytes": 10407877, - "allocated_bytes": 5270018288, - "gc_pause_seconds": 0.205670951 - }, - "query_latency_ms": { - "count": 0, - "mean_ms": 0, - "p50_ms": 0, - "p95_ms": 0, - "p99_ms": 0, - "max_ms": 0 - }, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "mixed-p1-r1", - "kind": "sweep", - "signal": "mixed", - "duration_seconds": 15, - "processes_per_signal": 1, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "query_rate": 20, - "repetition": 1 - }, - "elapsed_seconds": 15.105, - "offered_items_per_second": 60000, - "accepted": { - "spans": 178638, - "log_records": 166238, - "metric_points": 169703 - }, - "accepted_items_per_second": 34025.17120551659, - "acceptance_ratio": 0.5717544444444445, - "rows_dropped": 0, - "generator_processes": 3, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 2.810199748210586, - "peak_rss_bytes": 1226956800, - "end_rss_bytes": 1071779840, - "peak_heap_bytes": 200814480, - "end_heap_bytes": 25165320, - "peak_queue_depth": 0, - "parquet_files_end": 206, - "parquet_bytes_end": 183788729, - "disk_growth_bytes": 20469831, - "allocated_bytes": 4143554184, - "gc_pause_seconds": 0.313339707 - }, - "query_latency_ms": { - "count": 302, - "mean_ms": 76.44, - "p50_ms": 54.22, - "p95_ms": 232.46, - "p99_ms": 577.03, - "max_ms": 664.91 - }, - "query_attempts": 302, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "mixed-p2-r1", - "kind": "sweep", - "signal": "mixed", - "duration_seconds": 15, - "processes_per_signal": 2, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "query_rate": 20, - "repetition": 1 - }, - "elapsed_seconds": 15.151, - "offered_items_per_second": 120000, - "accepted": { - "spans": 304612, - "log_records": 279083, - "metric_points": 297167 - }, - "accepted_items_per_second": 57541.90946573893, - "acceptance_ratio": 0.4893677777777778, - "rows_dropped": 0, - "generator_processes": 6, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 3.353108901140449, - "peak_rss_bytes": 1529233408, - "end_rss_bytes": 1244499968, - "peak_heap_bytes": 290514592, - "end_heap_bytes": 58865600, - "peak_queue_depth": 1, - "parquet_files_end": 206, - "parquet_bytes_end": 183788729, - "allocated_bytes": 5983231848, - "gc_pause_seconds": 0.5904725389999999 - }, - "query_latency_ms": { - "count": 306, - "mean_ms": 579.46, - "p50_ms": 332.05, - "p95_ms": 1853.81, - "p99_ms": 2626.92, - "max_ms": 3481.45 - }, - "query_attempts": 306, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit" - ] - }, - { - "settings": { - "name": "mixed-p4-r1", - "kind": "sweep", - "signal": "mixed", - "duration_seconds": 15, - "processes_per_signal": 4, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "query_rate": 20, - "repetition": 1 - }, - "elapsed_seconds": 15.311, - "offered_items_per_second": 240000, - "accepted": { - "spans": 593658, - "log_records": 533307, - "metric_points": 544276 - }, - "accepted_items_per_second": 108195.93052474891, - "acceptance_ratio": 0.4642336111111111, - "rows_dropped": 0, - "generator_processes": 12, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 3.2603000173083094, - "peak_rss_bytes": 1586159616, - "end_rss_bytes": 1428008960, - "peak_heap_bytes": 353143856, - "end_heap_bytes": 132104912, - "peak_queue_depth": 7, - "parquet_files_end": 206, - "parquet_bytes_end": 183788729, - "disk_growth_bytes": 325161567, - "allocated_bytes": 8585677136, - "gc_pause_seconds": 0.9356431259999998 - }, - "query_latency_ms": { - "count": 262, - "mean_ms": 2777.53, - "p50_ms": 2394.97, - "p95_ms": 6950.86, - "p99_ms": 10047.83, - "max_ms": 10665.08 - }, - "query_attempts": 308, - "query_shed": 46, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit", - "the requested query rate was not sustained" - ] - }, - { - "settings": { - "name": "mixed-p8-r1", - "kind": "sweep", - "signal": "mixed", - "duration_seconds": 15, - "processes_per_signal": 8, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "query_rate": 20, - "repetition": 1 - }, - "elapsed_seconds": 15.6, - "offered_items_per_second": 480000, - "accepted": { - "spans": 565430, - "log_records": 839637, - "metric_points": 866436 - }, - "accepted_items_per_second": 141714.59332933233, - "acceptance_ratio": 0.3154865277777778, - "rows_dropped": 0, - "generator_processes": 24, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 2.8879418275981976, - "peak_rss_bytes": 1924915200, - "end_rss_bytes": 1614061568, - "peak_heap_bytes": 290342376, - "end_heap_bytes": 283401632, - "peak_queue_depth": 55, - "parquet_files_end": 206, - "parquet_bytes_end": 183788729, - "disk_growth_bytes": 56715770, - "allocated_bytes": 9935051928, - "gc_pause_seconds": 0.7110637479999999 - }, - "query_latency_ms": { - "count": 155, - "mean_ms": 5035.68, - "p50_ms": 4833.55, - "p95_ms": 11558.23, - "p99_ms": 14173.79, - "max_ms": 14820.64 - }, - "query_attempts": 320, - "query_failures": 8, - "query_shed": 157, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit", - "the requested query rate was not sustained" - ] - }, - { - "settings": { - "name": "mixed-sustained", - "kind": "sustained", - "signal": "mixed", - "duration_seconds": 60, - "processes_per_signal": 2, - "workers_per_process": 8, - "rate_per_worker": 2500, - "batch_size": 1000, - "query_rate": 20, - "repetition": 1 - }, - "elapsed_seconds": 60.201, - "offered_items_per_second": 120000, - "accepted": { - "spans": 1225816, - "log_records": 1155986, - "metric_points": 1181083 - }, - "accepted_items_per_second": 58962.631044860806, - "acceptance_ratio": 0.49484513888888887, - "rows_dropped": 0, - "generator_processes": 6, - "generator_failures": 0, - "fanout_restarted": false, - "resources": { - "average_cpu_cores": 3.5281080676064134, - "peak_rss_bytes": 2799415296, - "end_rss_bytes": 2799415296, - "peak_heap_bytes": 335405800, - "end_heap_bytes": 62366256, - "peak_queue_depth": 1, - "parquet_files_end": 1961, - "parquet_bytes_end": 311126046, - "allocated_bytes": 20909988768, - "gc_pause_seconds": 2.9660317000000003 - }, - "query_latency_ms": { - "count": 393, - "mean_ms": 5097.47, - "p50_ms": 4527, - "p95_ms": 12470.1, - "p99_ms": 14548.95, - "max_ms": 14963.32 - }, - "query_attempts": 1208, - "query_failures": 47, - "query_shed": 768, - "classification": "saturated-or-generator-limited", - "classification_notes": [ - "the target was not sustained; separate load-generator hosts are needed to attribute the limit", - "the requested query rate was not sustained" - ] - } - ], - "durable_ingestion": { - "baseline": { - "repetitions": [ - { - "allocated_bytes_per_row": 3841.016279264214, - "concurrency": 16, - "cpu_cores_client_and_server": 1.386670723965906, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 32.983173, - "p95_ms": 37.233546, - "p99_ms": 46.579118, - "rows": 4784000, - "rows_per_second": 478232.5354001488, - "seconds": 10.00350174, - "cpu_seconds_per_million_rows": 2.8995742056856186 - }, - { - "allocated_bytes_per_row": 3827.215571197411, - "concurrency": 16, - "cpu_cores_client_and_server": 1.3389345902710774, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 32.267016, - "p95_ms": 35.18989, - "p99_ms": 36.741222, - "rows": 4944000, - "rows_per_second": 493113.6031033487, - "seconds": 10.026087232, - "cpu_seconds_per_million_rows": 2.715265978964401 - }, - { - "allocated_bytes_per_row": 3827.789229773463, - "concurrency": 16, - "cpu_cores_client_and_server": 1.3422942175040822, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 32.22202, - "p95_ms": 35.682391, - "p99_ms": 37.382098, - "rows": 4944000, - "rows_per_second": 493050.47558628663, - "seconds": 10.027370918, - "cpu_seconds_per_million_rows": 2.722427588996764 - } - ], - "median": { - "rows_per_second": 493050.47558628663, - "allocated_bytes_per_row": 3827.789229773463, - "cpu_cores_client_and_server": 1.3422942175040822, - "cpu_seconds_per_million_rows": 2.722427588996764, - "p50_ms": 32.267016, - "p95_ms": 35.682391, - "p99_ms": 37.382098 - }, - "binary_sha256": "d588363630fa003e81a8582acd50416ab77bed189b3f2daa662d32ed30dd49ce" - }, - "final": { - "repetitions": [ - { - "allocated_bytes_per_row": 4018.0189930362117, - "concurrency": 16, - "cpu_cores_client_and_server": 2.0168743622096437, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 27.349099, - "p95_ms": 32.508177, - "p99_ms": 44.273124, - "rows": 5744000, - "rows_per_second": 573131.9162908524, - "seconds": 10.022125512, - "cpu_seconds_per_million_rows": 3.5190403899721447 - }, - { - "allocated_bytes_per_row": 4017.786380108992, - "concurrency": 16, - "cpu_cores_client_and_server": 2.017148073950804, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 26.789268, - "p95_ms": 31.753598, - "p99_ms": 43.314047, - "rows": 5872000, - "rows_per_second": 586812.7884127768, - "seconds": 10.006598554, - "cpu_seconds_per_million_rows": 3.437464407356948 - }, - { - "allocated_bytes_per_row": 4037.289988031915, - "concurrency": 16, - "cpu_cores_client_and_server": 2.0203992820509, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 26.447858, - "p95_ms": 29.780439, - "p99_ms": 32.90123, - "rows": 6016000, - "rows_per_second": 600172.8755069904, - "seconds": 10.023778557, - "cpu_seconds_per_million_rows": 3.366362200797873 - } - ], - "median": { - "rows_per_second": 586812.7884127768, - "allocated_bytes_per_row": 4018.0189930362117, - "cpu_cores_client_and_server": 2.017148073950804, - "cpu_seconds_per_million_rows": 3.437464407356948, - "p50_ms": 26.789268, - "p95_ms": 31.753598, - "p99_ms": 43.314047 - }, - "binary_sha256": "1ea1109e8d50ec07ecd9a2269a7cdbbfc1e82ee04301eb525fccbcaaad655d36" - } - }, - "prior_read_trial": [ - { - "cpu_seconds_per_request": 0.16752330000000004, - "implementation": "raw", - "median_ms": 47.8396225, - "method": "endpoints", - "p95_ms": 50.53142, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.046351799999999874, - "implementation": "completed", - "median_ms": 31.010892, - "method": "endpoints", - "p95_ms": 33.511919, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.18807199999999985, - "implementation": "raw", - "median_ms": 60.092862, - "method": "trace", - "p95_ms": 64.072038, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.036477199999999856, - "implementation": "completed", - "median_ms": 19.475008000000003, - "method": "trace", - "p95_ms": 20.060926, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.18967860000000022, - "implementation": "raw", - "median_ms": 60.604592499999995, - "method": "logs", - "p95_ms": 65.922501, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.027257000000000177, - "implementation": "completed", - "median_ms": 18.129984, - "method": "logs", - "p95_ms": 22.312432, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.02699890000000025, - "implementation": "raw", - "median_ms": 14.6436625, - "method": "endpoints", - "p95_ms": 17.652086, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.0363745999999999, - "implementation": "completed", - "median_ms": 23.761456, - "method": "endpoints", - "p95_ms": 35.10009, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.0638614999999998, - "implementation": "raw", - "median_ms": 27.057069, - "method": "trace", - "p95_ms": 28.324614, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.04826499999999996, - "implementation": "completed", - "median_ms": 28.1259885, - "method": "trace", - "p95_ms": 29.4372, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.10301620000000007, - "implementation": "raw", - "median_ms": 38.5614945, - "method": "logs", - "p95_ms": 40.326145, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.020283799999999984, - "implementation": "completed", - "median_ms": 15.2523725, - "method": "logs", - "p95_ms": 16.176833, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - } - ] -} diff --git a/docs/benchmarks/duckdb2-followups-2026-10-01.md b/docs/benchmarks/duckdb2-followups-2026-10-01.md deleted file mode 100644 index d9e745552..000000000 --- a/docs/benchmarks/duckdb2-followups-2026-10-01.md +++ /dev/null @@ -1,154 +0,0 @@ -# DuckDB 2 review follow-ups — 2026-10-01 - -This report retains the measurements before the nine-comment review at -`cd916d5`. They are historical results, not measurements of the subsequent -aggregate-only cache. See [the subsequent nine-comment review report](duckdb2-review-2026-10-01.md). #277 remains open for the encoding CPU regression. -These results do not establish general production capacity. -The complete measured values and binary identities are in -[the accompanying JSON](duckdb2-followups-2026-10-01.json). - -## Changes measured - -| Finding | Implementation | -|---|---| -| #273: file binding | Explicit immutable-file lists from actual span, log, and metric event-time footer bounds; unknown statistics include the file; newest event bounds first. | -| #274: endpoint tail | Completed-batch minute histograms plus exact partial minutes and uncached active files, without the five-minute endpoint watermark. | -| #275: notable traces | Exact candidate index for all namespace/service scope combinations; merge uncached contributions and recompute partial-window traces. | -| #276: log histogram | Scoped severity minute counts and exact boundary counts; text searches retain redacted-body matching. | -| #277: ingest regression | Reuse scalar buffers, dispatch by column, encode shared canonical resources once with a bounded identity cache, and shorten group admission from 20 to 10ms. | - -All derived contributions and the completed-batch marker commit atomically. -Dashboard reads pin immutable files and use one DuckDB transaction for cache -markers and rows. Compaction/retention invalidate retired contributions. -Narrow scalar projections answer partial windows without decoding attributes or -bodies. They consume additional rebuildable DuckDB storage. The cache worker -yields between bounded transactions and is independent of analytical rollup lag. -Format 3 and its physical schema are unchanged. No earlier-format reader exists. - -## Environment and method - -Native Linux ARM64, Go 1.27.1, pinned DuckDB `v2.0.0-alpha43763`, Docker limit -4 CPUs / 6 GiB. No concurrent benchmark containers. The baseline is the #270 -merge, `30ba9b9ba8a4217cc73637760d7acdcd5f89f15a`. - -The durable ingestion experiment uses 16 concurrent clients, 1,000 rows/export, -equal spans/logs/metrics, and no dashboard/background query work. Client and -server share the measured process. Three baseline and three final ten-second -measurement intervals alternate in the order baseline/final/final/baseline/ -baseline/final. CPU profiles cover the measurement intervals; post-load heap -profiles cover the whole experiment and must not be treated as interval-only -allocations. Every run waits for durable publication and then verifies storage. - -| Median metric | #270 | Final DuckDB 2 | -|---|---:|---:| -| Durable acknowledged rows/s | 493,050 | 586,813 | -| Go allocated bytes/row | 3,828 | 4,018 | -| Client + server CPU cores | 1.342 | 2.017 | -| CPU seconds per million rows | 2.722 | 3.437 | -| Export p50 | 32.27ms | 26.79ms | -| Export p95 | 35.68ms | 31.75ms | - -All six runs have zero export errors and pass durable storage verification. -Throughput increases **19%**, and p95 decreases **11%**. This closed-loop fixture is admission-limited: halving the admission window -accounts for most of the throughput gain. It does not demonstrate that the -encoding regression is resolved. Typed ingestion still costs -**26% more CPU per row** and **5% more Go allocation per row** than #270. -The admission change contributes to the throughput gain; this comparison does -not establish faster VARIANT encoding. Independent fixed-one-CPU trials from -the review measured 395k/373k rows/s for #270 versus 322k/298k for `cd916d5`, -about 19% slower. Keep #277 open; reducing acknowledgement latency does not -remove the normalized CPU regression. - -## Identical-file dashboard comparison - -The opt-in read fixture has 1.2 million spans and 1.2 million logs across twelve -hourly batches, plus 300 old tiny log/metric batches. Both paths read the same -format-3 files. The raw path runs the existing query kernels; the completed path -runs the actual production dashboard methods after cache backfill (3.59s). -No compaction or background rollup runs during measurement. Each case has a warmup -and ten measured requests. These p95 values are the maximum of ten samples, so -they are not a robust estimate of production tail latency. - -| Window / method | Raw median / p95 | Completed median / p95 | Raw / completed CPU ms per request | -|---|---:|---:|---:| -| 24h endpoints | 47.13 / 50.55ms | 32.38 / 35.04ms | 165.2 / 49.1 | -| 24h notable trace | 60.81 / 66.32ms | 20.04 / 31.82ms | 189.1 / 42.2 | -| 24h logs | 61.01 / 63.73ms | 18.21 / 19.20ms | 193.1 / 26.7 | -| 20m endpoints | 12.84 / 15.07ms | 23.98 / 25.48ms | 23.1 / 36.6 | -| 20m notable trace | 30.05 / 30.70ms | 29.03 / 30.26ms | 72.0 / 50.8 | -| 20m logs | 39.63 / 42.29ms | 15.44 / 17.12ms | 106.1 / 21.0 | - -Both windows end one second after the fixture's hour boundary and exercise exact -partial minutes. Broad reads improve with the caches fully built. This fixture measures -cache-current reads only; it contains no concurrent ingest or cache lag. Small endpoint windows still -pay additional histogram/union planning overhead, and narrow trace latency is -approximately flat. No claim of uniform latency improvement is made. Endpoints use -the established fixed-boundary histogram estimate rather than the raw kernel's -`approx_quantile`; cold and completed production reads use the same estimator. -The JSON also retains the prior identical-file trial; both show broad gains -and extra planning cost for narrow endpoint windows. - -## Mixed-load saturation diagnostic - -The unchanged adapted `../fanout-bench` harness at -`a9843c1fd55b9465d4d77a42fcf8805c467af731` runs the standard sweep with -telemetrygen 0.157.0 on the same host, 20 requested reads/s, fifteen-second sweep -phases, sixty-second sustained load, and fifteen-second cooldown. Sibling -repositories were not changed. The native server binary SHA256 is recorded in -JSON. Data and full logs are retained locally in the `fanout-review272-final` -Docker volume, under `/optimized`. - -| Diagnostic | Final result | -|---|---:| -| Mixed p1 accepted rows/s / completed reads/s | 34,025 / 20.0 | -| Mixed p2 accepted rows/s / completed reads/s | 57,542 / 20.2 | -| Mixed p8 accepted rows/s / completed reads/s | 141,715 / 9.9 | -| Sustained offered / accepted rows/s | 120,000 / 58,963 | -| Sustained completed reads/s / p95 | 6.5 / 12.47s | -| Sustained failed / shed reads | 47 / 768 | -| Cooldown RSS | 1.48 GiB | - -The p1 and p2 sweeps have zero failed or shed reads. The standard harness chooses -its sustained phase adaptively: this build selects **120,000** offered rows/s, -where the historical #270 and pre-follow-up runs selected **60,000**. Sustained -results therefore are **not a matched comparison**. Mixed p8 offered settings -match the historical baseline, which accepted 145,941 rows/s; the final -141,715 is within 3% in these single trials, with different accepted datasets. - -**The sustained 20-read/s capacity gate fails.** This is a failed saturation -diagnostic, not a publishable capacity result. Generator exits, server-dropped -rows, server restarts, and full authoritative storage verification all pass. -Same-host generators, adaptive sustained selection, short sweeps, and one trial -limit interpretation. CPU per row, narrow endpoint planning cost, and saturated -read capacity remain measured limitations. The independent review reproduced -cache backlog growth from 51 to 843 batches and read shedding under sustained -load; short sweeps and fully built-cache reads do not establish cache stability. - -## Reproduce - -On native Linux or macOS, build with the pinned wrapper: - -```sh -bash scripts/with-duckdb.sh go test -tags=transportbench ./internal/ingest \ - -run 'TestTransportComparison/durable=true/c=16/rep=1/shared' -v -count=1 -FANOUT_BENCH_PROFILE_DIR=/tmp/fanout-profiles \ - bash scripts/with-duckdb.sh go test -tags=transportbench ./internal/ingest \ - -run 'TestTransportComparison/durable=true/c=16/rep=1/shared' -v -count=1 -FANOUT_READ_BENCH_LARGE=1 \ - bash scripts/with-duckdb.sh go test -tags=readbench ./internal/observability \ - -run TestCompletedReadBenchmark -v -count=1 -``` - -Run one benchmark at a time. For the reported container limits, compile native -Linux test binaries with `go test -c` through the wrapper, then run each binary -in an otherwise idle `--cpus 4 --memory 6g` container. Repeat the ingestion -selector for repetitions 1, 2, and 3 on both builds. The baseline profiling overlay -changes only measurement collection, not the #270 writer. - -Local validation covers cold/warm parity, late publications, partial minutes, -trace scopes and ranking, duplicate log timestamps, compaction/retention, -transaction failure rollback, prompt worker draining and cancellation, public -SQL rejection of private caches, optional footer statistics and signed times, -and mixed shared/unique/null resources across chunks and row groups. The normal -`just check` gate and targeted query/observability/telemetry/storage race suite -pass; native Linux tests exercise the same pinned engine. diff --git a/docs/benchmarks/duckdb2-review-2026-10-01-harness.patch b/docs/benchmarks/duckdb2-review-2026-10-01-harness.patch deleted file mode 100644 index 609638aab..000000000 --- a/docs/benchmarks/duckdb2-review-2026-10-01-harness.patch +++ /dev/null @@ -1,69 +0,0 @@ ---- a/cmd/fanout-bench/main.go -+++ b/cmd/fanout-bench/main.go -@@ -147,0 +148 @@ -+ fs.StringVar(&cfg.FanoutCommit, "fanout-commit", "", "Fanout source identity") ---- a/internal/bench/managed.go -+++ b/internal/bench/managed.go -@@ -31,0 +32 @@ -+ FanoutCommit string -@@ -68,8 +68,0 @@ -- grpcPort, err := freePort() -- if err != nil { -- return err -- } -- httpOTLPPort, err := freePort() -- if err != nil { -- return err -- } -@@ -81,2 +74,2 @@ -- grpcAddr := fmt.Sprintf("127.0.0.1:%d", grpcPort) -- metricsURL := "http://" + httpAddr + "/-/metrics" -+ grpcAddr := httpAddr -+ metricsURL := "http://" + httpAddr + "/metrics" -@@ -90,3 +83 @@ -- "FANOUT_HTTP_ADDR="+httpAddr, -- "FANOUT_OTLP_GRPC_ADDR="+grpcAddr, -- fmt.Sprintf("FANOUT_OTLP_HTTP_ADDR=127.0.0.1:%d", httpOTLPPort), -+ "FANOUT_ADDR="+httpAddr, -@@ -186 +177 @@ -- cfg := Config{Host: managed.ProvisionedHost, SSHPort: managed.ProvisionedSSHPort, SSHIdentity: managed.ProvisionedSSHIdentity, Preset: managed.Preset, Endpoint: grpcAddr, MetricsURL: metricsURL, Insecure: true, Telemetrygen: managed.TelemetrygenPath, Token: ingestToken, OutputDir: managed.OutputDir, FanoutVersion: actualVersion, FanoutArtifact: managed.FanoutArtifact, DataDir: dataDir, SampleInterval: managed.SampleInterval, GeneratorTimeout: 30 * time.Second, Cooldown: managed.Cooldown, Repeat: managed.Repeat, QueryURL: baseURL, QueryCookie: queryCookie, QueryRate: managed.QueryRate, SameHost: true, BatchSize: managed.BatchSize, TelemetrygenVersion: managed.TelemetrygenVersion, ProcessOverride: managed.ProcessOverride, WorkerOverride: managed.WorkerOverride, RateOverride: managed.RateOverride, DurationOverride: managed.DurationOverride} -+ cfg := Config{Host: managed.ProvisionedHost, SSHPort: managed.ProvisionedSSHPort, SSHIdentity: managed.ProvisionedSSHIdentity, Preset: managed.Preset, Endpoint: grpcAddr, MetricsURL: metricsURL, Insecure: true, Telemetrygen: managed.TelemetrygenPath, Token: ingestToken, OutputDir: managed.OutputDir, FanoutVersion: actualVersion, FanoutCommit: managed.FanoutCommit, FanoutArtifact: managed.FanoutArtifact, DataDir: dataDir, SampleInterval: managed.SampleInterval, GeneratorTimeout: 30 * time.Second, Cooldown: managed.Cooldown, Repeat: managed.Repeat, QueryURL: baseURL, QueryCookie: queryCookie, QueryRate: managed.QueryRate, SameHost: true, BatchSize: managed.BatchSize, TelemetrygenVersion: managed.TelemetrygenVersion, ProcessOverride: managed.ProcessOverride, WorkerOverride: managed.WorkerOverride, RateOverride: managed.RateOverride, DurationOverride: managed.DurationOverride} ---- a/internal/bench/preset.go -+++ b/internal/bench/preset.go -@@ -13,0 +14,8 @@ -+ } -+ // Fixed review reproduction: no adaptive phase selection and five full -+ // minutes of mixed ingest with the production open-loop reader scheduler. -+ if cfg.Preset == "review272" { -+ return []PhaseSpec{ -+ {Name: "warmup", Kind: "warmup", Signal: Mixed, Duration: 10 * time.Second, DurationSeconds: 10, ProcessesPerSignal: 1, WorkersPerProcess: 8, RatePerWorker: 2500, BatchSize: 1000, Repetition: 1}, -+ {Name: "mixed-sustained", Kind: "sustained", Signal: Mixed, Duration: 5 * time.Minute, DurationSeconds: 300, ProcessesPerSignal: 2, WorkersPerProcess: 8, RatePerWorker: 2500, BatchSize: 1000, QueryRate: 20, Repetition: 1}, -+ }, nil ---- a/internal/bench/runner.go -+++ b/internal/bench/runner.go -@@ -333 +333,2 @@ -- AtSeconds: round(at, 3), CPUSeconds: s.total("process_cpu_seconds_total", nil), -+ ReadCachePendingBatches: s.total("fanout_read_cache_pending_batches", nil), -+ AtSeconds: round(at, 3), CPUSeconds: s.total("process_cpu_seconds_total", nil), ---- a/internal/bench/types.go -+++ b/internal/bench/types.go -@@ -126,9 +126,10 @@ -- AtSeconds float64 `json:"at_seconds"` -- CPUSeconds float64 `json:"cpu_seconds"` -- RSSBytes float64 `json:"rss_bytes"` -- HeapBytes float64 `json:"heap_bytes"` -- QueueDepth float64 `json:"queue_depth"` -- ParquetFiles float64 `json:"parquet_files"` -- ParquetBytes float64 `json:"parquet_bytes"` -- PublishWaiters float64 `json:"publish_waiters"` -- PublishTimeouts float64 `json:"publish_timeouts_total"` -+ ReadCachePendingBatches float64 `json:"fanout_read_cache_pending_batches"` -+ AtSeconds float64 `json:"at_seconds"` -+ CPUSeconds float64 `json:"cpu_seconds"` -+ RSSBytes float64 `json:"rss_bytes"` -+ HeapBytes float64 `json:"heap_bytes"` -+ QueueDepth float64 `json:"queue_depth"` -+ ParquetFiles float64 `json:"parquet_files"` -+ ParquetBytes float64 `json:"parquet_bytes"` -+ PublishWaiters float64 `json:"publish_waiters"` -+ PublishTimeouts float64 `json:"publish_timeouts_total"` diff --git a/docs/benchmarks/duckdb2-review-2026-10-01.json b/docs/benchmarks/duckdb2-review-2026-10-01.json deleted file mode 100644 index a37ad2589..000000000 --- a/docs/benchmarks/duckdb2-review-2026-10-01.json +++ /dev/null @@ -1,3137 +0,0 @@ -{ - "review_head": "cd916d5c0a3ba12acd2f21a3007368b1740c3be5", - "engine_version": "v2.0.0-alpha43763", - "go_version": "1.27.1", - "platform": "linux/arm64", - "cpu_limit": 4, - "memory_limit_bytes": 6442450944, - "benchmark_artifact_sha256": { - "fanout-cache-v5": "d4df1d59e13da33624d52054ea9039827748fd00182b9f9680043c73a0c9ae0a", - "fanout-cache-v6": "ea73930b184ec646597e2e25d5e305f8e0946919e6ecaea1777231feed56163f", - "file-binder-v6.test": "04b7e1d9a7a002d2e84020a62d3c5f55d21eb9ac7df63505851429d35b1404b6", - "batch-reads-v7.test": "a96eb28840b8a574e695fbf7f9634e5ed731d618ece319d36d8e36efe966d603", - "batch-reads-before.test": "c85ae9126dad890985c4e703d6fdad4533ee1d2f46424d0ae592d790982a684e", - "baseline-ingest.test": "d588363630fa003e81a8582acd50416ab77bed189b3f2daa662d32ed30dd49ce", - "ingest-v3.test": "d4c5b97c3f579c72f89929534c87b978b3c268af060da6f9877340082a9a236a", - "fanout-bench-v4": "ea2b038c5e0789e42daab1edd8a69aa272d0f4158843aa9bcc0f294972c89207" - }, - "harness_source_base_commit": "a9843c1fd55b9465d4d77a42fcf8805c467af731", - "harness_source_patch": "duckdb2-review-2026-10-01-harness.patch", - "build_identity_note": "Harness Go build-info inherits the containing Fanout checkout. Harness source identity is its base commit plus the complete patch and binary hash, not the containing checkout VCS claim. The sustained cache-v6 artifact predates the scoped ambiguity probe: its unscoped workload path is unchanged. The final scoped probe is measured with batch-reads-v7.test.", - "sustained": { - "elapsed_seconds": 300.154, - "nominal_offered_rows_per_second": 120000, - "accepted": { - "spans": 6864204, - "log_records": 6516659, - "metric_points": 6607440 - }, - "accepted_rows_per_second": 66085.52706253018, - "resources": { - "average_cpu_cores": 3.053846981499492, - "peak_rss_bytes": 3562332160, - "end_rss_bytes": 3198099456, - "peak_heap_bytes": 435610216, - "end_heap_bytes": 61116888, - "peak_queue_depth": 1, - "parquet_files_end": 4342, - "parquet_bytes_end": 434447968, - "disk_growth_bytes": 1334017571, - "allocated_bytes": 136859988440, - "gc_pause_seconds": 8.892415844 - }, - "query_latency_ms": { - "count": 4931, - "mean_ms": 1106.96, - "p50_ms": 88.12, - "p95_ms": 5955.36, - "p99_ms": 10630.47, - "max_ms": 14978.67 - }, - "query_attempts": 6049, - "query_failures": 19, - "query_shed": 1099, - "completed_queries_per_second": 16.42823350679984, - "cooldown_rss_bytes": 1048268800, - "quality_gates": [ - { - "name": "generator-processes", - "passed": true, - "detail": "all exited successfully" - }, - { - "name": "server-dropped-rows", - "passed": true, - "detail": "zero" - }, - { - "name": "server-restarts", - "passed": true, - "detail": "none" - }, - { - "name": "authoritative-counters", - "passed": true, - "detail": "Fanout accepted rows in every phase" - }, - { - "name": "sustained-query-load", - "passed": false, - "detail": "16.4/20.0 completed/s, 19 errors, 1099 shed" - }, - { - "name": "storage-repair-verify", - "passed": true, - "detail": "authoritative Parquet storage verified" - } - ], - "samples": [ - { - "fanout_read_cache_pending_batches": 4, - "at_seconds": 2.003, - "cpu_seconds": 7.82, - "rss_bytes": 336797696, - "heap_bytes": 43707800, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 4, - "at_seconds": 4.003, - "cpu_seconds": 9.89, - "rss_bytes": 335478784, - "heap_bytes": 42220056, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 7, - "at_seconds": 6.004, - "cpu_seconds": 12.19, - "rss_bytes": 334188544, - "heap_bytes": 18581960, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 5, - "at_seconds": 8.003, - "cpu_seconds": 14.5, - "rss_bytes": 348209152, - "heap_bytes": 21289096, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 3, - "at_seconds": 10.003, - "cpu_seconds": 17.27, - "rss_bytes": 593018880, - "heap_bytes": 285277264, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 40, - "at_seconds": 12.004, - "cpu_seconds": 21.7, - "rss_bytes": 765079552, - "heap_bytes": 192168456, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 4, - "at_seconds": 14.005, - "cpu_seconds": 24.29, - "rss_bytes": 358281216, - "heap_bytes": 18158192, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 4, - "at_seconds": 16.002, - "cpu_seconds": 27.25, - "rss_bytes": 400695296, - "heap_bytes": 36773912, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 6, - "at_seconds": 18.006, - "cpu_seconds": 30.07, - "rss_bytes": 412983296, - "heap_bytes": 26200768, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 7, - "at_seconds": 20.003, - "cpu_seconds": 33.34, - "rss_bytes": 641957888, - "heap_bytes": 253290416, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 6, - "at_seconds": 22.028, - "cpu_seconds": 38.53, - "rss_bytes": 665243648, - "heap_bytes": 186261112, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 21, - "at_seconds": 24.003, - "cpu_seconds": 42.9, - "rss_bytes": 442142720, - "heap_bytes": 25373160, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 4, - "at_seconds": 26.003, - "cpu_seconds": 46.15, - "rss_bytes": 413257728, - "heap_bytes": 21351672, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 6, - "at_seconds": 28.01, - "cpu_seconds": 49.51, - "rss_bytes": 418480128, - "heap_bytes": 18689584, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 7, - "at_seconds": 30.003, - "cpu_seconds": 53.88, - "rss_bytes": 672423936, - "heap_bytes": 214358304, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 36, - "at_seconds": 32.222, - "cpu_seconds": 59.52, - "rss_bytes": 841682944, - "heap_bytes": 285799520, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 34.004, - "cpu_seconds": 64.09, - "rss_bytes": 466870272, - "heap_bytes": 21048464, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 8, - "at_seconds": 36.003, - "cpu_seconds": 67.9, - "rss_bytes": 478158848, - "heap_bytes": 28370544, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 30, - "at_seconds": 38.017, - "cpu_seconds": 72.13, - "rss_bytes": 584593408, - "heap_bytes": 22750200, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 7, - "at_seconds": 40.006, - "cpu_seconds": 76.55, - "rss_bytes": 695418880, - "heap_bytes": 233002536, - "queue_depth": 1, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 42.011, - "cpu_seconds": 82.65, - "rss_bytes": 706039808, - "heap_bytes": 238168080, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 44.009, - "cpu_seconds": 87.46, - "rss_bytes": 517992448, - "heap_bytes": 20085240, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 46.011, - "cpu_seconds": 92.02, - "rss_bytes": 514842624, - "heap_bytes": 11922096, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 48.003, - "cpu_seconds": 96.29, - "rss_bytes": 519180288, - "heap_bytes": 15403992, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 50.097, - "cpu_seconds": 101.14, - "rss_bytes": 671211520, - "heap_bytes": 151603632, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 52.004, - "cpu_seconds": 108.49, - "rss_bytes": 1020702720, - "heap_bytes": 361381008, - "queue_depth": 0, - "parquet_files": 0, - "parquet_bytes": 0, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 7, - "at_seconds": 54.01, - "cpu_seconds": 115.35, - "rss_bytes": 937611264, - "heap_bytes": 314400912, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 56.714, - "cpu_seconds": 121.92, - "rss_bytes": 743788544, - "heap_bytes": 32752392, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 58.004, - "cpu_seconds": 128.68, - "rss_bytes": 819073024, - "heap_bytes": 170097152, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 60.013, - "cpu_seconds": 133.73, - "rss_bytes": 629583872, - "heap_bytes": 24843736, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 62.005, - "cpu_seconds": 137.98, - "rss_bytes": 564883456, - "heap_bytes": 17340432, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 64.017, - "cpu_seconds": 142.37, - "rss_bytes": 563535872, - "heap_bytes": 22579008, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 66.005, - "cpu_seconds": 146.92, - "rss_bytes": 579686400, - "heap_bytes": 25329208, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 68.005, - "cpu_seconds": 151.53, - "rss_bytes": 574836736, - "heap_bytes": 23666624, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 70.005, - "cpu_seconds": 156.81, - "rss_bytes": 777928704, - "heap_bytes": 214931216, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 51, - "at_seconds": 72.008, - "cpu_seconds": 163.05, - "rss_bytes": 840839168, - "heap_bytes": 205725592, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 74.013, - "cpu_seconds": 169.66, - "rss_bytes": 898129920, - "heap_bytes": 178165040, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 76.005, - "cpu_seconds": 175.25, - "rss_bytes": 617504768, - "heap_bytes": 14660832, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 78.005, - "cpu_seconds": 179.79, - "rss_bytes": 661741568, - "heap_bytes": 18602352, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 80.005, - "cpu_seconds": 184.6, - "rss_bytes": 796381184, - "heap_bytes": 141715608, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 82.004, - "cpu_seconds": 190.71, - "rss_bytes": 651091968, - "heap_bytes": 17920528, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 84.003, - "cpu_seconds": 195.17, - "rss_bytes": 626724864, - "heap_bytes": 18284048, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 86.011, - "cpu_seconds": 199.82, - "rss_bytes": 647811072, - "heap_bytes": 27671112, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 88.002, - "cpu_seconds": 204.46, - "rss_bytes": 642605056, - "heap_bytes": 19882264, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 90.238, - "cpu_seconds": 209.6, - "rss_bytes": 947892224, - "heap_bytes": 206103064, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 92.009, - "cpu_seconds": 216.26, - "rss_bytes": 994648064, - "heap_bytes": 300378408, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 94.007, - "cpu_seconds": 222.19, - "rss_bytes": 657219584, - "heap_bytes": 22210688, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 96.003, - "cpu_seconds": 227.02, - "rss_bytes": 672452608, - "heap_bytes": 20411744, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 8, - "at_seconds": 98.008, - "cpu_seconds": 231.95, - "rss_bytes": 649474048, - "heap_bytes": 34383560, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 100.012, - "cpu_seconds": 237.27, - "rss_bytes": 831995904, - "heap_bytes": 206960560, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 102.006, - "cpu_seconds": 243.82, - "rss_bytes": 1027526656, - "heap_bytes": 171023680, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 104.008, - "cpu_seconds": 250.38, - "rss_bytes": 775327744, - "heap_bytes": 27949488, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 106.008, - "cpu_seconds": 255.51, - "rss_bytes": 691048448, - "heap_bytes": 21422256, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 8, - "at_seconds": 108.007, - "cpu_seconds": 260.82, - "rss_bytes": 710938624, - "heap_bytes": 36213328, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 110.166, - "cpu_seconds": 266.38, - "rss_bytes": 852226048, - "heap_bytes": 183996256, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 112.223, - "cpu_seconds": 273.75, - "rss_bytes": 936796160, - "heap_bytes": 40766744, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 114.011, - "cpu_seconds": 281.15, - "rss_bytes": 833261568, - "heap_bytes": 20026912, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 116.054, - "cpu_seconds": 288.42, - "rss_bytes": 1191587840, - "heap_bytes": 48086496, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 118.107, - "cpu_seconds": 295.75, - "rss_bytes": 1292636160, - "heap_bytes": 28044448, - "queue_depth": 0, - "parquet_files": 597, - "parquet_bytes": 98343244, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 120.003, - "cpu_seconds": 301.83, - "rss_bytes": 954802176, - "heap_bytes": 187458216, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 122.006, - "cpu_seconds": 308.63, - "rss_bytes": 1079488512, - "heap_bytes": 136380528, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 124.017, - "cpu_seconds": 315.2, - "rss_bytes": 1015812096, - "heap_bytes": 156226800, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 126.017, - "cpu_seconds": 321.58, - "rss_bytes": 844378112, - "heap_bytes": 30556736, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 128.009, - "cpu_seconds": 327.79, - "rss_bytes": 765550592, - "heap_bytes": 24849672, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 130.006, - "cpu_seconds": 333.23, - "rss_bytes": 940605440, - "heap_bytes": 238979992, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 132.007, - "cpu_seconds": 339.12, - "rss_bytes": 750379008, - "heap_bytes": 21905496, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 134.011, - "cpu_seconds": 344.28, - "rss_bytes": 760762368, - "heap_bytes": 21608400, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 136.009, - "cpu_seconds": 349.48, - "rss_bytes": 768393216, - "heap_bytes": 15457368, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 138.004, - "cpu_seconds": 354.8, - "rss_bytes": 807518208, - "heap_bytes": 21171936, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 140.007, - "cpu_seconds": 360.52, - "rss_bytes": 925188096, - "heap_bytes": 195269760, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 142.004, - "cpu_seconds": 367.23, - "rss_bytes": 1107034112, - "heap_bytes": 312607480, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 144.015, - "cpu_seconds": 374.03, - "rss_bytes": 875528192, - "heap_bytes": 154544048, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 146.005, - "cpu_seconds": 380.46, - "rss_bytes": 782508032, - "heap_bytes": 21774920, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 148.004, - "cpu_seconds": 385.68, - "rss_bytes": 805621760, - "heap_bytes": 17597912, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 150.006, - "cpu_seconds": 391.37, - "rss_bytes": 948867072, - "heap_bytes": 185559008, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 152.01, - "cpu_seconds": 398.21, - "rss_bytes": 1236824064, - "heap_bytes": 285143160, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 154.01, - "cpu_seconds": 403.64, - "rss_bytes": 838578176, - "heap_bytes": 25968904, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 156.01, - "cpu_seconds": 409.08, - "rss_bytes": 829218816, - "heap_bytes": 29127392, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 158.01, - "cpu_seconds": 414.72, - "rss_bytes": 828338176, - "heap_bytes": 28287072, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 9, - "at_seconds": 160.014, - "cpu_seconds": 420.59, - "rss_bytes": 979824640, - "heap_bytes": 165093816, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 162.026, - "cpu_seconds": 427.52, - "rss_bytes": 1128681472, - "heap_bytes": 318549352, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 164.011, - "cpu_seconds": 434.36, - "rss_bytes": 1132134400, - "heap_bytes": 207614984, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 166.007, - "cpu_seconds": 441.3, - "rss_bytes": 1090134016, - "heap_bytes": 19977704, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 168.006, - "cpu_seconds": 446.9, - "rss_bytes": 860962816, - "heap_bytes": 29242768, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 170.138, - "cpu_seconds": 452.7, - "rss_bytes": 1006714880, - "heap_bytes": 170137336, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 172.028, - "cpu_seconds": 460.05, - "rss_bytes": 1554419712, - "heap_bytes": 435610216, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 174.891, - "cpu_seconds": 467.42, - "rss_bytes": 1751400448, - "heap_bytes": 41800688, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 10, - "at_seconds": 176.807, - "cpu_seconds": 474.82, - "rss_bytes": 1655918592, - "heap_bytes": 28042184, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 178.026, - "cpu_seconds": 482.23, - "rss_bytes": 1968574464, - "heap_bytes": 62504704, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 20, - "at_seconds": 180.788, - "cpu_seconds": 489.55, - "rss_bytes": 1396744192, - "heap_bytes": 42857584, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 182.29, - "cpu_seconds": 496.9, - "rss_bytes": 1964470272, - "heap_bytes": 20750144, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 184.614, - "cpu_seconds": 504.28, - "rss_bytes": 2089271296, - "heap_bytes": 25290360, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 17, - "at_seconds": 187.343, - "cpu_seconds": 511.65, - "rss_bytes": 1753260032, - "heap_bytes": 42761088, - "queue_depth": 0, - "parquet_files": 890, - "parquet_bytes": 202463215, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 188.351, - "cpu_seconds": 518.99, - "rss_bytes": 1746817024, - "heap_bytes": 31526984, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 19, - "at_seconds": 192.596, - "cpu_seconds": 526.27, - "rss_bytes": 1301815296, - "heap_bytes": 40017888, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 193.119, - "cpu_seconds": 535.8, - "rss_bytes": 1650618368, - "heap_bytes": 22378976, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 195.269, - "cpu_seconds": 540.93, - "rss_bytes": 1436434432, - "heap_bytes": 42070240, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 22, - "at_seconds": 196.988, - "cpu_seconds": 548.24, - "rss_bytes": 1515511808, - "heap_bytes": 20654560, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 17, - "at_seconds": 201.286, - "cpu_seconds": 555.57, - "rss_bytes": 1508245504, - "heap_bytes": 38182176, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 202.021, - "cpu_seconds": 567.53, - "rss_bytes": 1429553152, - "heap_bytes": 23933880, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 8, - "at_seconds": 202.188, - "cpu_seconds": 570.19, - "rss_bytes": 1172783104, - "heap_bytes": 37921032, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 12, - "at_seconds": 204.068, - "cpu_seconds": 577.48, - "rss_bytes": 1576767488, - "heap_bytes": 14477264, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 206.779, - "cpu_seconds": 584.8, - "rss_bytes": 1808478208, - "heap_bytes": 154871808, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 19, - "at_seconds": 208.114, - "cpu_seconds": 592.11, - "rss_bytes": 1070755840, - "heap_bytes": 30977080, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 11, - "at_seconds": 210.093, - "cpu_seconds": 599.48, - "rss_bytes": 1502879744, - "heap_bytes": 31563384, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 17, - "at_seconds": 212.3, - "cpu_seconds": 606.82, - "rss_bytes": 1714495488, - "heap_bytes": 32191080, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 29, - "at_seconds": 214.299, - "cpu_seconds": 613.99, - "rss_bytes": 1351618560, - "heap_bytes": 23175184, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 216.43, - "cpu_seconds": 621.3, - "rss_bytes": 1301000192, - "heap_bytes": 36119984, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 218.047, - "cpu_seconds": 628.63, - "rss_bytes": 1397272576, - "heap_bytes": 37225008, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 20, - "at_seconds": 220.986, - "cpu_seconds": 635.94, - "rss_bytes": 1762250752, - "heap_bytes": 63778288, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 223.947, - "cpu_seconds": 643.14, - "rss_bytes": 2869264384, - "heap_bytes": 24409936, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 224.251, - "cpu_seconds": 650.27, - "rss_bytes": 2094632960, - "heap_bytes": 26801184, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 17, - "at_seconds": 226.178, - "cpu_seconds": 657.6, - "rss_bytes": 2074718208, - "heap_bytes": 54883784, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 228.321, - "cpu_seconds": 664.96, - "rss_bytes": 1702998016, - "heap_bytes": 38919392, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 232.365, - "cpu_seconds": 672.28, - "rss_bytes": 1408356352, - "heap_bytes": 40437136, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 19, - "at_seconds": 232.47, - "cpu_seconds": 681.01, - "rss_bytes": 1168019456, - "heap_bytes": 38623224, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 25, - "at_seconds": 235.924, - "cpu_seconds": 687.12, - "rss_bytes": 1509343232, - "heap_bytes": 39735128, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 236.897, - "cpu_seconds": 694.5, - "rss_bytes": 1518051328, - "heap_bytes": 31404728, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 238.037, - "cpu_seconds": 701.87, - "rss_bytes": 2159759360, - "heap_bytes": 30017344, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 242.849, - "cpu_seconds": 708.14, - "rss_bytes": 1559597056, - "heap_bytes": 166755976, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 63, - "at_seconds": 245.893, - "cpu_seconds": 718.58, - "rss_bytes": 1348485120, - "heap_bytes": 36724712, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 250.9, - "cpu_seconds": 729.81, - "rss_bytes": 2638647296, - "heap_bytes": 55381936, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 19, - "at_seconds": 252.21, - "cpu_seconds": 748.12, - "rss_bytes": 3201306624, - "heap_bytes": 50870520, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 256.043, - "cpu_seconds": 752.67, - "rss_bytes": 2039754752, - "heap_bytes": 54909680, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 23, - "at_seconds": 257.177, - "cpu_seconds": 766.87, - "rss_bytes": 2472386560, - "heap_bytes": 32179976, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 24, - "at_seconds": 263.004, - "cpu_seconds": 774.03, - "rss_bytes": 2564640768, - "heap_bytes": 40456064, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 25, - "at_seconds": 265.019, - "cpu_seconds": 792.52, - "rss_bytes": 3562332160, - "heap_bytes": 27386880, - "queue_depth": 0, - "parquet_files": 1227, - "parquet_bytes": 307590170, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 18, - "at_seconds": 265.692, - "cpu_seconds": 799.66, - "rss_bytes": 1413971968, - "heap_bytes": 57489248, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 32, - "at_seconds": 268.703, - "cpu_seconds": 803.27, - "rss_bytes": 1213562880, - "heap_bytes": 54946272, - "queue_depth": 1, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 19, - "at_seconds": 270.304, - "cpu_seconds": 813.17, - "rss_bytes": 2406940672, - "heap_bytes": 22878352, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 23, - "at_seconds": 271.237, - "cpu_seconds": 818.9, - "rss_bytes": 2567802880, - "heap_bytes": 26971816, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 26, - "at_seconds": 272.81, - "cpu_seconds": 824.96, - "rss_bytes": 1256222720, - "heap_bytes": 34641008, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 26, - "at_seconds": 274.092, - "cpu_seconds": 832.27, - "rss_bytes": 1850556416, - "heap_bytes": 216637968, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 0, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 30, - "at_seconds": 281.006, - "cpu_seconds": 839.64, - "rss_bytes": 1860952064, - "heap_bytes": 28732104, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 15, - "at_seconds": 282.313, - "cpu_seconds": 858.04, - "rss_bytes": 1960357888, - "heap_bytes": 76102000, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 20, - "at_seconds": 282.639, - "cpu_seconds": 862.85, - "rss_bytes": 1770057728, - "heap_bytes": 40138848, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 21, - "at_seconds": 284.413, - "cpu_seconds": 869.02, - "rss_bytes": 2033672192, - "heap_bytes": 41213528, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 13, - "at_seconds": 286.989, - "cpu_seconds": 876.35, - "rss_bytes": 2119417856, - "heap_bytes": 31424568, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 24, - "at_seconds": 291.964, - "cpu_seconds": 883.7, - "rss_bytes": 1764507648, - "heap_bytes": 39184408, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 26, - "at_seconds": 292.277, - "cpu_seconds": 898.34, - "rss_bytes": 1899151360, - "heap_bytes": 41126928, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 14, - "at_seconds": 293.376, - "cpu_seconds": 899.51, - "rss_bytes": 1844273152, - "heap_bytes": 26290496, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 294.718, - "cpu_seconds": 906, - "rss_bytes": 2165788672, - "heap_bytes": 36159792, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - }, - { - "fanout_read_cache_pending_batches": 16, - "at_seconds": 296.886, - "cpu_seconds": 913.29, - "rss_bytes": 1690193920, - "heap_bytes": 36233368, - "queue_depth": 0, - "parquet_files": 4342, - "parquet_bytes": 434447968, - "publish_waiters": 1, - "publish_timeouts_total": 0 - } - ], - "minute_bins": [ - { - "minute": 1, - "sample_count": 29, - "pending_median": 7, - "pending_max": 40, - "pending_last": 10, - "preceding_failed_iteration_pending_median": 8, - "preceding_failed_iteration_pending_last": 9, - "server_handler_latency_ms": { - "count": 1199, - "p50": 32.865469, - "p95": 203.274115 - } - }, - { - "minute": 2, - "sample_count": 30, - "pending_median": 11.0, - "pending_max": 51, - "pending_last": 11, - "preceding_failed_iteration_pending_median": 17.5, - "preceding_failed_iteration_pending_last": 29, - "server_handler_latency_ms": { - "count": 1200, - "p50": 60.913534, - "p95": 398.813089 - } - }, - { - "minute": 3, - "sample_count": 30, - "pending_median": 12.0, - "pending_max": 18, - "pending_last": 18, - "preceding_failed_iteration_pending_median": 82.5, - "preceding_failed_iteration_pending_last": 293, - "server_handler_latency_ms": { - "count": 1175, - "p50": 84.076258, - "p95": 2285.376599 - } - }, - { - "minute": 4, - "sample_count": 30, - "pending_median": 16.0, - "pending_max": 29, - "pending_last": 14, - "preceding_failed_iteration_pending_median": 579, - "preceding_failed_iteration_pending_last": 871, - "server_handler_latency_ms": { - "count": 871, - "p50": 2192.894489, - "p95": 8187.412754 - } - }, - { - "minute": 5, - "sample_count": 24, - "pending_median": 20.5, - "pending_max": 63, - "pending_last": 16, - "preceding_failed_iteration_pending_median": 1238, - "preceding_failed_iteration_pending_last": 1690, - "server_handler_latency_ms": { - "count": 457, - "p50": 4448.361887, - "p95": 13594.371239 - } - } - ] - }, - "failed_preceding_iteration": { - "accepted_rows_per_second": 65583.94647883318, - "query_latency_ms": { - "count": 4379, - "mean_ms": 1224.34, - "p50_ms": 93.19, - "p95_ms": 7499.14, - "p99_ms": 13011.96, - "max_ms": 14837.75 - }, - "query_failures": 60, - "query_shed": 1570, - "resources": { - "average_cpu_cores": 3.1289983482854327, - "peak_rss_bytes": 3257733120, - "end_rss_bytes": 1892614144, - "peak_heap_bytes": 335704864, - "end_heap_bytes": 45933096, - "peak_queue_depth": 3, - "parquet_files_end": 4868, - "parquet_bytes_end": 411585608, - "disk_growth_bytes": 1147436045, - "allocated_bytes": 130723389744, - "gc_pause_seconds": 9.509500441 - }, - "quality_gates": [ - { - "name": "generator-processes", - "passed": true, - "detail": "all exited successfully" - }, - { - "name": "server-dropped-rows", - "passed": true, - "detail": "zero" - }, - { - "name": "server-restarts", - "passed": true, - "detail": "none" - }, - { - "name": "authoritative-counters", - "passed": true, - "detail": "Fanout accepted rows in every phase" - }, - { - "name": "sustained-query-load", - "passed": false, - "detail": "14.6/20.0 completed/s, 60 errors, 1570 shed" - }, - { - "name": "storage-repair-verify", - "passed": true, - "detail": "authoritative Parquet storage verified" - } - ], - "pending_samples": [ - { - "at_seconds": 2.029, - "pending_batches": 24 - }, - { - "at_seconds": 4.003, - "pending_batches": 6 - }, - { - "at_seconds": 6.003, - "pending_batches": 5 - }, - { - "at_seconds": 8.003, - "pending_batches": 8 - }, - { - "at_seconds": 10.003, - "pending_batches": 3 - }, - { - "at_seconds": 12.007, - "pending_batches": 6 - }, - { - "at_seconds": 14.004, - "pending_batches": 27 - }, - { - "at_seconds": 16.004, - "pending_batches": 6 - }, - { - "at_seconds": 18.008, - "pending_batches": 5 - }, - { - "at_seconds": 20.007, - "pending_batches": 7 - }, - { - "at_seconds": 22.012, - "pending_batches": 5 - }, - { - "at_seconds": 24.008, - "pending_batches": 6 - }, - { - "at_seconds": 26.004, - "pending_batches": 6 - }, - { - "at_seconds": 28.003, - "pending_batches": 21 - }, - { - "at_seconds": 30.014, - "pending_batches": 8 - }, - { - "at_seconds": 32.008, - "pending_batches": 9 - }, - { - "at_seconds": 34.006, - "pending_batches": 5 - }, - { - "at_seconds": 36.004, - "pending_batches": 5 - }, - { - "at_seconds": 38.007, - "pending_batches": 6 - }, - { - "at_seconds": 40.013, - "pending_batches": 8 - }, - { - "at_seconds": 42.141, - "pending_batches": 39 - }, - { - "at_seconds": 44.003, - "pending_batches": 13 - }, - { - "at_seconds": 46.02, - "pending_batches": 8 - }, - { - "at_seconds": 48.006, - "pending_batches": 6 - }, - { - "at_seconds": 50.021, - "pending_batches": 8 - }, - { - "at_seconds": 52.012, - "pending_batches": 8 - }, - { - "at_seconds": 54.009, - "pending_batches": 11 - }, - { - "at_seconds": 56.009, - "pending_batches": 10 - }, - { - "at_seconds": 58.021, - "pending_batches": 9 - }, - { - "at_seconds": 60.009, - "pending_batches": 9 - }, - { - "at_seconds": 62.071, - "pending_batches": 51 - }, - { - "at_seconds": 64.009, - "pending_batches": 20 - }, - { - "at_seconds": 66.12, - "pending_batches": 60 - }, - { - "at_seconds": 68.008, - "pending_batches": 35 - }, - { - "at_seconds": 70.004, - "pending_batches": 14 - }, - { - "at_seconds": 72.007, - "pending_batches": 9 - }, - { - "at_seconds": 74.007, - "pending_batches": 8 - }, - { - "at_seconds": 76.009, - "pending_batches": 8 - }, - { - "at_seconds": 78.268, - "pending_batches": 5 - }, - { - "at_seconds": 80.01, - "pending_batches": 21 - }, - { - "at_seconds": 82.011, - "pending_batches": 44 - }, - { - "at_seconds": 84.005, - "pending_batches": 55 - }, - { - "at_seconds": 86.05, - "pending_batches": 15 - }, - { - "at_seconds": 88.01, - "pending_batches": 44 - }, - { - "at_seconds": 90.004, - "pending_batches": 13 - }, - { - "at_seconds": 92.391, - "pending_batches": 11 - }, - { - "at_seconds": 94.008, - "pending_batches": 28 - }, - { - "at_seconds": 96.008, - "pending_batches": 11 - }, - { - "at_seconds": 98.007, - "pending_batches": 34 - }, - { - "at_seconds": 100.014, - "pending_batches": 15 - }, - { - "at_seconds": 102.005, - "pending_batches": 9 - }, - { - "at_seconds": 104.112, - "pending_batches": 12 - }, - { - "at_seconds": 106.01, - "pending_batches": 11 - }, - { - "at_seconds": 108.004, - "pending_batches": 34 - }, - { - "at_seconds": 110.033, - "pending_batches": 5 - }, - { - "at_seconds": 112.005, - "pending_batches": 28 - }, - { - "at_seconds": 114.014, - "pending_batches": 34 - }, - { - "at_seconds": 116.063, - "pending_batches": 41 - }, - { - "at_seconds": 118.008, - "pending_batches": 29 - }, - { - "at_seconds": 120.004, - "pending_batches": 8 - }, - { - "at_seconds": 122.009, - "pending_batches": 8 - }, - { - "at_seconds": 124.012, - "pending_batches": 52 - }, - { - "at_seconds": 126.021, - "pending_batches": 26 - }, - { - "at_seconds": 128.057, - "pending_batches": 46 - }, - { - "at_seconds": 130.033, - "pending_batches": 46 - }, - { - "at_seconds": 132.03, - "pending_batches": 76 - }, - { - "at_seconds": 134.194, - "pending_batches": 21 - }, - { - "at_seconds": 136.003, - "pending_batches": 17 - }, - { - "at_seconds": 138.034, - "pending_batches": 65 - }, - { - "at_seconds": 140.011, - "pending_batches": 29 - }, - { - "at_seconds": 142.211, - "pending_batches": 19 - }, - { - "at_seconds": 144.058, - "pending_batches": 77 - }, - { - "at_seconds": 146.054, - "pending_batches": 124 - }, - { - "at_seconds": 148.017, - "pending_batches": 119 - }, - { - "at_seconds": 150.008, - "pending_batches": 81 - }, - { - "at_seconds": 153.353, - "pending_batches": 41 - }, - { - "at_seconds": 154.395, - "pending_batches": 84 - }, - { - "at_seconds": 156.079, - "pending_batches": 108 - }, - { - "at_seconds": 158.025, - "pending_batches": 116 - }, - { - "at_seconds": 160.173, - "pending_batches": 146 - }, - { - "at_seconds": 162.229, - "pending_batches": 161 - }, - { - "at_seconds": 164.123, - "pending_batches": 165 - }, - { - "at_seconds": 166.199, - "pending_batches": 170 - }, - { - "at_seconds": 168.359, - "pending_batches": 187 - }, - { - "at_seconds": 172.13, - "pending_batches": 209 - }, - { - "at_seconds": 172.188, - "pending_batches": 232 - }, - { - "at_seconds": 174.884, - "pending_batches": 243 - }, - { - "at_seconds": 176.375, - "pending_batches": 280 - }, - { - "at_seconds": 178.745, - "pending_batches": 293 - }, - { - "at_seconds": 181.514, - "pending_batches": 304 - }, - { - "at_seconds": 182.068, - "pending_batches": 327 - }, - { - "at_seconds": 184.329, - "pending_batches": 335 - }, - { - "at_seconds": 187.82, - "pending_batches": 367 - }, - { - "at_seconds": 190.101, - "pending_batches": 429 - }, - { - "at_seconds": 192.444, - "pending_batches": 446 - }, - { - "at_seconds": 192.982, - "pending_batches": 458 - }, - { - "at_seconds": 195.862, - "pending_batches": 468 - }, - { - "at_seconds": 197.598, - "pending_batches": 487 - }, - { - "at_seconds": 198.167, - "pending_batches": 513 - }, - { - "at_seconds": 201.156, - "pending_batches": 528 - }, - { - "at_seconds": 202.604, - "pending_batches": 545 - }, - { - "at_seconds": 205.323, - "pending_batches": 559 - }, - { - "at_seconds": 206.224, - "pending_batches": 579 - }, - { - "at_seconds": 209.76, - "pending_batches": 592 - }, - { - "at_seconds": 210.519, - "pending_batches": 604 - }, - { - "at_seconds": 213.682, - "pending_batches": 614 - }, - { - "at_seconds": 214.735, - "pending_batches": 629 - }, - { - "at_seconds": 216.389, - "pending_batches": 649 - }, - { - "at_seconds": 218.004, - "pending_batches": 652 - }, - { - "at_seconds": 224.39, - "pending_batches": 674 - }, - { - "at_seconds": 225.114, - "pending_batches": 737 - }, - { - "at_seconds": 226.338, - "pending_batches": 767 - }, - { - "at_seconds": 230.795, - "pending_batches": 777 - }, - { - "at_seconds": 231.334, - "pending_batches": 796 - }, - { - "at_seconds": 237.009, - "pending_batches": 810 - }, - { - "at_seconds": 238.06, - "pending_batches": 871 - }, - { - "at_seconds": 240.378, - "pending_batches": 882 - }, - { - "at_seconds": 242.056, - "pending_batches": 923 - }, - { - "at_seconds": 242.699, - "pending_batches": 938 - }, - { - "at_seconds": 249.011, - "pending_batches": 954 - }, - { - "at_seconds": 251.927, - "pending_batches": 1015 - }, - { - "at_seconds": 252.488, - "pending_batches": 1021 - }, - { - "at_seconds": 252.491, - "pending_batches": 1021 - }, - { - "at_seconds": 254.981, - "pending_batches": 1113 - }, - { - "at_seconds": 256.752, - "pending_batches": 1113 - }, - { - "at_seconds": 258.862, - "pending_batches": 1189 - }, - { - "at_seconds": 262.796, - "pending_batches": 1222 - }, - { - "at_seconds": 264.448, - "pending_batches": 1238 - }, - { - "at_seconds": 266.518, - "pending_batches": 1249 - }, - { - "at_seconds": 267.1, - "pending_batches": 1273 - }, - { - "at_seconds": 272.203, - "pending_batches": 1296 - }, - { - "at_seconds": 273.276, - "pending_batches": 1337 - }, - { - "at_seconds": 275.051, - "pending_batches": 1353 - }, - { - "at_seconds": 277.964, - "pending_batches": 1364 - }, - { - "at_seconds": 283.009, - "pending_batches": 1411 - }, - { - "at_seconds": 288.023, - "pending_batches": 1470 - }, - { - "at_seconds": 288.211, - "pending_batches": 1488 - }, - { - "at_seconds": 295.004, - "pending_batches": 1488 - }, - { - "at_seconds": 295.719, - "pending_batches": 1690 - } - ] - }, - "file_binding_and_index_append": [ - { - "median_ms": 57.7043635, - "mode": "list", - "p95_ms": 62.433517, - "selected_files": 1000, - "statement_bytes": 139610, - "uses_glob": false, - "window_minutes": 1440 - }, - { - "median_ms": 57.633155, - "mode": "snapshot_glob", - "p95_ms": 61.936017, - "selected_files": 1000, - "statement_bytes": 42760, - "uses_glob": true, - "window_minutes": 1440 - }, - { - "median_ms": 44.735218, - "mode": "glob", - "p95_ms": 46.542491, - "selected_files": 1000, - "statement_bytes": 709, - "uses_glob": true, - "window_minutes": 1440 - }, - { - "median_ms": 2.3448575, - "mode": "list", - "p95_ms": 4.541507, - "selected_files": 11, - "statement_bytes": 2139, - "uses_glob": false, - "window_minutes": 10 - }, - { - "median_ms": 2.2485665, - "mode": "snapshot_glob", - "p95_ms": 3.776714, - "selected_files": 11, - "statement_bytes": 2139, - "uses_glob": false, - "window_minutes": 10 - }, - { - "median_ms": 39.954272, - "mode": "glob", - "p95_ms": 42.018818, - "selected_files": 11, - "statement_bytes": 709, - "uses_glob": true, - "window_minutes": 10 - }, - { - "added_spans_per_pass": 1000, - "median_ms": 16.84786, - "p95_ms": 20.537199, - "passes": 5, - "retained_spans": 200000, - "retained_trace_rows": 100000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "MATERIALIZED", - "median_ms": 4.912174, - "p95_ms": 5.888426, - "retained_spans": 200000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "NOT MATERIALIZED", - "median_ms": 4.831132, - "p95_ms": 5.054925, - "retained_spans": 200000 - }, - { - "added_spans_per_pass": 1000, - "median_ms": 18.57703, - "p95_ms": 19.652865, - "passes": 5, - "retained_spans": 800000, - "retained_trace_rows": 400000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "MATERIALIZED", - "median_ms": 7.178844, - "p95_ms": 7.889804, - "retained_spans": 800000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "NOT MATERIALIZED", - "median_ms": 5.167633, - "p95_ms": 5.546509, - "retained_spans": 800000 - }, - { - "added_spans_per_pass": 1000, - "median_ms": 17.550153, - "p95_ms": 24.316747, - "passes": 5, - "retained_spans": 2000000, - "retained_trace_rows": 1000000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "MATERIALIZED", - "median_ms": 13.037687, - "p95_ms": 13.397063, - "retained_spans": 2000000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "NOT MATERIALIZED", - "median_ms": 8.883723, - "p95_ms": 9.212849, - "retained_spans": 2000000 - }, - { - "added_spans_per_pass": 1000, - "median_ms": 21.73816, - "p95_ms": 28.380629, - "passes": 5, - "retained_spans": 8000000, - "retained_trace_rows": 4000000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "MATERIALIZED", - "median_ms": 43.961445, - "p95_ms": 45.477032, - "retained_spans": 8000000 - }, - { - "experiment": "trace_candidate_materialization", - "hint": "NOT MATERIALIZED", - "median_ms": 26.385876, - "p95_ms": 26.949585, - "retained_spans": 8000000 - } - ], - "completed_read_and_footprint": [ - { - "catalog_bytes": 70004736, - "empty_catalog_bytes": 274432, - "experiment": "cache_footprint", - "parquet_bytes": 52287828, - "rows_per_signal": 1200000 - }, - { - "cpu_seconds_per_request": 0.16473080000000007, - "implementation": "raw", - "median_ms": 46.623008999999996, - "method": "endpoints", - "p95_ms": 48.805784, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.016570699999999938, - "implementation": "completed", - "median_ms": 14.225793, - "method": "endpoints", - "p95_ms": 17.075152, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.17839930000000007, - "implementation": "raw", - "median_ms": 56.7078375, - "method": "trace", - "p95_ms": 59.966135, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.030159000000000092, - "implementation": "completed", - "median_ms": 17.4647145, - "method": "trace", - "p95_ms": 19.199613, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.17325420000000005, - "implementation": "raw", - "median_ms": 55.712982, - "method": "logs", - "p95_ms": 59.063341, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.021732800000000017, - "implementation": "completed", - "median_ms": 15.309127499999999, - "method": "logs", - "p95_ms": 17.624319, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.08889969999999998, - "implementation": "raw", - "median_ms": 33.495781, - "method": "logs", - "p95_ms": 36.856432, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.019686499999999895, - "implementation": "completed", - "median_ms": 14.137314, - "method": "logs", - "p95_ms": 16.056149, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.020265400000000256, - "implementation": "raw", - "median_ms": 11.529101, - "method": "endpoints", - "p95_ms": 13.528313, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.023323800000000006, - "implementation": "completed", - "median_ms": 17.488506, - "method": "endpoints", - "p95_ms": 18.32132, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.06347800000000028, - "implementation": "raw", - "median_ms": 26.030061, - "method": "trace", - "p95_ms": 27.04175, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.0344379, - "implementation": "completed", - "median_ms": 20.7213235, - "method": "trace", - "p95_ms": 22.162451, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - } - ], - "review_head_read_and_footprint": [ - { - "catalog_bytes": 180891648, - "empty_catalog_bytes": 274432, - "experiment": "cache_footprint", - "parquet_bytes": 52287828, - "rows_per_signal": 1200000 - }, - { - "cpu_seconds_per_request": 0.15753640000000005, - "implementation": "raw", - "median_ms": 44.659694, - "method": "endpoints", - "p95_ms": 45.798738, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.04933440000000004, - "implementation": "completed", - "median_ms": 29.9615255, - "method": "endpoints", - "p95_ms": 32.952843, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.17756730000000012, - "implementation": "raw", - "median_ms": 56.189087, - "method": "trace", - "p95_ms": 60.878094, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.034318199999999875, - "implementation": "completed", - "median_ms": 17.9254235, - "method": "trace", - "p95_ms": 18.634737, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.20452259999999994, - "implementation": "raw", - "median_ms": 67.964147, - "method": "logs", - "p95_ms": 69.523066, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.03202080000000009, - "implementation": "completed", - "median_ms": 21.5996165, - "method": "logs", - "p95_ms": 26.303999, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.025213700000000117, - "implementation": "raw", - "median_ms": 12.020227, - "method": "endpoints", - "p95_ms": 14.169939, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.03471489999999981, - "implementation": "completed", - "median_ms": 22.4875555, - "method": "endpoints", - "p95_ms": 27.704209, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.06506910000000019, - "implementation": "raw", - "median_ms": 27.121813, - "method": "trace", - "p95_ms": 29.288129, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.04784429999999986, - "implementation": "completed", - "median_ms": 27.674459499999998, - "method": "trace", - "p95_ms": 30.064255, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.08966289999999973, - "implementation": "raw", - "median_ms": 34.116885999999994, - "method": "logs", - "p95_ms": 35.555971, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.01770870000000002, - "implementation": "completed", - "median_ms": 12.898854, - "method": "logs", - "p95_ms": 14.038855, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - } - ], - "warm_read_regression_diagnostic": [ - { - "catalog_bytes": 70004736, - "empty_catalog_bytes": 274432, - "experiment": "cache_footprint", - "parquet_bytes": 52287828, - "rows_per_signal": 1200000 - }, - { - "cpu_seconds_per_request": 0.1653568, - "implementation": "raw", - "median_ms": 46.113405, - "method": "endpoints", - "p95_ms": 71.05161, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.01688679999999998, - "implementation": "completed", - "median_ms": 14.6557315, - "method": "endpoints", - "p95_ms": 15.928317, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.1798228, - "implementation": "raw", - "median_ms": 56.227941, - "method": "trace", - "p95_ms": 63.336931, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.19180889999999984, - "implementation": "completed", - "median_ms": 62.389514000000005, - "method": "trace", - "p95_ms": 65.095851, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.18129619999999988, - "implementation": "raw", - "median_ms": 57.898174, - "method": "logs", - "p95_ms": 64.41385, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.021251200000000026, - "implementation": "completed", - "median_ms": 14.7375235, - "method": "logs", - "p95_ms": 18.146528, - "rows_per_signal": 1200000, - "window_minutes": 1440.0166666666667 - }, - { - "cpu_seconds_per_request": 0.022954599999999915, - "implementation": "raw", - "median_ms": 11.9913725, - "method": "endpoints", - "p95_ms": 12.753978, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.022601900000000084, - "implementation": "completed", - "median_ms": 16.3775045, - "method": "endpoints", - "p95_ms": 18.251862, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.06422249999999999, - "implementation": "raw", - "median_ms": 27.0913545, - "method": "trace", - "p95_ms": 28.676086, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.10683830000000008, - "implementation": "completed", - "median_ms": 43.268254999999996, - "method": "trace", - "p95_ms": 47.001156, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.09288650000000018, - "implementation": "raw", - "median_ms": 35.0072835, - "method": "logs", - "p95_ms": 37.805225, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - }, - { - "cpu_seconds_per_request": 0.01983619999999995, - "implementation": "completed", - "median_ms": 14.198293, - "method": "logs", - "p95_ms": 16.45965, - "rows_per_signal": 1200000, - "window_minutes": 20.016666666666666 - } - ], - "limitations": [ - "Single trials and small per-case read samples do not establish production capacity.", - "Same-host generators are CPU-limited; offered is nominal, accepted counters are authoritative.", - "The sustained 20-read/s gate still fails despite stable cache backlog.", - "Peak RSS increased relative to the earlier sustained run; cache catalog reduction is not proof of lower peak process memory.", - "The snapshot-preserving broad glob remains slower than an unguarded glob.", - "CPU-limited writer regression remains open in #277." - ], - "cpu_limited_ingestion": { - "cpu_limit": 1, - "memory_limit_bytes": 6442450944, - "order": [ - "baseline1", - "final1", - "final2", - "baseline2", - "baseline3", - "final3" - ], - "baseline_commit": "30ba9b9ba8a4217cc73637760d7acdcd5f89f15a", - "writer_note": "Cache follow-up does not modify writer code; final is the earlier current-writer binary ingest-v3.test with measurement profiles.", - "all_six_storage_verifications_pass": true, - "baseline": { - "repetitions": [ - { - "allocated_bytes_per_row": 3756.246417666827, - "concurrency": 16, - "cpu_cores_client_and_server": 0.9974945835137421, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 35.203179, - "p95_ms": 64.048766, - "p99_ms": 70.4594, - "rows": 4166000, - "rows_per_second": 414699.6813000996, - "seconds": 10.045823973, - "cpu_seconds_per_million_rows": 2.4053420547287567 - }, - { - "allocated_bytes_per_row": 3737.617022498803, - "concurrency": 16, - "cpu_cores_client_and_server": 0.9954562546948289, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 34.901804, - "p95_ms": 64.566558, - "p99_ms": 76.955827, - "rows": 4178000, - "rows_per_second": 415984.279287794, - "seconds": 10.043648782, - "cpu_seconds_per_million_rows": 2.393014121589277 - }, - { - "allocated_bytes_per_row": 3745.67462644545, - "concurrency": 16, - "cpu_cores_client_and_server": 0.9975928171836536, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 35.680805, - "p95_ms": 65.671809, - "p99_ms": 73.680738, - "rows": 3978000, - "rows_per_second": 396682.0441252176, - "seconds": 10.028182669, - "cpu_seconds_per_million_rows": 2.514842383107089 - } - ], - "median": { - "allocated_bytes_per_row": 3745.67462644545, - "concurrency": 16, - "cpu_cores_client_and_server": 0.9974945835137421, - "errors": 0, - "p50_ms": 35.203179, - "p95_ms": 64.566558, - "p99_ms": 73.680738, - "rows": 4166000, - "rows_per_second": 414699.6813000996, - "seconds": 10.043648782, - "cpu_seconds_per_million_rows": 2.4053420547287567 - } - }, - "final": { - "repetitions": [ - { - "allocated_bytes_per_row": 3945.571900704873, - "concurrency": 16, - "cpu_cores_client_and_server": 1.0045131767164426, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 37.824641, - "p95_ms": 76.653076, - "p99_ms": 81.539876, - "rows": 3263000, - "rows_per_second": 325908.3559937839, - "seconds": 10.012016998, - "cpu_seconds_per_million_rows": 3.0821952191235065 - }, - { - "allocated_bytes_per_row": 3927.4635657568238, - "concurrency": 16, - "cpu_cores_client_and_server": 1.0003739569750103, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 45.957195, - "p95_ms": 79.843706, - "p99_ms": 87.479593, - "rows": 3224000, - "rows_per_second": 320543.3561450339, - "seconds": 10.05792177, - "cpu_seconds_per_million_rows": 3.1208694168734494 - }, - { - "allocated_bytes_per_row": 3943.2126353745725, - "concurrency": 16, - "cpu_cores_client_and_server": 1.0017059400170478, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 43.514858, - "p95_ms": 76.853409, - "p99_ms": 81.000792, - "rows": 3217000, - "rows_per_second": 321598.33610887046, - "seconds": 10.003161207, - "cpu_seconds_per_million_rows": 3.114773391358409 - } - ], - "median": { - "allocated_bytes_per_row": 3943.2126353745725, - "concurrency": 16, - "cpu_cores_client_and_server": 1.0017059400170478, - "errors": 0, - "p50_ms": 43.514858, - "p95_ms": 76.853409, - "p99_ms": 81.539876, - "rows": 3224000, - "rows_per_second": 321598.33610887046, - "seconds": 10.012016998, - "cpu_seconds_per_million_rows": 3.114773391358409 - } - } - } -} diff --git a/docs/benchmarks/duckdb2-review-2026-10-01.md b/docs/benchmarks/duckdb2-review-2026-10-01.md deleted file mode 100644 index c63a31f8b..000000000 --- a/docs/benchmarks/duckdb2-review-2026-10-01.md +++ /dev/null @@ -1,124 +0,0 @@ -# DuckDB 2 cache review — 2026-10-01 - -This report addresses the nine comments in [review 5383705777](https://github.com/labstack/fanout/pull/272#pullrequestreview-5383705777) at `cd916d5`. The [previous report](duckdb2-followups-2026-10-01.md) retains the admission-limited ingestion results and failed short saturation diagnostic. Neither fully built-cache reads nor short sweeps establish sustained cache stability. #277 remains open for encoding CPU cost. [#278](https://github.com/labstack/fanout/issues/278) tracks the remaining sustained read-capacity and peak-memory limits. - -## Implementation - -| Finding | Change | -|---|---| -| Cache worker lag | Separate completed-batch and analytical write gates, with two writer connections over disjoint tables; checkpointing holds both gates. Bind each signal once per pass, rather than one UNION branch per file. Derive candidates from newly aggregated trace parts without a third Parquet scan. | -| Retained trace-index rewrite | One candidate row per trace, incrementally upserted from new contributions; no global DELETE/reinsert on normal append. Temporary trace parts contain only the current pass. | -| Compaction markers disable the index | Transfer complete cached contributions before the namespace swap. Follow replacement lineage across multiple compaction levels. Retention and incomplete replacements repair only affected trace IDs. | -| Cache footprint | Remove full span-event and log-timestamp copies and the four scope-expanded trace indexes. Keep minute counts, scoped per-batch trace parts, and one candidate per trace. | -| Stale severity semantics | A schema/aggregation version atomically discards all private read tables on mismatch and rebuilds from format-3 Parquet. Matching versions preserve acknowledgements. | -| Broad file lists | Use a glob at 64+ selected files when at least 80% of active signal files overlap; retain a captured-ID filename filter. Narrow selections retain explicit lists. Only relations actually referenced are bound. | -| Ingestion throughput interpretation | Keep #277 open. The 20→10ms admission change helps closed-loop acknowledgement latency; it does not remove CPU cost per row. | -| Warm read comparison | Explicitly label cache-current reads and add the five-minute mixed run with pending-batch samples. | - -A worker pass admits at most 64 files and 64,000 new span/log rows, except that one oversized immutable file must be admitted to make progress. These are work-selection bounds, not a hard elapsed-time limit. Completed files and their aggregates/acknowledgements commit atomically. A pinned read sees active files and cache contributions from one transaction. Uncached format-3 files remain immediately visible through the authoritative Parquet path. Publication preparation releases the cache gate before waiting for readers to drain; prepared output markers are protected until publication finishes. No prior-format reader or schema conversion is introduced. - -Mixed namespace/service traces use exact scoped batch parts. Single-scope traces use the one candidate row; a snapshot-local ambiguity probe avoids adding an unused batch-parts aggregation to scoped plans. Clipped event-time windows retain exact boundary reads; a complete file contained entirely within the window can contribute its cached minute counts even when the window clips that minute. Trace candidate CTEs are not materialized into a full temporary relation for every request. Ranking still scans candidates; this is not a claim that DuckDB provides an ordered top-one index. - -## Environment and evidence - -Native Linux ARM64, Go 1.27.1, pinned DuckDB `v2.0.0-alpha43763`, Docker limits 4 CPUs / 6 GiB except the explicitly labelled one-CPU ingestion runs. Benchmarks run sequentially. Same-host telemetrygen 0.157.0 and the benchmark scheduler consume part of the same CPU allowance. The nominal offered rate is a configured target; accepted Fanout counters measure the actual ingestion rate. - -[Measured JSON](duckdb2-review-2026-10-01.json) records binary SHA256 values, samples, limits and failed iterations. Full local data/evidence are retained in `fanout-review272-cache-v6`, under `/runs/optimized`; the preceding failed gate remains in `fanout-review272-cache-v5`. These are development measurements of the worktree following `cd916d5`, not a production capacity guarantee. - -## Five-minute mixed workload - -The 300.154-second phase accepted **66,086 rows/s**, with Fanout using **3.05 CPU cores** on average. The scheduler completed **16.4/20 reads/s**, with **19 failed** and **1,099 shed** requests. Successful-client latency was **88.12ms p50 / 5,955.36ms p95 / 10,630.47ms p99**. **The sustained read-capacity gate fails.** Generator exits, dropped rows, restarts, and complete authoritative storage verification pass. - -| Minute | Pending batches median / sampled max / last | Preceding failed iteration, last pending | Server handler p50 / p95 | -|---|---:|---:|---:| -| 1 | 7 / 40 / 10 | 9 | 32.87 / 203.27ms | -| 2 | 11 / 51 / 11 | 29 | 60.91 / 398.81ms | -| 3 | 12 / 18 / 18 | 293 | 84.08 / 2285.38ms | -| 4 | 16 / 29 / 14 | 871 | 2192.89 / 8187.41ms | -| 5 | 20.5 / 63 / 16 | 1690 | 4448.36 / 13594.37ms | - -The cache stays near the incoming batch rate instead of accumulating 1,690 pending batches as the preceding iteration did. A manual cooldown scrape observed zero pending batches. Read latency nevertheless deteriorates late in the run: cache-current ranking/file scans, analytical work and publication scheduling still share finite resources. The gate failure is retained as a separate capacity limitation; cache backlog improvement does not establish that all overload problems are solved. The per-minute latency values above come from server request logs grouped by request start, not client latencies or completed-read rates. Requested two-second metric samples occasionally stall during overloaded scrapes; their actual timestamps and sample counts are retained. - -Peak process RSS is **3.32 GiB**, with **0.98 GiB** at the end of cooldown. The preceding failed iteration peaked at **3.03 GiB**. This does not show lower peak process memory, even though the derived catalog is smaller. Different accepted datasets, cache lifecycles, concurrent queries and compaction prevent treating process RSS as cache size. Full logs and failed gates remain inspectable. - -The later scoped ambiguity probe is measured separately by the fully built-cache fixture below. It does not change the unscoped sustained workload's query path; the JSON distinguishes the two measured binaries. - - -The fixed preset has a ten-second warmup, six generator processes (two per signal, eight workers each, 2,500 rows/s per worker), a full 300-second mixed phase, 1,000 rows/export, 20 open-loop reads/s, and a thirty-second cooldown. The unchanged reader scheduler uses 40 workers and a 40-item queue. The pending-batch gauge is sampled every two seconds. The review's shell driver used a different scheduling/in-flight limit, so the runs are not exact matched throughput comparisons. - -## Retained-index append and read experiment - -Five disjoint 1,000-span files are appended at each retained size, with one trace per two spans. Fixture creation is excluded. These synthetic caches exercise normal append only: no stale cleanup, compaction, publication wait, or concurrent workload. The pass must still bind/aggregate its new file and mutate the keyed index; constant-time behavior is not claimed. - -| Retained spans / trace rows | Append median / max | Materialized / non-materialized candidate read median | -|---|---:|---:| -| 200k / 100k | 16.85 / 20.54ms | 4.91 / 4.83ms | -| 800k / 400k | 18.58 / 19.65ms | 7.18 / 5.17ms | -| 2M / 1M | 17.55 / 24.32ms | 13.04 / 8.88ms | -| 8M / 4M | 21.74 / 28.38ms | 43.96 / 26.39ms | - -The synthetic read measures the top-one and touched-trace branches over the same candidate table. It does not include file capture, scoped ambiguity, dashboard serialization or concurrent writes. EXPLAIN shows no retained-candidate CTE materialization in the non-materialized form; the table scans remain. Maxima of five samples are not robust tail estimates. - -## File binding - -One thousand one-row log files span one thousand minutes. Each case warms once and measures ten requests. Counts agree for the captured data. - -| Window | Explicit list median | Captured-ID glob median | Unguarded glob median | Selected files | -|---|---:|---:|---:|---:| -| 24h | 57.70ms | 57.63ms | 44.74ms | 1,000 | -| 10m | 2.34ms | 2.25ms (explicit list selected) | 39.95ms | 11 | - -The broad statement shrinks from 139,610 to 42,760 bytes. The exact captured-ID glob is effectively flat with the explicit list in this fixture and still slower than an unguarded glob. An unguarded glob can admit new files published after snapshot capture; it is not substituted for the snapshot contract. A regression test publishes a new batch after capture and verifies exclusion, including a Unicode data path. Rollup-only queries no longer construct unused Parquet lists. - -## Cache-current dashboard reads and footprint - -| Window / method | Raw median / p95 | Completed median / p95 | Completed CPU ms / request | -|---|---:|---:|---:| -| 24h endpoints | 46.62 / 48.81ms | 14.23 / 17.08ms | 16.57 | -| 24h trace | 56.71 / 59.97ms | 17.46 / 19.20ms | 30.16 | -| 24h logs | 55.71 / 59.06ms | 15.31 / 17.62ms | 21.73 | -| 20m endpoints | 11.53 / 13.53ms | 17.49 / 18.32ms | 23.32 | -| 20m trace | 26.03 / 27.04ms | 20.72 / 22.16ms | 34.44 | -| 20m logs | 33.50 / 36.86ms | 14.14 / 16.06ms | 19.69 | - -Using the same fixture and footprint instrumentation, the review head's catalog is **180,891,648 bytes** versus **70,004,736 bytes** now (**61.3% smaller**). Parquet is **52,287,828 bytes** for both, and the empty catalog is **274,432 bytes**. The new catalog is still **1.34×** the highly compressible Parquet fixture; caches are not free. Backfill changes from **3.355s to 1.856s**. This is one trial, not a universal storage ratio. - -The persisted-cache experiment caught a transient scoped-trace regression (**62.39ms** broad median): an unnecessary mixed-scope batch-parts branch was duplicated by the non-materialized plan even when no ambiguity existed. The snapshot-local ambiguity probe removes that branch in those scopes; the final broad trace median is **17.46ms**, versus **17.93ms** at the review head. The regression diagnostic is retained in JSON. Mixed scopes still use exact active parts. - - -The fixture contains 1.2 million spans and 1.2 million logs in twelve hourly batches plus 300 old tiny log/metric files. Both paths read identical format-3 data; caches are completely built, no background worker or ingest runs. Ten requests follow one warmup per case. Endpoint production reads use the established histogram estimate rather than the raw kernel's approx_quantile. Narrow endpoint planning cost remains an explicit limitation. Catalog sizes are measured after CHECKPOINT and include the empty cache/catalog baseline; they are not peak memory measurements. The sustained run supplies process RSS. - -## CPU-limited ingestion - -| Median metric | #270 | Current typed writer | -|---|---:|---:| -| Durable acknowledged rows/s | 414,700 | 321,598 | -| CPU seconds / million rows | 2.405 | 3.115 | -| Go allocated bytes / row | 3,746 | 3,943 | -| Export p50 (ms) | 35.20 | 43.51 | -| Export p95 (ms) | 64.57 | 76.85 | - -All six runs have zero export errors and pass storage verification. With CPU saturated at approximately one core, current throughput is **22.5% lower**, normalized CPU cost **29.5% higher**, and Go allocation/row **5.3% higher**. This independently confirms the review's direction. **#277 remains open; the encoding regression is not resolved.** The earlier four-CPU +19% throughput result remains labelled admission-limited in the historical report. - - -The baseline is #270 merge `30ba9b9ba8a4217cc73637760d7acdcd5f89f15a`, with a measurement-only profiling overlay. The current writer is unchanged by this cache follow-up. Three alternating baseline/current repetitions use one CPU / 6 GiB, durable mode, 16 in-process clients, equal signals, 1,000 rows/export, and ten-second measurement intervals. No dashboard/cache worker runs. CPU profiles cover the interval; heap profiles cover the whole experiment. Every run verifies authoritative storage. These single-host fixture medians do not prove broader capacity. - -## Reproduce and validation - -Run opt-in fixtures alone with the pinned wrapper: - -```sh -bash scripts/with-duckdb.sh go test -tags=filebench ./internal/query \ - -run 'TestSnapshotFileBindingBenchmark|TestReadCacheAppendScalingBenchmark' -v -count=1 -FANOUT_READ_BENCH_LARGE=1 bash scripts/with-duckdb.sh go test -tags=readbench \ - ./internal/observability -run TestCompletedReadBenchmark -v -count=1 -FANOUT_BENCH_PROFILE_DIR=/tmp/fanout-profiles \ - bash scripts/with-duckdb.sh go test -tags=transportbench ./internal/ingest \ - -run 'TestTransportComparison/durable=true/c=16/rep=1/shared' -v -count=1 -``` - -For CPU-capped measurements, compile native Linux test binaries with `go test -c` through the wrapper, then execute them in otherwise idle `--cpus 4 --memory 6g` containers (one CPU for the CPU-limited ingestion comparison). - -For the sustained test, copy `fanout-bench` at `a9843c1fd55b9465d4d77a42fcf8805c467af731` into a temporary workspace and apply [the complete local harness patch](duckdb2-review-2026-10-01-harness.patch) with `git apply --unidiff-zero`. The patch adapts current Fanout addresses/metrics, adds source identity, installs the fixed preset, and records the gauge. Build the harness and Fanout for native Linux, then run `fanout-bench managed --fanout /tools/fanout --telemetrygen /tools/telemetrygen --preset review272 --query-rate 20 --sample-interval 2s --cooldown 30s --output /runs/optimized/evidence` in a 4-CPU/6-GiB container. Sibling repositories are not modified. - -Local validation passes: `just check`, the full `just test-race` suite with the documented CGO checkptr workaround, and the full native Linux ARM64 Go suite. New regression coverage exercises schema/semantic rebuild and matching-version preservation, original severity labels, captured-glob isolation with Unicode and relative paths, protected publication preparation and cancellation, cache progress during a disjoint analytical transaction, row-budget/oversized-file progress, two-level compaction lineage, retention scope/error repair, rollback and public-SQL cache rejection. Existing cold/warm, clipped-window, late-publication, trace-ranking and typed-data parity tests continue to pass. diff --git a/docs/benchmarks/single-port-2026-09-08.json b/docs/benchmarks/single-port-2026-09-08.json deleted file mode 100644 index 62bcfa505..000000000 --- a/docs/benchmarks/single-port-2026-09-08.json +++ /dev/null @@ -1,338 +0,0 @@ -[ - { - "allocated_bytes_per_row": 2065.429636767318, - "concurrency": 1, - "cpu_cores_client_and_server": 1.668108766224098, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 0.8185, - "p95_ms": 2.228833, - "p99_ms": 2.348875, - "rows": 9008000, - "rows_per_second": 900673.042659051, - "seconds": 10.001409583 - }, - { - "allocated_bytes_per_row": 2235.167287165282, - "concurrency": 1, - "cpu_cores_client_and_server": 1.7853236068469907, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 0.942958, - "p95_ms": 2.266666, - "p99_ms": 2.417417, - "rows": 8664000, - "rows_per_second": 866387.191851588, - "seconds": 10.000147834 - }, - { - "allocated_bytes_per_row": 2244.0646240875913, - "concurrency": 1, - "cpu_cores_client_and_server": 1.785360650700898, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 0.93325, - "p95_ms": 2.236125, - "p99_ms": 2.372625, - "rows": 8768000, - "rows_per_second": 876795.1629841244, - "seconds": 10.000055167 - }, - { - "allocated_bytes_per_row": 2066.4755709403594, - "concurrency": 1, - "cpu_cores_client_and_server": 1.6860995332039543, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 0.815, - "p95_ms": 2.21025, - "p99_ms": 2.328125, - "rows": 9071000, - "rows_per_second": 906928.1635046254, - "seconds": 10.001894709 - }, - { - "allocated_bytes_per_row": 2058.800469744509, - "concurrency": 1, - "cpu_cores_client_and_server": 1.692699146091521, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 0.841209, - "p95_ms": 2.23975, - "p99_ms": 2.377375, - "rows": 8924000, - "rows_per_second": 892354.6869428807, - "seconds": 10.000507792 - }, - { - "allocated_bytes_per_row": 2231.728527462008, - "concurrency": 1, - "cpu_cores_client_and_server": 1.773189609549387, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 0.986333, - "p95_ms": 2.330292, - "p99_ms": 2.48125, - "rows": 8357000, - "rows_per_second": 835663.8575381615, - "seconds": 10.0004325 - }, - { - "allocated_bytes_per_row": 1961.8700916580628, - "concurrency": 16, - "cpu_cores_client_and_server": 9.112946253819041, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 2.9295, - "p95_ms": 6.656667, - "p99_ms": 10.422292, - "rows": 47459000, - "rows_per_second": 4744631.522762389, - "seconds": 10.0026735 - }, - { - "allocated_bytes_per_row": 1983.2758534517795, - "concurrency": 16, - "cpu_cores_client_and_server": 10.30437775355329, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 2.88375, - "p95_ms": 6.07375, - "p99_ms": 7.402625, - "rows": 51669000, - "rows_per_second": 5165202.800618604, - "seconds": 10.003285833 - }, - { - "allocated_bytes_per_row": 1981.7025076054413, - "concurrency": 16, - "cpu_cores_client_and_server": 9.900270964204253, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 2.990417, - "p95_ms": 6.380041, - "p99_ms": 7.895125, - "rows": 49767000, - "rows_per_second": 4975016.06054384, - "seconds": 10.003384792 - }, - { - "allocated_bytes_per_row": 1954.849820058997, - "concurrency": 16, - "cpu_cores_client_and_server": 8.843126095230799, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 3.130334, - "p95_ms": 7.114459, - "p99_ms": 9.972708, - "rows": 46104000, - "rows_per_second": 4609332.824217873, - "seconds": 10.00231525 - }, - { - "allocated_bytes_per_row": 1961.0370257659515, - "concurrency": 16, - "cpu_cores_client_and_server": 10.23542983013505, - "durable": false, - "errors": 0, - "mode": "native", - "p50_ms": 2.757583, - "p95_ms": 6.046208, - "p99_ms": 7.41125, - "rows": 53365000, - "rows_per_second": 5335229.726618022, - "seconds": 10.002380916 - }, - { - "allocated_bytes_per_row": 1981.4924883456838, - "concurrency": 16, - "cpu_cores_client_and_server": 10.292444172154767, - "durable": false, - "errors": 0, - "mode": "shared", - "p50_ms": 2.8625, - "p95_ms": 6.068416, - "p99_ms": 7.409542, - "rows": 51955000, - "rows_per_second": 5194012.650847724, - "seconds": 10.002863584 - }, - { - "allocated_bytes_per_row": 3932.1484444444445, - "concurrency": 1, - "cpu_cores_client_and_server": 0.1526923273839045, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 38.967166, - "p95_ms": 45.813792, - "p99_ms": 49.388542, - "rows": 252000, - "rows_per_second": 25170.777251163567, - "seconds": 10.011609792 - }, - { - "allocated_bytes_per_row": 3967.582976, - "concurrency": 1, - "cpu_cores_client_and_server": 0.16019880419809268, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 39.060458, - "p95_ms": 45.031083, - "p99_ms": 48.052708, - "rows": 250000, - "rows_per_second": 24952.665001599085, - "seconds": 10.018969917 - }, - { - "allocated_bytes_per_row": 3934.2855843137254, - "concurrency": 1, - "cpu_cores_client_and_server": 0.16431193790352888, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 38.887167, - "p95_ms": 44.072084, - "p99_ms": 47.978208, - "rows": 255000, - "rows_per_second": 25450.023364104516, - "seconds": 10.019637167 - }, - { - "allocated_bytes_per_row": 3954.68196875, - "concurrency": 1, - "cpu_cores_client_and_server": 0.15213216881861105, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 38.871625, - "p95_ms": 43.972292, - "p99_ms": 46.758167, - "rows": 256000, - "rows_per_second": 25572.04123321355, - "seconds": 10.010933334 - }, - { - "allocated_bytes_per_row": 3962.5928971962617, - "concurrency": 1, - "cpu_cores_client_and_server": 0.10835271108696926, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 43.871792, - "p95_ms": 62.9245, - "p99_ms": 69.515125, - "rows": 214000, - "rows_per_second": 21316.058162309007, - "seconds": 10.039379625 - }, - { - "allocated_bytes_per_row": 3965.29671875, - "concurrency": 1, - "cpu_cores_client_and_server": 0.15934341461007848, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 38.152041, - "p95_ms": 44.984292, - "p99_ms": 47.031625, - "rows": 256000, - "rows_per_second": 25585.791889941116, - "seconds": 10.005553125 - }, - { - "allocated_bytes_per_row": 4384.688459183673, - "concurrency": 16, - "cpu_cores_client_and_server": 1.0979446755771467, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 51.31025, - "p95_ms": 58.145042, - "p99_ms": 61.531208, - "rows": 3136000, - "rows_per_second": 312193.6858291852, - "seconds": 10.045046208 - }, - { - "allocated_bytes_per_row": 4435.00769402985, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2326325489358088, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 49.280917, - "p95_ms": 56.741875, - "p99_ms": 58.569708, - "rows": 3216000, - "rows_per_second": 320595.9336104238, - "seconds": 10.03131875 - }, - { - "allocated_bytes_per_row": 4437.08422162324, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2485428225271538, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 47.988375, - "p95_ms": 54.267709, - "p99_ms": 57.947375, - "rows": 3339000, - "rows_per_second": 332345.6347210742, - "seconds": 10.046769541 - }, - { - "allocated_bytes_per_row": 4399.0799876237625, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2168461407210889, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 49.54425, - "p95_ms": 56.040916, - "p99_ms": 60.273375, - "rows": 3232000, - "rows_per_second": 323135.08351888845, - "seconds": 10.002008958 - }, - { - "allocated_bytes_per_row": 4394.639358536585, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2036325654104647, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 48.741959, - "p95_ms": 56.026459, - "p99_ms": 59.231583, - "rows": 3280000, - "rows_per_second": 326997.2370743304, - "seconds": 10.030665792 - }, - { - "allocated_bytes_per_row": 4453.083403381643, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2782997399849019, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 47.938416, - "p95_ms": 54.979458, - "p99_ms": 58.008667, - "rows": 3312000, - "rows_per_second": 330111.723454227, - "seconds": 10.032966916 - } -] diff --git a/docs/benchmarks/single-port-2026-09-08.log b/docs/benchmarks/single-port-2026-09-08.log deleted file mode 100644 index 76ad6c238..000000000 --- a/docs/benchmarks/single-port-2026-09-08.log +++ /dev/null @@ -1,101 +0,0 @@ -=== RUN TestTransportComparison - transport_bench_test.go:38: environment go=go1.27.0 os=darwin arch=arm64 CPUs=14 GOMAXPROCS=14 -=== RUN TestTransportComparison/durable=false/c=1/rep=1/native -2026/09/08 09:22:56 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2065.429636767318,"concurrency":1,"cpu_cores_client_and_server":1.668108766224098,"durable":false,"errors":0,"mode":"native","p50_ms":0.8185,"p95_ms":2.228833,"p99_ms":2.348875,"rows":9008000,"rows_per_second":900673.042659051,"seconds":10.001409583} -=== RUN TestTransportComparison/durable=false/c=1/rep=1/shared -2026/09/08 09:23:06 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2235.167287165282,"concurrency":1,"cpu_cores_client_and_server":1.7853236068469907,"durable":false,"errors":0,"mode":"shared","p50_ms":0.942958,"p95_ms":2.266666,"p99_ms":2.417417,"rows":8664000,"rows_per_second":866387.191851588,"seconds":10.000147834} -=== RUN TestTransportComparison/durable=false/c=1/rep=2/shared -2026/09/08 09:23:16 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2244.0646240875913,"concurrency":1,"cpu_cores_client_and_server":1.785360650700898,"durable":false,"errors":0,"mode":"shared","p50_ms":0.93325,"p95_ms":2.236125,"p99_ms":2.372625,"rows":8768000,"rows_per_second":876795.1629841244,"seconds":10.000055167} -=== RUN TestTransportComparison/durable=false/c=1/rep=2/native -2026/09/08 09:23:26 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2066.4755709403594,"concurrency":1,"cpu_cores_client_and_server":1.6860995332039543,"durable":false,"errors":0,"mode":"native","p50_ms":0.815,"p95_ms":2.21025,"p99_ms":2.328125,"rows":9071000,"rows_per_second":906928.1635046254,"seconds":10.001894709} -=== RUN TestTransportComparison/durable=false/c=1/rep=3/native -2026/09/08 09:23:36 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2058.800469744509,"concurrency":1,"cpu_cores_client_and_server":1.692699146091521,"durable":false,"errors":0,"mode":"native","p50_ms":0.841209,"p95_ms":2.23975,"p99_ms":2.377375,"rows":8924000,"rows_per_second":892354.6869428807,"seconds":10.000507792} -=== RUN TestTransportComparison/durable=false/c=1/rep=3/shared -2026/09/08 09:23:46 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":2231.728527462008,"concurrency":1,"cpu_cores_client_and_server":1.773189609549387,"durable":false,"errors":0,"mode":"shared","p50_ms":0.986333,"p95_ms":2.330292,"p99_ms":2.48125,"rows":8357000,"rows_per_second":835663.8575381615,"seconds":10.0004325} -=== RUN TestTransportComparison/durable=false/c=16/rep=1/native -2026/09/08 09:23:56 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1961.8700916580628,"concurrency":16,"cpu_cores_client_and_server":9.112946253819041,"durable":false,"errors":0,"mode":"native","p50_ms":2.9295,"p95_ms":6.656667,"p99_ms":10.422292,"rows":47459000,"rows_per_second":4744631.522762389,"seconds":10.0026735} -=== RUN TestTransportComparison/durable=false/c=16/rep=1/shared -2026/09/08 09:24:06 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1983.2758534517795,"concurrency":16,"cpu_cores_client_and_server":10.30437775355329,"durable":false,"errors":0,"mode":"shared","p50_ms":2.88375,"p95_ms":6.07375,"p99_ms":7.402625,"rows":51669000,"rows_per_second":5165202.800618604,"seconds":10.003285833} -=== RUN TestTransportComparison/durable=false/c=16/rep=2/shared -2026/09/08 09:24:16 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1981.7025076054413,"concurrency":16,"cpu_cores_client_and_server":9.900270964204253,"durable":false,"errors":0,"mode":"shared","p50_ms":2.990417,"p95_ms":6.380041,"p99_ms":7.895125,"rows":49767000,"rows_per_second":4975016.06054384,"seconds":10.003384792} -=== RUN TestTransportComparison/durable=false/c=16/rep=2/native -2026/09/08 09:24:27 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1954.849820058997,"concurrency":16,"cpu_cores_client_and_server":8.843126095230799,"durable":false,"errors":0,"mode":"native","p50_ms":3.130334,"p95_ms":7.114459,"p99_ms":9.972708,"rows":46104000,"rows_per_second":4609332.824217873,"seconds":10.00231525} -=== RUN TestTransportComparison/durable=false/c=16/rep=3/native -2026/09/08 09:24:37 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1961.0370257659515,"concurrency":16,"cpu_cores_client_and_server":10.23542983013505,"durable":false,"errors":0,"mode":"native","p50_ms":2.757583,"p95_ms":6.046208,"p99_ms":7.41125,"rows":53365000,"rows_per_second":5335229.726618022,"seconds":10.002380916} -=== RUN TestTransportComparison/durable=false/c=16/rep=3/shared -2026/09/08 09:24:47 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":1981.4924883456838,"concurrency":16,"cpu_cores_client_and_server":10.292444172154767,"durable":false,"errors":0,"mode":"shared","p50_ms":2.8625,"p95_ms":6.068416,"p99_ms":7.409542,"rows":51955000,"rows_per_second":5194012.650847724,"seconds":10.002863584} -=== RUN TestTransportComparison/durable=true/c=1/rep=1/native -2026/09/08 09:24:57 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3932.1484444444445,"concurrency":1,"cpu_cores_client_and_server":0.1526923273839045,"durable":true,"errors":0,"mode":"native","p50_ms":38.967166,"p95_ms":45.813792,"p99_ms":49.388542,"rows":252000,"rows_per_second":25170.777251163567,"seconds":10.011609792} -=== RUN TestTransportComparison/durable=true/c=1/rep=1/shared -2026/09/08 09:25:07 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3967.582976,"concurrency":1,"cpu_cores_client_and_server":0.16019880419809268,"durable":true,"errors":0,"mode":"shared","p50_ms":39.060458,"p95_ms":45.031083,"p99_ms":48.052708,"rows":250000,"rows_per_second":24952.665001599085,"seconds":10.018969917} -=== RUN TestTransportComparison/durable=true/c=1/rep=2/shared -2026/09/08 09:25:18 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3934.2855843137254,"concurrency":1,"cpu_cores_client_and_server":0.16431193790352888,"durable":true,"errors":0,"mode":"shared","p50_ms":38.887167,"p95_ms":44.072084,"p99_ms":47.978208,"rows":255000,"rows_per_second":25450.023364104516,"seconds":10.019637167} -=== RUN TestTransportComparison/durable=true/c=1/rep=2/native -2026/09/08 09:25:28 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3954.68196875,"concurrency":1,"cpu_cores_client_and_server":0.15213216881861105,"durable":true,"errors":0,"mode":"native","p50_ms":38.871625,"p95_ms":43.972292,"p99_ms":46.758167,"rows":256000,"rows_per_second":25572.04123321355,"seconds":10.010933334} -=== RUN TestTransportComparison/durable=true/c=1/rep=3/native -2026/09/08 09:25:39 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3962.5928971962617,"concurrency":1,"cpu_cores_client_and_server":0.10835271108696926,"durable":true,"errors":0,"mode":"native","p50_ms":43.871792,"p95_ms":62.9245,"p99_ms":69.515125,"rows":214000,"rows_per_second":21316.058162309007,"seconds":10.039379625} -=== RUN TestTransportComparison/durable=true/c=1/rep=3/shared -2026/09/08 09:25:49 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":3965.29671875,"concurrency":1,"cpu_cores_client_and_server":0.15934341461007848,"durable":true,"errors":0,"mode":"shared","p50_ms":38.152041,"p95_ms":44.984292,"p99_ms":47.031625,"rows":256000,"rows_per_second":25585.791889941116,"seconds":10.005553125} -=== RUN TestTransportComparison/durable=true/c=16/rep=1/native -2026/09/08 09:26:00 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4384.688459183673,"concurrency":16,"cpu_cores_client_and_server":1.0979446755771467,"durable":true,"errors":0,"mode":"native","p50_ms":51.31025,"p95_ms":58.145042,"p99_ms":61.531208,"rows":3136000,"rows_per_second":312193.6858291852,"seconds":10.045046208} -=== RUN TestTransportComparison/durable=true/c=16/rep=1/shared -2026/09/08 09:26:15 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4435.00769402985,"concurrency":16,"cpu_cores_client_and_server":1.2326325489358088,"durable":true,"errors":0,"mode":"shared","p50_ms":49.280917,"p95_ms":56.741875,"p99_ms":58.569708,"rows":3216000,"rows_per_second":320595.9336104238,"seconds":10.03131875} -=== RUN TestTransportComparison/durable=true/c=16/rep=2/shared -2026/09/08 09:26:29 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4437.08422162324,"concurrency":16,"cpu_cores_client_and_server":1.2485428225271538,"durable":true,"errors":0,"mode":"shared","p50_ms":47.988375,"p95_ms":54.267709,"p99_ms":57.947375,"rows":3339000,"rows_per_second":332345.6347210742,"seconds":10.046769541} -=== RUN TestTransportComparison/durable=true/c=16/rep=2/native -2026/09/08 09:26:44 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4399.0799876237625,"concurrency":16,"cpu_cores_client_and_server":1.2168461407210889,"durable":true,"errors":0,"mode":"native","p50_ms":49.54425,"p95_ms":56.040916,"p99_ms":60.273375,"rows":3232000,"rows_per_second":323135.08351888845,"seconds":10.002008958} -=== RUN TestTransportComparison/durable=true/c=16/rep=3/native -2026/09/08 09:26:59 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4394.639358536585,"concurrency":16,"cpu_cores_client_and_server":1.2036325654104647,"durable":true,"errors":0,"mode":"native","p50_ms":48.741959,"p95_ms":56.026459,"p99_ms":59.231583,"rows":3280000,"rows_per_second":326997.2370743304,"seconds":10.030665792} -=== RUN TestTransportComparison/durable=true/c=16/rep=3/shared -2026/09/08 09:27:14 INFO migrations applied - transport_bench_test.go:230: RESULT {"allocated_bytes_per_row":4453.083403381643,"concurrency":16,"cpu_cores_client_and_server":1.2782997399849019,"durable":true,"errors":0,"mode":"shared","p50_ms":47.938416,"p95_ms":54.979458,"p99_ms":58.008667,"rows":3312000,"rows_per_second":330111.723454227,"seconds":10.032966916} ---- PASS: TestTransportComparison (272.63s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=1/native (10.02s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=1/shared (10.01s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=2/shared (10.01s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=2/native (10.01s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=3/native (10.01s) - --- PASS: TestTransportComparison/durable=false/c=1/rep=3/shared (10.01s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=1/native (10.08s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=1/shared (10.08s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=2/shared (10.07s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=2/native (10.07s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=3/native (10.07s) - --- PASS: TestTransportComparison/durable=false/c=16/rep=3/shared (10.07s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=1/native (10.50s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=1/shared (10.51s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=2/shared (10.50s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=2/native (10.51s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=3/native (10.57s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=3/shared (10.53s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=1/native (14.75s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=1/shared (14.69s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=2/shared (15.12s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=2/native (14.69s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=3/native (15.02s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=3/shared (14.72s) -PASS -ok github.com/labstack/fanout/internal/ingest 273.072s diff --git a/docs/benchmarks/single-port-2026-09-08.md b/docs/benchmarks/single-port-2026-09-08.md deleted file mode 100644 index f653baada..000000000 --- a/docs/benchmarks/single-port-2026-09-08.md +++ /dev/null @@ -1,87 +0,0 @@ -# Single-port gRPC comparison — 2026-09-08 - -## Final implementation: Go's built-in HTTP/2 server - -The implementation uses `internal/server.New` with HTTP/1, HTTP/2 over TLS and unencrypted HTTP/2 enabled through `http.Protocols`. It needs no h2c wrapper. The benchmark harness now calls this production factory directly. - -All twelve durable-write cases passed: three repetitions each for native and shared gRPC at one and sixteen concurrent exporters. All acknowledgements matched the server row counts, and all Parquet repositories passed verification. The workload, machine and measurement method match the prototype experiment below. - -| Exporters | Native rows/s | Shared rows/s | Change | Native p95 | Shared p95 | Native p99 | Shared p99 | -|---|---:|---:|---:|---:|---:|---:|---:| -| 1 | 24,526 | 25,028 | +2.0% | 48.83 ms | 44.98 ms | 50.18 ms | 48.19 ms | -| 16 | 325,896 | 326,300 | +0.1% | 55.38 ms | 54.53 ms | 59.41 ms | 58.53 ms | - -Throughput and tail latency showed no regression in these local durable workloads. Treat the small throughput differences as run variation. At sixteen exporters, combined client/server CPU rose from 1.177 to 1.327 cores (12.8%); allocated bytes per accepted row rose from 4,409 to 4,442 (0.8%). The transport is not cost-free. These measurements do not isolate server CPU and do not certify production capacity or performance under concurrent dashboard traffic. - -Raw final evidence: [test log](single-port-stdlib-2026-09-08.log), [JSON measurements](single-port-stdlib-2026-09-08.json). - -Reproduce the final durable comparison: - -```sh -just test '-tags transportbench ./internal/ingest -run ^TestTransportComparison$/durable=true -v -count=1 -timeout=10m' -``` - -Functional validation also passed: full Go suite, 122 UI tests, 13 focused shared-listener tests under the race detector, lint, dependency notices, release-script tests, UI dependency audits, generated docs, diagram consistency and the documentation build. Those tests cover HTTP and gRPC on one port, plaintext and TLS, credential separation, invalid ingest routes, export draining and forced cancellation of long-lived HTTP/2 streams. - -## Original prototype experiment - -The following measurements are retained as historical evidence for the earlier `h2c.NewHandler` prototype. The current harness uses the final factory above, so rerunning it exercises the final implementation rather than reproducing that wrapper verbatim. - - -The local experiment found no durable-ingest throughput or tail-latency regression from using gRPC ServeHTTP behind the proposed h2c dispatcher. Median durable throughput was 1.1% higher with one exporter and 2.2% higher with sixteen; these small differences should be treated as run variation, not a demonstrated speedup. This is local engineering evidence, not a production capacity certification. - -## Durable ingest results - -Values are medians of three runs. Each latency value is the median of the three per-run percentiles, not a percentile pooled across runs. - -| Exporters | Native rows/s | Shared rows/s | Change | Native p95 | Shared p95 | Native p99 | Shared p99 | -|---|---:|---:|---:|---:|---:|---:|---:| -| 1 | 25,171 | 25,450 | +1.1% | 45.81 ms | 44.98 ms | 49.39 ms | 47.98 ms | -| 16 | 323,135 | 330,112 | +2.2% | 56.04 ms | 54.98 ms | 60.27 ms | 58.01 ms | - -## All measurements - -CPU includes the generator and server in the same process. Allocated bytes are cumulative allocation per accepted row, not retained memory or peak RSS. The non-durable runs count converted rows without storing them, and must not be interpreted as Fanout's durable ingest capacity. - -| Durable | Exporters | Mode | Median rows/s | Rows/s range | CPU cores | Allocated bytes/row | -|---|---:|---|---:|---|---:|---:| -| False | 1 | native | 900,673 | 892,355–906,928 | 1.686 | 2,065 | -| False | 1 | shared | 866,387 | 835,664–876,795 | 1.785 | 2,235 | -| False | 16 | native | 4,744,632 | 4,609,333–5,335,230 | 9.113 | 1,961 | -| False | 16 | shared | 5,165,203 | 4,975,016–5,194,013 | 10.292 | 1,982 | -| True | 1 | native | 25,171 | 21,316–25,572 | 0.152 | 3,955 | -| True | 1 | shared | 25,450 | 24,953–25,586 | 0.160 | 3,965 | -| True | 16 | native | 323,135 | 312,194–326,997 | 1.204 | 4,395 | -| True | 16 | shared | 330,112 | 320,596–332,346 | 1.249 | 4,437 | - -Without storage, one exporter was 3.8% slower on the shared transport and allocated approximately 8.2% more bytes per row. At sixteen exporters the shared median was higher, but the throughput ranges overlapped. This does not establish a universal transport advantage or penalty. With durable writes and sixteen exporters, combined CPU medians were 1.204 versus 1.249 cores, and allocation per row rose approximately 1.0%. - -## Method - -- Fanout revision: `9c6567c24a591f233faaaa950a4be2e7912efc32`; production code unchanged. -- Go 1.27.0, darwin/arm64, 14 logical CPUs and GOMAXPROCS=14; grpc-go v1.83.2. -- Real loopback TCP, plaintext gRPC. Native `grpc.Server.Serve` versus `grpc.Server.ServeHTTP` wrapped in `h2c.NewHandler` and the proposed content-type/path dispatcher. -- Same production ingest authentication interceptor, temporary SQLite settings store, and OTLP conversion handlers in both modes. Generated temporary credentials are never logged. -- 1,000-row exports cycling through traces, logs and gauge metric points. Each worker owns a persistent gRPC connection with one outstanding export. Requests reuse the existing small test fixtures; values have low cardinality and are repeated. -- Each connection and all three signals are warmed before timing. Each measured case issues requests for ten seconds, then waits for in-flight acknowledgements; that drain time is included in throughput. -- Three repetitions for each mode at one and sixteen workers, with native/shared order reversed in repetition two. Runs are sequential, with a fresh store for every case. -- Durable mode uses the production writer, default 50,000-row batch target, and a fresh temporary Parquet repository. Counting occurs only after Submit succeeds. Accepted server rows must exactly equal successful exports times 1,000. -- Zero export errors in all 24 cases. Every durable repository passed VerifyBatches after writer shutdown. The earlier four-second smoke matrix also passed, but its timings are not included here. - -## Limits and decision - -Proceeding with a single-port prototype is supported by these results: the tested durable workloads did not show a performance regression. They do not prove zero overhead or establish equivalence across workloads. There are only three short repetitions, the laptop is not a dedicated benchmark host, and client/server CPU contention and filesystem behavior affect the measurements. - -TLS, an actual reverse proxy, remote network latency, concurrent dashboard/API queries, RSS, OTLP/HTTP load, compression, larger or higher-cardinality payloads, sustained saturation and production lifecycle behavior were not measured. The normal app server was not started: its fallback in the isolated dispatcher returns 404. Production auth-boundary and shutdown integration still require their own tests when implementing the feature. - -The independent fanout-bench suite was inspected but not used: its managed path requires a provisioned Linux host. Before claiming deployment-wide performance equivalence, repeat the comparison on the target Linux host with representative telemetry and concurrent read load. - -## Reproduce - -The experiment is build-tagged and excluded from normal tests: - -```sh -GOTOOLCHAIN=go1.27.0 go test -tags transportbench ./internal/ingest -run '^TestTransportComparison$' -v -count=1 -timeout=10m -``` - -Raw evidence: [test log](single-port-2026-09-08.log), [JSON measurements](single-port-2026-09-08.json), [harness](../../internal/ingest/transport_bench_test.go). diff --git a/docs/benchmarks/single-port-stdlib-2026-09-08.json b/docs/benchmarks/single-port-stdlib-2026-09-08.json deleted file mode 100644 index 19ed47b33..000000000 --- a/docs/benchmarks/single-port-stdlib-2026-09-08.json +++ /dev/null @@ -1,170 +0,0 @@ -[ - { - "allocated_bytes_per_row": 3945.206243902439, - "concurrency": 1, - "cpu_cores_client_and_server": 0.146333894075857, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 39.925292, - "p95_ms": 47.1955, - "p99_ms": 49.865125, - "rows": 246000, - "rows_per_second": 24526.31976662056, - "seconds": 10.030041292 - }, - { - "allocated_bytes_per_row": 3930.7976356275303, - "concurrency": 1, - "cpu_cores_client_and_server": 0.14484815287822964, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 40.84975, - "p95_ms": 46.052958, - "p99_ms": 48.194958, - "rows": 247000, - "rows_per_second": 24618.919334652703, - "seconds": 10.032934291 - }, - { - "allocated_bytes_per_row": 3854.1562071713147, - "concurrency": 1, - "cpu_cores_client_and_server": 0.15097412191439982, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 39.2065, - "p95_ms": 44.976666, - "p99_ms": 48.267792, - "rows": 251000, - "rows_per_second": 25028.205131931874, - "seconds": 10.028685584 - }, - { - "allocated_bytes_per_row": 3897.151394957983, - "concurrency": 1, - "cpu_cores_client_and_server": 0.14661725025933367, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 42.032416, - "p95_ms": 48.907666, - "p99_ms": 51.136625, - "rows": 238000, - "rows_per_second": 23734.89863359627, - "seconds": 10.027428542 - }, - { - "allocated_bytes_per_row": 3882.825702811245, - "concurrency": 1, - "cpu_cores_client_and_server": 0.1426491135595872, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 39.078625, - "p95_ms": 48.826792, - "p99_ms": 50.178542, - "rows": 249000, - "rows_per_second": 24840.100170104146, - "seconds": 10.024114166 - }, - { - "allocated_bytes_per_row": 3860.12705511811, - "concurrency": 1, - "cpu_cores_client_and_server": 0.1874227890608438, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 38.953792, - "p95_ms": 43.977375, - "p99_ms": 47.464416, - "rows": 254000, - "rows_per_second": 25359.20412085933, - "seconds": 10.016087208 - }, - { - "allocated_bytes_per_row": 4408.656080882353, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2766668814177105, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 48.944, - "p95_ms": 55.227084, - "p99_ms": 59.405833, - "rows": 3264000, - "rows_per_second": 325895.7333843411, - "seconds": 10.01547325 - }, - { - "allocated_bytes_per_row": 4449.913323529412, - "concurrency": 16, - "cpu_cores_client_and_server": 1.3294230659456743, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 49.078709, - "p95_ms": 54.261042, - "p99_ms": 57.480583, - "rows": 3264000, - "rows_per_second": 326299.92381336645, - "seconds": 10.003067 - }, - { - "allocated_bytes_per_row": 4439.541574257426, - "concurrency": 16, - "cpu_cores_client_and_server": 1.3270159228705851, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 49.207083, - "p95_ms": 56.226542, - "p99_ms": 59.5035, - "rows": 3232000, - "rows_per_second": 322148.3627771322, - "seconds": 10.0326445 - }, - { - "allocated_bytes_per_row": 4385.397882681565, - "concurrency": 16, - "cpu_cores_client_and_server": 1.006868867588621, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 48.135958, - "p95_ms": 102.589958, - "p99_ms": 147.188333, - "rows": 2864000, - "rows_per_second": 286068.59668245824, - "seconds": 10.01158475 - }, - { - "allocated_bytes_per_row": 4411.760161057692, - "concurrency": 16, - "cpu_cores_client_and_server": 1.1766323221883561, - "durable": true, - "errors": 0, - "mode": "native", - "p50_ms": 47.915458, - "p95_ms": 55.384167, - "p99_ms": 59.140833, - "rows": 3328000, - "rows_per_second": 332615.7807498317, - "seconds": 10.0055385 - }, - { - "allocated_bytes_per_row": 4442.297849282297, - "concurrency": 16, - "cpu_cores_client_and_server": 1.2361489518626845, - "durable": true, - "errors": 0, - "mode": "shared", - "p50_ms": 47.525083, - "p95_ms": 54.529584, - "p99_ms": 58.530417, - "rows": 3344000, - "rows_per_second": 332983.25474749616, - "seconds": 10.042547042 - } -] diff --git a/docs/benchmarks/single-port-stdlib-2026-09-08.log b/docs/benchmarks/single-port-stdlib-2026-09-08.log deleted file mode 100644 index 1303de98e..000000000 --- a/docs/benchmarks/single-port-stdlib-2026-09-08.log +++ /dev/null @@ -1,53 +0,0 @@ -=== RUN TestTransportComparison - transport_bench_test.go:36: environment go=go1.27.0 os=darwin arch=arm64 CPUs=14 GOMAXPROCS=14 -=== RUN TestTransportComparison/durable=true/c=1/rep=1/native -2026/09/08 09:42:42 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3945.206243902439,"concurrency":1,"cpu_cores_client_and_server":0.146333894075857,"durable":true,"errors":0,"mode":"native","p50_ms":39.925292,"p95_ms":47.1955,"p99_ms":49.865125,"rows":246000,"rows_per_second":24526.31976662056,"seconds":10.030041292} -=== RUN TestTransportComparison/durable=true/c=1/rep=1/shared -2026/09/08 09:42:53 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3930.7976356275303,"concurrency":1,"cpu_cores_client_and_server":0.14484815287822964,"durable":true,"errors":0,"mode":"shared","p50_ms":40.84975,"p95_ms":46.052958,"p99_ms":48.194958,"rows":247000,"rows_per_second":24618.919334652703,"seconds":10.032934291} -=== RUN TestTransportComparison/durable=true/c=1/rep=2/shared -2026/09/08 09:43:03 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3854.1562071713147,"concurrency":1,"cpu_cores_client_and_server":0.15097412191439982,"durable":true,"errors":0,"mode":"shared","p50_ms":39.2065,"p95_ms":44.976666,"p99_ms":48.267792,"rows":251000,"rows_per_second":25028.205131931874,"seconds":10.028685584} -=== RUN TestTransportComparison/durable=true/c=1/rep=2/native -2026/09/08 09:43:14 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3897.151394957983,"concurrency":1,"cpu_cores_client_and_server":0.14661725025933367,"durable":true,"errors":0,"mode":"native","p50_ms":42.032416,"p95_ms":48.907666,"p99_ms":51.136625,"rows":238000,"rows_per_second":23734.89863359627,"seconds":10.027428542} -=== RUN TestTransportComparison/durable=true/c=1/rep=3/native -2026/09/08 09:43:24 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3882.825702811245,"concurrency":1,"cpu_cores_client_and_server":0.1426491135595872,"durable":true,"errors":0,"mode":"native","p50_ms":39.078625,"p95_ms":48.826792,"p99_ms":50.178542,"rows":249000,"rows_per_second":24840.100170104146,"seconds":10.024114166} -=== RUN TestTransportComparison/durable=true/c=1/rep=3/shared -2026/09/08 09:43:35 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":3860.12705511811,"concurrency":1,"cpu_cores_client_and_server":0.1874227890608438,"durable":true,"errors":0,"mode":"shared","p50_ms":38.953792,"p95_ms":43.977375,"p99_ms":47.464416,"rows":254000,"rows_per_second":25359.20412085933,"seconds":10.016087208} -=== RUN TestTransportComparison/durable=true/c=16/rep=1/native -2026/09/08 09:43:45 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4408.656080882353,"concurrency":16,"cpu_cores_client_and_server":1.2766668814177105,"durable":true,"errors":0,"mode":"native","p50_ms":48.944,"p95_ms":55.227084,"p99_ms":59.405833,"rows":3264000,"rows_per_second":325895.7333843411,"seconds":10.01547325} -=== RUN TestTransportComparison/durable=true/c=16/rep=1/shared -2026/09/08 09:44:00 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4449.913323529412,"concurrency":16,"cpu_cores_client_and_server":1.3294230659456743,"durable":true,"errors":0,"mode":"shared","p50_ms":49.078709,"p95_ms":54.261042,"p99_ms":57.480583,"rows":3264000,"rows_per_second":326299.92381336645,"seconds":10.003067} -=== RUN TestTransportComparison/durable=true/c=16/rep=2/shared -2026/09/08 09:44:15 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4439.541574257426,"concurrency":16,"cpu_cores_client_and_server":1.3270159228705851,"durable":true,"errors":0,"mode":"shared","p50_ms":49.207083,"p95_ms":56.226542,"p99_ms":59.5035,"rows":3232000,"rows_per_second":322148.3627771322,"seconds":10.0326445} -=== RUN TestTransportComparison/durable=true/c=16/rep=2/native -2026/09/08 09:44:29 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4385.397882681565,"concurrency":16,"cpu_cores_client_and_server":1.006868867588621,"durable":true,"errors":0,"mode":"native","p50_ms":48.135958,"p95_ms":102.589958,"p99_ms":147.188333,"rows":2864000,"rows_per_second":286068.59668245824,"seconds":10.01158475} -=== RUN TestTransportComparison/durable=true/c=16/rep=3/native -2026/09/08 09:44:44 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4411.760161057692,"concurrency":16,"cpu_cores_client_and_server":1.1766323221883561,"durable":true,"errors":0,"mode":"native","p50_ms":47.915458,"p95_ms":55.384167,"p99_ms":59.140833,"rows":3328000,"rows_per_second":332615.7807498317,"seconds":10.0055385} -=== RUN TestTransportComparison/durable=true/c=16/rep=3/shared -2026/09/08 09:44:59 INFO migrations applied - transport_bench_test.go:213: RESULT {"allocated_bytes_per_row":4442.297849282297,"concurrency":16,"cpu_cores_client_and_server":1.2361489518626845,"durable":true,"errors":0,"mode":"shared","p50_ms":47.525083,"p95_ms":54.529584,"p99_ms":58.530417,"rows":3344000,"rows_per_second":332983.25474749616,"seconds":10.042547042} ---- PASS: TestTransportComparison (152.17s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=1/native (10.56s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=1/shared (10.51s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=2/shared (10.54s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=2/native (10.51s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=3/native (10.54s) - --- PASS: TestTransportComparison/durable=true/c=1/rep=3/shared (10.50s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=1/native (14.72s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=1/shared (14.68s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=2/shared (14.72s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=2/native (14.62s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=3/native (15.38s) - --- PASS: TestTransportComparison/durable=true/c=16/rep=3/shared (14.91s) -PASS -ok github.com/labstack/fanout/internal/ingest 152.730s diff --git a/docs/superpowers/specs/2026-09-05-logged-in-ui-shell-design.md b/docs/specs/2026-09-05-logged-in-ui-shell-design.md similarity index 100% rename from docs/superpowers/specs/2026-09-05-logged-in-ui-shell-design.md rename to docs/specs/2026-09-05-logged-in-ui-shell-design.md diff --git a/docs/superpowers/specs/2026-10-04-agent-dashboards-design.md b/docs/specs/2026-10-04-agent-dashboards-design.md similarity index 100% rename from docs/superpowers/specs/2026-10-04-agent-dashboards-design.md rename to docs/specs/2026-10-04-agent-dashboards-design.md diff --git a/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md b/docs/specs/2026-10-10-agent-dashboards-m5-design.md similarity index 99% rename from docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md rename to docs/specs/2026-10-10-agent-dashboards-m5-design.md index df5d044e0..c884c2596 100644 --- a/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md +++ b/docs/specs/2026-10-10-agent-dashboards-m5-design.md @@ -272,8 +272,8 @@ real Collector observing real local containers confirms the same page. S1 to S16, S20 and S22 run on the final commit: the eval harness (within $5, with new public host-metric prompts for S22), the browser collector, the Playwright smoke, accessibility and performance tests at demo volume, and the -demo-data metric cross-check. Results go into -`docs/benchmarks/2026-10-agent-dashboards-m5.md`. +demo-data metric cross-check. Verification records stay outside the +repository. ## Testing diff --git a/docs/superpowers/plans/2026-09-05-logged-in-ui-shell.md b/docs/superpowers/plans/2026-09-05-logged-in-ui-shell.md deleted file mode 100644 index a0f02d081..000000000 --- a/docs/superpowers/plans/2026-09-05-logged-in-ui-shell.md +++ /dev/null @@ -1,3639 +0,0 @@ -# Logged-in UI Shell Rebuild Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the header-toggle shell with a persistent rail, turn the chat page into a pane with a sticky composer and visible agent activity, give dashboard widgets native ECharts bodies with per-widget configuration, and replace the sign-in code field with a six-digit PinInput, fixing the CSP font, "Ms", new-chat 404, header clipping and fake "Live" defects on the way. - -**Architecture:** `ui/host` is a React 19 + Mantine 9 single-page app served from `internal/ui/dist`; chat runs over AG-UI (`HttpAgent`) and renders tool results as sandboxed MCP-app iframes built from `ui/apps` into `internal/mcp/apps`. The work splits `App.tsx` into a session (`App.tsx`), a shell (`shell.tsx`), a rail (`rail.tsx`) and a chat pane (`chat.tsx`), moves the pure chart and format helpers up to `ui/` so both bundles share them, and rebuilds `dashboard.tsx` around a `widgets/` directory with one file per widget type. - -**Tech Stack:** React 19.2.8, Mantine 9.5.2 (`@mantine/core`, `@mantine/hooks`), TanStack Router 1.170 and Query 5.102, `@ag-ui/client` 0.0.59, `@modelcontextprotocol/ext-apps` 1.7.5, `react-grid-layout` 2.2.4 (`/legacy` import), ECharts 6.1.0 (SVG renderer), Phosphor icons, Vitest 4 + happy-dom, Bun for package management, `just` recipes for builds and the gate. - -**Spec:** `docs/superpowers/specs/2026-09-05-logged-in-ui-shell-design.md` - -## Global Constraints - -- Branch `ui/shell-rebuild`. -- The lefthook pre-commit hook runs `cd ui/host && bun run lint` (which is `tsc --noEmit`) whenever staged files match `ui/host/**/*.{ts,tsx}` or `ui/theme.ts`. Every task must leave `bun run lint` green in `ui/host` before committing. `ui/apps` has the same `lint` script; run it when apps files change. -- Naming: the feature is **Chat / Chats**, the button is **New chat**. The strings "Investigations", "Investigation", "Conversation history" and "Guidance" must not appear in `ui/host/src` or `ui/apps/src` when the plan is done. -- `ui/` (the directory holding `theme.ts`, `tokens.ts`) has no `node_modules`. Files placed there may import only sibling files with relative paths, never bare package names. -- Embedded assets (`internal/ui/dist`, `internal/mcp/apps`) are committed. `just check` fails when they are stale, so a build in a task is followed either by committing the rebuilt assets (Tasks 7 and 14) or by discarding them with `git checkout -- internal/mcp/apps internal/ui/dist` (every other task). -- `echarts` is pinned to `6.1.0` in `ui/host`, the same version `ui/apps` uses. -- Widget types stay exactly `overview | topology | activity | assistant | performance | trace | logs` (server allowlist in `internal/dashboard/service.go`). No Go changes. -- Dashboard windows stay `15m | 1h | 6h | 24h | 168h | 720h`. -- Widget data refetch interval 30 000 ms, 15 000 ms while a query is in error. Dashboard list and selected-dashboard polls 30 000 ms. -- Copy in code comments, commit messages and docs is plain English prose. -- Run tests from `ui/host` with `bun run test ` (Vitest accepts a path filter). Run the full suite with `just ui-test`. - ---- - -## File Structure - -**Shared (`ui/`)** -- `ui/format.ts` — moved from `ui/apps/src/format.ts` unchanged: `integer`, `percent`, `duration`, `windowLabel`, `timelineTimestamp`. -- `ui/contracts.ts` — moved from `ui/apps/src/contracts.ts` unchanged: the observability result types. -- `ui/chart.ts` — new: `healthColor`, `chartTheme`, `statusHex`, `seriesColor`, `severityColor`, `severityHex`, pure functions over `./tokens`. - -**Host (`ui/host/src/`)** -- `api.ts` — new: `getJSON(url)` and the dashboard record types, shared by `rail.tsx` and `dashboard.tsx`. -- `app-context.ts` — new: `FanoutAppContext`, `useFanoutApp`, `createDashboardPrompt`. Breaks the circular import between the session, the shell and the pages. -- `App.tsx` — the session only: agent, thread state, drafts, subscriptions, keyboard "/" shortcut; renders `` inside the context provider. -- `shell.tsx` — new: `AppShell` with header (burger, brand, colour-scheme toggle, account menu), navbar ``, mobile `` with the same rail, Cmd-K handling. -- `rail.tsx` — new: New chat button, Chats section (threads, groups, rename/delete modals), Dashboards section, search field. Router-free; callbacks only. -- `chat.tsx` — new: `ChatPage`, `Composer`, `Welcome`, `ChatMessage`, `toolTitle`, `activityLabel`. -- `echart.tsx` — new: the `EChart` component (copy of `ui/apps/src/echart.tsx` with `height: number | string`). -- `dashboard-layout.ts` — gains `WidgetType`, `widgetDefaults`, `nextDashboardSlot`. -- `widgets/data.ts` — new: `Filters`, `widgetParams`, `useObservability`, `useLastUpdated`, `dashboardWindows`, `windowName`. -- `widgets/widget-card.tsx` — new: card chrome, hover actions menu, remove, configure modal wiring. -- `widgets/configure.tsx` — new: `ConfigureWidget` modal (service / severity / search / trace id). -- `widgets/overview.tsx`, `topology.tsx`, `activity.tsx`, `performance.tsx`, `trace.tsx`, `logs.tsx`, `assistant.tsx` — new: one body per widget type. -- `dashboard.tsx` — page rewrite: header, control row, updated line, grid. -- `auth.tsx` — `ViewerContext`/`useViewer`, PinInput code step, resend and change-email, heading sizes. -- `mcp-app-frame.tsx` — CSP `font-src`, 240px floor. -- `index.css` — rail row, chat pane, widget card rules. -- `routes/chat.index.tsx` — renders `ChatPage` (draft) instead of redirecting. -- Deleted: `chat-history.tsx`, `chat-history.test.tsx`. - -**Apps (`ui/apps/src/`)** -- `components.tsx` — `ViewHeader` slimmed; chart helpers removed (imported from `ui/chart.ts`). -- `overview.tsx`, `topology.tsx`, `performance.tsx`, `trace.tsx`, `logs.tsx` — import paths updated, `eyebrow` prop dropped. -- Deleted: `format.ts`, `contracts.ts` (moved up). - -**Tests (`ui/host/src/`)** -- `mcp-app-frame.test.tsx` (extend), `auth.test.tsx` (extend), `dashboard-layout.test.ts` (extend), `rail.test.tsx` (new, replaces `chat-history.test.tsx`), `chat.test.tsx` (new), `app.test.tsx` (new), `widgets/widgets.test.tsx` (new), `dashboard.test.tsx` (new). - ---- - -## Phase A: foundations and defects - -### Task 1: Always allow inlined fonts in the MCP app CSP - -**Files:** -- Modify: `ui/host/src/mcp-app-frame.tsx:583-602` -- Test: `ui/host/src/mcp-app-frame.test.tsx` - -**Interfaces:** -- Produces: `mcpAppCSP(meta: unknown): string` unchanged signature; output now always contains `font-src 'self' data:`. - -- [ ] **Step 1: Write the failing test** - -Append inside the `describe("MCPAppFrame")` block, after the existing "builds CSP directives only from safe declared domains" test: - -```tsx - it("always allows inlined fonts so embedded views render in the product typeface", () => { - expect(mcpAppCSP({})).toContain("font-src 'self' data:"); - expect(mcpAppCSP(undefined)).toContain("font-src 'self' data:"); - const policy = mcpAppCSP({ ui: { csp: { resourceDomains: ["https://cdn.example.com"] } } }); - expect(policy).toContain("font-src 'self' data: https://cdn.example.com"); - }); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cd ui/host && bun run test src/mcp-app-frame.test.tsx` -Expected: FAIL, `expected "default-src 'none'; …" to contain "font-src 'self' data:"`. - -- [ ] **Step 3: Change the directive list** - -In `mcpAppCSP`, replace the `directives` array and the conditional `font-src` push with: - -```ts - const directives = [ - "default-src 'none'", - `script-src 'self' 'unsafe-inline'${resourceSuffix}`, - `style-src 'self' 'unsafe-inline'${resourceSuffix}`, - `img-src 'self' data:${resourceSuffix}`, - `media-src 'self' data:${resourceSuffix}`, - // The apps build inlines their woff2 files as data: URIs; without this - // every embedded view falls back to the system font. - `font-src 'self' data:${resourceSuffix}`, - `connect-src ${connect.length ? connect.join(" ") : "'none'"}`, - ]; - if (frames.length) directives.push(`frame-src ${frames.join(" ")}`); - if (bases.length) directives.push(`base-uri ${bases.join(" ")}`); -``` - -Delete the line `if (resources.length) directives.push(`font-src 'self' ${resources.join(" ")}`);`. - -- [ ] **Step 4: Run the test file to verify it passes** - -Run: `cd ui/host && bun run test src/mcp-app-frame.test.tsx` -Expected: PASS, all tests in the file. - -- [ ] **Step 5: Lint and commit** - -```bash -cd ui/host && bun run lint -cd "$(git rev-parse --show-toplevel)" -git add ui/host/src/mcp-app-frame.tsx ui/host/src/mcp-app-frame.test.tsx -git commit -m "fix(ui): let embedded views load their inlined fonts - -The MCP app CSP only emitted font-src when a resource domain was declared, so -the data: woff2 files the apps build inlines were blocked and every embedded -view rendered in the system font." -``` - ---- - -### Task 2: Move format, contracts and chart helpers to `ui/` - -**Files:** -- Move: `ui/apps/src/format.ts` → `ui/format.ts`; `ui/apps/src/contracts.ts` → `ui/contracts.ts` -- Create: `ui/chart.ts` -- Modify: `ui/apps/src/components.tsx`, `ui/apps/src/overview.tsx`, `ui/apps/src/topology.tsx`, `ui/apps/src/performance.tsx`, `ui/apps/src/trace.tsx`, `ui/apps/src/logs.tsx` - -**Interfaces:** -- Produces (for Tasks 9 to 11): `import { integer, percent, duration, windowLabel, timelineTimestamp } from "../../format"` and `import type { Overview, Topology, Performance, TraceDetail, Logs, Result, ServiceHealth, Endpoint, LogEntry, TraceSpan, Edge } from "../../contracts"` from `ui/host/src`, and `import { healthColor, chartTheme, statusHex, seriesColor, severityColor, severityHex } from "../../chart"`. From `ui/host/src/widgets` the same paths are `../../../format`, `../../../contracts`, `../../../chart`. - -- [ ] **Step 1: Move the two pure modules** - -```bash -cd "$(git rev-parse --show-toplevel)" -git mv ui/apps/src/format.ts ui/format.ts -git mv ui/apps/src/contracts.ts ui/contracts.ts -``` - -- [ ] **Step 2: Create `ui/chart.ts`** - -```ts -/* Chart and status helpers shared by the browser host and the embedded views. - * Pure functions over ./tokens only: this directory has no node_modules, so - * nothing here may import a package. */ -import { bad, chart, info, ok, series, warn } from "./tokens"; - -export function healthColor(health: string) { - return health === "healthy" ? "ok" : health === "degraded" ? "warn" : "bad"; -} - -/* A chart is drawn into a canvas or SVG that cannot read CSS custom - properties, so these hand ECharts resolved values from the same ramps - Mantine gets. The shade differs by scheme for the same reason the accent - does: the palette's own hue reads on Ayu, a darker stop is needed on white. */ -export function chartTheme(dark: boolean) { - return chart[dark ? "dark" : "light"]; -} - -export function statusHex(dark: boolean) { - const shade = dark ? 5 : 7; - return { ok: ok[shade], warn: warn[shade], bad: bad[shade], info: info[shade] }; -} - -/** One colour per service or metric, where the colour identifies rather than - * grades. Hashed so a service keeps its colour between renders, and drawn from - * a palette with no health hue in it. */ -export function seriesColor(name: string, dark: boolean) { - const palette = series[dark ? "dark" : "light"]; - let hash = 0; - for (const character of name) hash = (hash * 31 + character.charCodeAt(0)) | 0; - return palette[Math.abs(hash) % palette.length]; -} - -export function severityColor(value: string) { - const severity = String(value).toUpperCase(); - if (severity === "ERROR" || severity === "FATAL") return "bad"; - if (severity === "WARN" || severity === "WARNING") return "warn"; - if (severity === "INFO") return "info"; - return "gray"; -} - -export function severityHex(value: string, dark: boolean) { - const status = statusHex(dark); - const severity = String(value).toUpperCase(); - if (severity === "ERROR" || severity === "FATAL") return status.bad; - if (severity === "WARN" || severity === "WARNING") return status.warn; - if (severity === "INFO") return status.info; - return chartTheme(dark).muted; -} -``` - -- [ ] **Step 3: Strip the helpers from `ui/apps/src/components.tsx`** - -Delete the `healthColor`, `chartTheme`, `statusHex` and `seriesColor` functions and the comment block above `chartTheme` (lines 96 to 122 of the current file). Change the tokens import at the top to: - -```ts -import { fanoutCssVariables, fanoutThemeConfig } from "../../theme"; -``` - -(remove the `import { bad, chart, info, ok, series, warn } from "../../tokens";` line). - -- [ ] **Step 4: Update the five app entry files** - -In each of `overview.tsx`, `topology.tsx`, `performance.tsx`, `trace.tsx`, `logs.tsx`: - -- change `from "./contracts"` to `from "../../contracts"`; -- change `from "./format"` to `from "../../format"`; -- remove `healthColor`, `chartTheme`, `statusHex`, `seriesColor` from the `./components` import and add `import { … } from "../../chart";` with exactly the names that file uses. For reference: `overview.tsx` uses `healthColor`; `topology.tsx` uses `chartTheme`, `statusHex`; `performance.tsx` uses `chartTheme`, `healthColor`, `seriesColor`, `statusHex`; `trace.tsx` uses `seriesColor` and its own `severityColor`; `logs.tsx` uses `chartTheme`, `statusHex` and its own `severityColor` and `severityHex`. -- in `trace.tsx` delete the local `function severityColor(value: string) {…}` and import `severityColor` from `"../../chart"`; -- in `logs.tsx` delete the local `severityColor` and `severityHex` functions and import both from `"../../chart"`. - -- [ ] **Step 5: Type-check and build the apps, then discard the built HTML** - -```bash -cd ui/apps && bun run lint && bun run build -cd "$(git rev-parse --show-toplevel)" && git checkout -- internal/mcp/apps -git status --short -``` - -Expected: lint clean, five `✓ built` lines, and `git status` shows only the `ui/` changes. - -- [ ] **Step 6: Commit** - -```bash -git add ui/format.ts ui/contracts.ts ui/chart.ts ui/apps/src -git commit -m "refactor(ui): share format, contracts and chart helpers between host and apps" -``` - ---- - -### Task 3: Expose the signed-in account to the app - -**Files:** -- Modify: `ui/host/src/auth.tsx` -- Test: `ui/host/src/auth.test.tsx` - -**Interfaces:** -- Produces: `export type Viewer = { id: string; email: string; name: string; role: string }` and `export function useViewer(): Viewer` from `./auth`. Available anywhere under `AuthGate` once authenticated. - -- [ ] **Step 1: Write the failing test** - -Append to `auth.test.tsx`, inside the `describe` block: - -```tsx - it("exposes the signed-in account through useViewer", async () => { - window.happyDOM.setURL("https://fanout.example.com/"); - fetchMock.mockImplementation(async (input) => authResponse(input, { - id: "viewer-123", - email: "v@example.com", - name: "Vee", - role: "viewer", - })); - function Probe() { - const viewer = useViewer(); - return
Signed in as {viewer.email} ({viewer.role})
; - } - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - await act(async () => root.render( - - - , - )); - - await vi.waitFor(() => expect(document.body.textContent).toContain("Signed in as v@example.com (viewer)")); - await act(async () => root.unmount()); - }); -``` - -Change the import line to `import AuthGate, { useViewer } from "./auth";`. - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cd ui/host && bun run test src/auth.test.tsx` -Expected: FAIL, `useViewer` is not exported (type error surfaces as a runtime `TypeError: useViewer is not a function`). - -- [ ] **Step 3: Add the viewer context** - -In `auth.tsx`, after the `RuntimeStatusContext` declaration: - -```ts -export type Viewer = { id: string; email: string; name: string; role: string }; -const ViewerContext = createContext(null); - -export function useViewer(): Viewer { - const viewer = useContext(ViewerContext); - if (!viewer) throw new Error("Fanout viewer is unavailable"); - return viewer; -} - -function viewerFromMe(user: unknown): Viewer | null { - if (!user || typeof user !== "object") return null; - const record = user as Record; - if (typeof record.id !== "string" || record.id === "") return null; - return { - id: record.id, - email: typeof record.email === "string" ? record.email : "", - name: typeof record.name === "string" ? record.name : "", - role: typeof record.role === "string" ? record.role : "", - }; -} -``` - -Inside `AuthGate`, add state `const [account, setAccount] = useState(null);` and a loader: - -```ts - async function loadAccount() { - const response = await fetch("/api/auth/me", { credentials: "same-origin" }); - if (!response.ok) { setViewer("none"); setAccount(null); return; } - const user = await response.json().catch(() => null); - setViewer(browserViewerFromMe(user)); - setAccount(viewerFromMe(user)); - } -``` - -Replace the inline `/api/auth/me` fetch chain in the first effect with `loadAccount().catch(() => setViewer("none")).finally(() => setSessionReady(true));`. - -Everywhere the code sets `setViewer("user")` after a successful `login-link`, `verify` or `setup` call, follow it with `void loadAccount();` (three places: the login-token effect, `submit` for the setup and verify branches, and the "Continue to Fanout" button). The `browserViewerFromMe` import stays. - -Wrap the authenticated render: - -```tsx - if (authenticated) return - {children} - ; -``` - -The fallback covers the instant between a successful verify and the `/me` reload; `email` and `name` are the form fields already in state. - -- [ ] **Step 4: Run the test file to verify it passes** - -Run: `cd ui/host && bun run test src/auth.test.tsx` -Expected: PASS, seven tests. - -- [ ] **Step 5: Lint and commit** - -```bash -cd ui/host && bun run lint -cd "$(git rev-parse --show-toplevel)" -git add ui/host/src/auth.tsx ui/host/src/auth.test.tsx -git commit -m "feat(ui): expose the signed-in account to the app" -``` - ---- - -## Phase B: shell - -### Task 4: Shared API helper and the rail - -**Files:** -- Create: `ui/host/src/api.ts`, `ui/host/src/rail.tsx` -- Test: `ui/host/src/rail.test.tsx` (new) -- Delete: `ui/host/src/chat-history.tsx`, `ui/host/src/chat-history.test.tsx` -- Modify: `ui/host/src/index.css` (rename the drawer row rules) - -**Interfaces:** -- Produces from `./api`: `getJSON(url: string): Promise`, `type DashboardSummary = { id: string; name: string; description: string; is_default: boolean; widget_count: number; updated_at: string }`, `type DashboardRecord`, `type Envelope = { data: T }`, `dashboardsQueryKey = ["dashboards"] as const`. -- Produces from `./rail`: `export const threadHistoryQueryKey = ["agent-threads"] as const`, `export type RailHandle = { focusSearch(): void }`, `export type RailProps = { agentAvailable: boolean; activeThreadID?: string; activeDashboardID?: string; autoFocusSearch?: boolean; onNewChat: () => void; onSelectThread: (threadID: string) => void; onDeletedThread: (threadID: string) => void; onSelectDashboard: (dashboardID: string) => void; onCreateDashboard: () => void; ref?: Ref }`, default export `Rail`. -- Consumes: `authorizedFetch` from `./auth`. - -- [ ] **Step 1: Create `api.ts`** - -```ts -import { authorizedFetch } from "./auth"; - -export type Envelope = { data: T }; -export type DashboardSummary = { id: string; name: string; description: string; is_default: boolean; widget_count: number; updated_at: string }; -export type DashboardLayoutRecord = { i: string; x: number; y: number; w: number; h: number; minW?: number; minH?: number }; -export type DashboardWidgetRecord = { id: string; type: string; title: string; config?: Record; enabled: boolean }; -export type DashboardState = { layout: DashboardLayoutRecord[]; widgets: DashboardWidgetRecord[]; filters: { window: string; namespace: string } }; -export type DashboardRecord = { id: string; name: string; description: string; is_default: boolean; state: DashboardState; updated_at: string }; - -export const dashboardsQueryKey = ["dashboards"] as const; - -export async function getJSON(url: string): Promise { - const response = await authorizedFetch(url); - if (!response.ok) throw new Error(`Request failed (${response.status})`); - return response.json() as Promise; -} -``` - -- [ ] **Step 2: Write the failing rail test** - -Create `rail.test.tsx`: - -```tsx -import { MantineProvider } from "@mantine/core"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { act, createRef } from "react"; -import { createRoot } from "react-dom/client"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import Rail, { type RailHandle } from "./rail"; - -const fetchMock = vi.fn(); - -function json(body: unknown, status = 200) { - return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); -} - -function threadsPage(query = "") { - return json({ - threads: [{ threadId: "thread-checkout", title: query ? `Result for ${query}` : "Checkout latency", updatedAt: "2026-07-22 03:00:00" }], - nextCursor: "", - }); -} - -function dashboards() { - return json({ dashboards: [ - { id: "dash-main", name: "System overview", description: "", is_default: true, widget_count: 4, updated_at: "2026-07-22 03:00:00" }, - { id: "dash-checkout", name: "Checkout", description: "", is_default: false, widget_count: 2, updated_at: "2026-07-22 03:00:00" }, - ] }); -} - -function respond(input: RequestInfo | URL, init?: RequestInit) { - const url = new URL(String(input), "http://localhost"); - if (url.pathname === "/api/dashboards") return dashboards(); - if (url.pathname === "/api/agent/threads") return threadsPage(url.searchParams.get("q") ?? ""); - if (url.pathname.startsWith("/api/agent/threads/") && init?.method === "PATCH") return json({ title: "Checkout follow-up" }); - if (url.pathname.startsWith("/api/agent/threads/") && init?.method === "DELETE") return new Response(null, { status: 204 }); - throw new Error(`unexpected request: ${url.pathname}`); -} - -function mount(props: Partial[0]> = {}) { - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - const handlers = { - onNewChat: vi.fn(), onSelectThread: vi.fn(), onDeletedThread: vi.fn(), onSelectDashboard: vi.fn(), onCreateDashboard: vi.fn(), - }; - const ref = createRef(); - const render = () => root.render( - - - - - , - ); - return { root, handlers, ref, render }; -} - -function setValue(input: HTMLInputElement, value: string) { - Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(input, value); - input.dispatchEvent(new InputEvent("input", { bubbles: true, data: value, inputType: "insertText" })); - input.dispatchEvent(new Event("change", { bubbles: true })); -} - -describe("Rail", () => { - beforeEach(() => { - vi.stubGlobal("fetch", fetchMock); - fetchMock.mockReset(); - fetchMock.mockImplementation(async (input, init) => respond(input, init)); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - document.body.innerHTML = ""; - }); - - it("lists chats and dashboards, marks the active rows, and selects", async () => { - const { root, handlers, render } = mount(); - await act(async () => render()); - await vi.waitFor(() => expect(document.body.textContent).toContain("Checkout latency")); - await vi.waitFor(() => expect(document.body.textContent).toContain("System overview")); - expect(document.body.textContent).toContain("Chats"); - expect(document.body.textContent).toContain("Dashboards"); - expect(document.body.textContent).not.toContain("Investigation"); - - const activeRows = document.querySelectorAll('[aria-current="page"]'); - expect(activeRows).toHaveLength(2); - - const thread = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.includes("Checkout latency")); - await act(async () => thread?.click()); - expect(handlers.onSelectThread).toHaveBeenCalledWith("thread-checkout"); - - const dashboard = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.includes("Checkout") && !button.textContent.includes("latency")); - await act(async () => dashboard?.click()); - expect(handlers.onSelectDashboard).toHaveBeenCalledWith("dash-checkout"); - - const newChat = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.trim() === "New chat"); - await act(async () => newChat?.click()); - expect(handlers.onNewChat).toHaveBeenCalled(); - await act(async () => root.unmount()); - }); - - it("hides chat affordances when the agent is unavailable", async () => { - const { root, render } = mount({ agentAvailable: false }); - await act(async () => render()); - await vi.waitFor(() => expect(document.body.textContent).toContain("System overview")); - expect(document.body.textContent).not.toContain("New chat"); - expect(document.body.textContent).not.toContain("Chats"); - expect(document.body.textContent).not.toContain("Create with AI"); - expect(fetchMock.mock.calls.some(([input]) => String(input).includes("/api/agent/threads"))).toBe(false); - await act(async () => root.unmount()); - }); - - it("searches chats on the server and filters dashboards locally", async () => { - const { root, ref, render } = mount(); - await act(async () => render()); - await vi.waitFor(() => expect(document.body.textContent).toContain("Checkout latency")); - act(() => ref.current?.focusSearch()); - const search = document.querySelector('input[aria-label="Search chats and dashboards"]') as HTMLInputElement; - expect(document.activeElement).toBe(search); - await act(async () => setValue(search, "checkout")); - await vi.waitFor(() => expect(fetchMock.mock.calls.some(([input]) => String(input).includes("q=checkout"))).toBe(true), { timeout: 1500 }); - await vi.waitFor(() => expect(document.body.textContent).toContain("Result for checkout")); - expect(document.body.textContent).toContain("Checkout"); - expect(document.body.textContent).not.toContain("System overview"); - await act(async () => root.unmount()); - }); - - it("renames and deletes a chat", async () => { - const { root, handlers, render } = mount(); - await act(async () => render()); - await vi.waitFor(() => expect(document.querySelector('button[aria-label="Actions for Checkout latency"]')).not.toBeNull()); - const openActions = () => document.querySelector('button[aria-label="Actions for Checkout latency"]') as HTMLButtonElement; - - await act(async () => openActions().click()); - const rename = Array.from(document.querySelectorAll('[role="menuitem"]')).find((item) => item.textContent?.includes("Rename")); - await act(async () => rename?.click()); - const name = document.querySelector('input[value="Checkout latency"]') as HTMLInputElement; - await act(async () => setValue(name, "Checkout follow-up")); - const save = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.trim() === "Save"); - await act(async () => save?.click()); - await vi.waitFor(() => expect(fetchMock.mock.calls.some(([, init]) => init?.method === "PATCH")).toBe(true)); - - await act(async () => openActions().click()); - const remove = Array.from(document.querySelectorAll('[role="menuitem"]')).find((item) => item.textContent?.includes("Delete")); - await act(async () => remove?.click()); - const confirm = Array.from(document.querySelectorAll('[role="dialog"] button')).find((button) => button.textContent?.trim() === "Delete"); - await act(async () => confirm?.click()); - await vi.waitFor(() => expect(fetchMock.mock.calls.some(([, init]) => init?.method === "DELETE")).toBe(true)); - expect(handlers.onDeletedThread).toHaveBeenCalledWith("thread-checkout"); - await act(async () => root.unmount()); - }); -}); -``` - -- [ ] **Step 3: Run the test to verify it fails** - -Run: `cd ui/host && bun run test src/rail.test.tsx` -Expected: FAIL, `Cannot find module './rail'`. - -- [ ] **Step 4: Create `rail.tsx`** - -```tsx -import { ActionIcon, Alert, Badge, Box, Button, Center, Group, Kbd, Loader, Menu, Modal, ScrollArea, Stack, Text, TextInput, UnstyledButton } from "@mantine/core"; -import { useDebouncedValue } from "@mantine/hooks"; -import { DotsThree, MagnifyingGlass, PencilSimple, Plus, Sparkle, Trash } from "@phosphor-icons/react"; -import { useInfiniteQuery, useQuery, useQueryClient } from "@tanstack/react-query"; -import { useEffect, useImperativeHandle, useMemo, useRef, useState, type Ref } from "react"; -import { dashboardsQueryKey, getJSON, type DashboardSummary } from "./api"; -import { authorizedFetch } from "./auth"; - -export const threadHistoryQueryKey = ["agent-threads"] as const; - -export type RailHandle = { focusSearch(): void }; - -export type RailProps = { - agentAvailable: boolean; - activeThreadID?: string; - activeDashboardID?: string; - /** The mobile drawer mounts a second rail; it focuses search on open. */ - autoFocusSearch?: boolean; - onNewChat: () => void; - onSelectThread: (threadID: string) => void; - onDeletedThread: (threadID: string) => void; - onSelectDashboard: (dashboardID: string) => void; - onCreateDashboard: () => void; - ref?: Ref; -}; - -type ThreadSummary = { threadId: string; title: string; updatedAt: string }; -type ThreadPage = { threads: ThreadSummary[]; nextCursor: string }; - -async function fetchThreads(query: string, cursor: string): Promise { - const params = new URLSearchParams({ limit: "30" }); - if (query) params.set("q", query); - if (cursor) params.set("cursor", cursor); - const response = await authorizedFetch(`/api/agent/threads?${params}`); - if (!response.ok) throw new Error(`Unable to load chats (${response.status})`); - return response.json(); -} - -export default function Rail({ agentAvailable, activeThreadID, activeDashboardID, autoFocusSearch = false, onNewChat, onSelectThread, onDeletedThread, onSelectDashboard, onCreateDashboard, ref }: RailProps) { - const queryClient = useQueryClient(); - const [search, setSearch] = useState(""); - const [query] = useDebouncedValue(search.trim(), 250); - const [renaming, setRenaming] = useState(null); - const [deleting, setDeleting] = useState(null); - const [renameTitle, setRenameTitle] = useState(""); - const [mutationError, setMutationError] = useState(""); - const [busy, setBusy] = useState(false); - const searchRef = useRef(null); - useImperativeHandle(ref, () => ({ focusSearch: () => searchRef.current?.focus() }), []); - useEffect(() => { if (autoFocusSearch) requestAnimationFrame(() => searchRef.current?.focus()); }, [autoFocusSearch]); - - const history = useInfiniteQuery({ - queryKey: [...threadHistoryQueryKey, query], - queryFn: ({ pageParam }) => fetchThreads(query, pageParam), - initialPageParam: "", - getNextPageParam: (last) => last.nextCursor || undefined, - enabled: agentAvailable, - }); - const dashboards = useQuery({ queryKey: dashboardsQueryKey, queryFn: () => getJSON<{ dashboards: DashboardSummary[] }>("/api/dashboards"), refetchInterval: 30_000 }); - const threads = useMemo(() => history.data?.pages.flatMap((page) => page.threads) ?? [], [history.data]); - const groups = useMemo(() => groupThreads(threads), [threads]); - const visibleDashboards = useMemo(() => { - const items = dashboards.data?.dashboards ?? []; - const needle = query.toLowerCase(); - return needle ? items.filter((item) => item.name.toLowerCase().includes(needle)) : items; - }, [dashboards.data, query]); - - function beginRename(thread: ThreadSummary) { setMutationError(""); setRenameTitle(thread.title); setRenaming(thread); } - - async function renameThread() { - if (!renaming || !renameTitle.trim()) return; - setBusy(true); setMutationError(""); - try { - const response = await authorizedFetch(`/api/agent/threads/${encodeURIComponent(renaming.threadId)}`, { method: "PATCH", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ title: renameTitle.trim() }) }); - if (!response.ok) throw new Error(`Unable to rename chat (${response.status})`); - await queryClient.invalidateQueries({ queryKey: threadHistoryQueryKey }); - setRenaming(null); - } catch (cause) { - setMutationError(cause instanceof Error ? cause.message : "Unable to rename this chat."); - } finally { setBusy(false); } - } - - async function deleteThread() { - if (!deleting) return; - setBusy(true); setMutationError(""); - try { - const response = await authorizedFetch(`/api/agent/threads/${encodeURIComponent(deleting.threadId)}`, { method: "DELETE" }); - if (!response.ok) throw new Error(`Unable to delete chat (${response.status})`); - const deletedID = deleting.threadId; - setDeleting(null); - await queryClient.invalidateQueries({ queryKey: threadHistoryQueryKey }); - onDeletedThread(deletedID); - } catch (cause) { - setMutationError(cause instanceof Error ? cause.message : "Unable to delete this chat."); - } finally { setBusy(false); } - } - - return - {agentAvailable && } - - - {agentAvailable && - Chats - {history.isLoading &&
} - {history.isError && Chats could not be loaded.} - {!history.isLoading && !history.isError && threads.length === 0 && {query ? "No matching chats" : "No chats yet"}} - {groups.map((group) => - {groups.length > 1 && {group.label}} - {group.threads.map((thread) => onSelectThread(thread.threadId)} onRename={() => beginRename(thread)} onDelete={() => { setMutationError(""); setDeleting(thread); }} />)} - )} - {history.hasNextPage && } -
} - - Dashboards - {dashboards.isLoading &&
} - {dashboards.isError && Dashboards could not be loaded.} - {visibleDashboards.map((dashboard) => { - const active = dashboard.id === activeDashboardID; - return onSelectDashboard(dashboard.id)}> - - {dashboard.name} - {dashboard.is_default && Default} - - ; - })} - {agentAvailable && } -
-
-
- setSearch(event.currentTarget.value)} leftSection={} rightSection={⌘K} rightSectionWidth={44} placeholder="Search" aria-label="Search chats and dashboards" size="sm" /> - !busy && setRenaming(null)} title="Rename chat" centered> -
{ event.preventDefault(); void renameThread(); }}> - - setRenameTitle(event.currentTarget.value)} maxLength={120} autoFocus /> - {mutationError && {mutationError}} - - - - - -
-
- !busy && setDeleting(null)} title="Delete chat?" centered> - - This permanently removes {deleting?.title} and its saved messages. - {mutationError && {mutationError}} - - - - - - -
; -} - -function SectionLabel({ children }: { children: string }) { - return {children}; -} - -function ThreadRow({ thread, active, onSelect, onRename, onDelete }: { thread: ThreadSummary; active: boolean; onSelect: () => void; onRename: () => void; onDelete: () => void }) { - return - - - - {thread.title} - {threadTime(thread.updatedAt)} - - - - - - - - } onClick={onRename}>Rename - } onClick={onDelete}>Delete - - - - ; -} - -function parseSQLiteTime(value: string): Date { - if (value.includes("T")) return new Date(value); - return new Date(`${value.replace(" ", "T")}Z`); -} - -function dayStart(value: Date): number { - return new Date(value.getFullYear(), value.getMonth(), value.getDate()).getTime(); -} - -export function groupThreads(threads: ThreadSummary[]): Array<{ label: string; threads: ThreadSummary[] }> { - const today = dayStart(new Date()); - const day = 24 * 60 * 60 * 1000; - const groups = new Map(); - for (const thread of threads) { - const age = Math.floor((today - dayStart(parseSQLiteTime(thread.updatedAt))) / day); - const label = age <= 0 ? "Today" : age === 1 ? "Yesterday" : age <= 7 ? "Previous 7 days" : "Older"; - const group = groups.get(label) ?? []; - group.push(thread); - groups.set(label, group); - } - return [...groups].map(([label, items]) => ({ label, threads: items })); -} - -export function threadTime(value: string): string { - const date = parseSQLiteTime(value); - const today = dayStart(new Date()); - const age = Math.floor((today - dayStart(date)) / (24 * 60 * 60 * 1000)); - if (age <= 1) return new Intl.DateTimeFormat(undefined, { hour: "numeric", minute: "2-digit" }).format(date); - return new Intl.DateTimeFormat(undefined, { month: "short", day: "numeric" }).format(date); -} -``` - -- [ ] **Step 5: Rename the row rules in `index.css`** - -Replace the four `.chat-history-*` rules with: - -```css -.rail-row { - border-radius: var(--mantine-radius-md); - transition: background-color 120ms ease; -} - -.rail-row:hover { - background: var(--mantine-color-default-hover); -} - -.rail-row[data-active] { - background: var(--mantine-primary-color-light); -} - -.rail-row-actions { - opacity: 0; - transition: opacity 120ms ease; -} - -.rail-row:hover .rail-row-actions, -.rail-row:focus-within .rail-row-actions { - opacity: 1; -} - -@media (hover: none) { - .rail-row-actions { - opacity: 1; - } -} -``` - -- [ ] **Step 6: Delete the drawer** - -```bash -cd "$(git rev-parse --show-toplevel)" -git rm -q ui/host/src/chat-history.tsx ui/host/src/chat-history.test.tsx -``` - -`App.tsx` still imports `./chat-history`; Task 5 replaces that import. Until then `bun run lint` fails on `App.tsx`, so this task and Task 5 are committed together at the end of Task 5. Run the rail test alone now. - -- [ ] **Step 7: Run the rail test to verify it passes** - -Run: `cd ui/host && bun run test src/rail.test.tsx` -Expected: PASS, four tests. - ---- - -### Task 5: App context, shell, and the session split - -**Files:** -- Create: `ui/host/src/app-context.ts`, `ui/host/src/shell.tsx` -- Modify: `ui/host/src/App.tsx` (rewrite), `ui/host/src/routes/dashboards.index.tsx`, `ui/host/src/routes/dashboards.$dashboardId.tsx`, `ui/host/src/routes/chat.index.tsx`, `ui/host/src/routes/chat.$threadId.tsx`, `ui/host/src/dashboard.tsx` (imports only), `ui/host/src/index.css` -- Test: `ui/host/src/app.test.tsx` (new) - -**Interfaces:** -- Produces from `./app-context`: - -```ts -export type FanoutAppContextValue = { - agentAvailable: boolean; - threadID: string; // the thread the composer sends to (route param or draft id) - threadMissing: boolean; // the route thread returned 404 - messages: Message[]; - messageTimes: Record; // message id -> epoch ms, when known - ready: boolean; - running: boolean; - activity: string; // "Checking system health…" while a tool runs, else "" - input: string; - setInput: (value: string) => void; - error: string; - bottomRef: RefObject; - inputRef: RefObject; - send: (text: string) => Promise; - submit: (event: FormEvent) => void; - stop: () => void; - retry: () => void; - openChat: (prompt?: string) => void; // new thread, optional first prompt - newThread: () => void; // go to /chat draft - selectThread: (threadID: string) => void; -}; -export const FanoutAppContext: Context; -export function useFanoutApp(): FanoutAppContextValue; -export const createDashboardPrompt: string; -``` - -- Consumes: `Rail` and `RailHandle` from `./rail`; `useViewer`, `logout`, `useRuntimeStatus`, `authorizedFetch` from `./auth`; `ChatPage` from `./chat` (Task 6 creates it; until then `routes/chat.*` keep importing `ChatPage` from `../App`, see Step 8). - -- [ ] **Step 1: Create `app-context.ts`** - -```ts -import type { Message } from "@ag-ui/client"; -import { createContext, useContext, type FormEvent, type RefObject } from "react"; - -export type FanoutAppContextValue = { - agentAvailable: boolean; - threadID: string; - threadMissing: boolean; - messages: Message[]; - messageTimes: Record; - ready: boolean; - running: boolean; - activity: string; - input: string; - setInput: (value: string) => void; - error: string; - bottomRef: RefObject; - inputRef: RefObject; - send: (text: string) => Promise; - submit: (event: FormEvent) => void; - stop: () => void; - retry: () => void; - openChat: (prompt?: string) => void; - newThread: () => void; - selectThread: (threadID: string) => void; -}; - -export const FanoutAppContext = createContext(null); - -export function useFanoutApp(): FanoutAppContextValue { - const context = useContext(FanoutAppContext); - if (!context) throw new Error("Fanout app context is unavailable"); - return context; -} - -export const createDashboardPrompt = "Create a new dashboard for me. First ask what I want to monitor, then design it when you have enough context."; -``` - -- [ ] **Step 2: Create `shell.tsx`** - -```tsx -import { ActionIcon, Alert, AppShell, Avatar, Burger, Drawer, Group, Menu, Tooltip, useComputedColorScheme, useMantineColorScheme } from "@mantine/core"; -import { useDisclosure } from "@mantine/hooks"; -import { GithubLogo, GlobeHemisphereWest, Moon, SignOut, Sun } from "@phosphor-icons/react"; -import { useNavigate, useParams, useRouterState } from "@tanstack/react-router"; -import { useEffect, useRef, useState, type ReactNode } from "react"; -import { createDashboardPrompt, useFanoutApp } from "./app-context"; -import { logout, useViewer } from "./auth"; -import { BrandLockup } from "./brand"; -import Rail, { type RailHandle, type RailProps } from "./rail"; - -export default function Shell({ children }: { children: ReactNode }) { - const { agentAvailable, threadID, threadMissing, newThread, selectThread, openChat } = useFanoutApp(); - const navigate = useNavigate(); - const pathname = useRouterState({ select: (state) => state.location.pathname }); - const { dashboardId } = useParams({ strict: false }) as { dashboardId?: string }; - const isChat = pathname === "/chat" || pathname.startsWith("/chat/"); - const [drawerOpened, drawer] = useDisclosure(false); - const [signOutError, setSignOutError] = useState(""); - const railRef = useRef(null); - - // A navigation from inside the drawer should close it. - useEffect(() => { drawer.close(); }, [pathname]); - - useEffect(() => { - const shortcuts = (event: KeyboardEvent) => { - if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "k") { - event.preventDefault(); - if (railRef.current) railRef.current.focusSearch(); else drawer.open(); - } - }; - window.addEventListener("keydown", shortcuts); - return () => window.removeEventListener("keydown", shortcuts); - }, []); - - const railProps: Omit = { - agentAvailable, - activeThreadID: isChat && !threadMissing && pathname !== "/chat" ? threadID : undefined, - activeDashboardID: dashboardId, - onNewChat: newThread, - onSelectThread: selectThread, - onDeletedThread: (deletedID) => { if (deletedID === threadID) newThread(); }, - onSelectDashboard: (id) => void navigate({ to: "/dashboards/$dashboardId", params: { dashboardId: id } }), - onCreateDashboard: () => openChat(createDashboardPrompt), - }; - - return - - - - - - - - - - - - - - } overlayProps={{ backgroundOpacity: 0.24, blur: 1 }}> - - - - {signOutError && setSignOutError("")}>{signOutError}} - {children} - - ; -} - -function ColorSchemeToggle() { - const { setColorScheme } = useMantineColorScheme(); - // Reading the computed scheme rather than the stored one means the button - // offers the opposite of what is on screen even while the setting is "auto". - const scheme = useComputedColorScheme("light", { getInitialValueInEffect: true }); - const next = scheme === "dark" ? "light" : "dark"; - return - setColorScheme(next)}> - {scheme === "dark" ? : } - - ; -} - -function AccountMenu({ onError }: { onError: (message: string) => void }) { - const viewer = useViewer(); - const initial = (viewer.name || viewer.email || "?").trim().charAt(0).toUpperCase(); - return - - - {initial} - - - - {viewer.email || "Signed in"} - }>Fanout on GitHub - }>LabStack - - } onClick={() => void logout().catch((cause) => onError(cause instanceof Error ? cause.message : "Sign-out failed — your session is still active."))}>Sign out - - ; -} -``` - -- [ ] **Step 3: Rewrite `App.tsx` as the session** - -Replace the whole file with: - -```tsx -import { HttpAgent, type Message } from "@ag-ui/client"; -import { QueryClient, QueryClientProvider, useQueryClient } from "@tanstack/react-query"; -import { Outlet, useNavigate, useParams } from "@tanstack/react-router"; -import { FormEvent, useEffect, useMemo, useRef, useState } from "react"; -import { FanoutAppContext } from "./app-context"; -import AuthGate, { authorizedFetch, useRuntimeStatus } from "./auth"; -import { activityLabel } from "./chat"; -import { createID } from "./id"; -import { threadHistoryQueryKey } from "./rail"; -import Shell from "./shell"; - -function Session() { - const { agent_available: agentAvailable } = useRuntimeStatus(); - const navigate = useNavigate(); - const queryClient = useQueryClient(); - const { threadId: routeThreadID } = useParams({ strict: false }) as { threadId?: string }; - // A draft is a thread the browser has named but the server has not seen. - // Its id is remembered so the route change on first send does not trigger - // a fetch for a thread that does not exist yet. - const [draftID, setDraftID] = useState(createID); - const draftsRef = useRef(new Set()); - const threadID = routeThreadID ?? draftID; - const [messages, setMessages] = useState([]); - const [messageTimes, setMessageTimes] = useState>({}); - const [loadedThreadID, setLoadedThreadID] = useState(""); - const [threadMissing, setThreadMissing] = useState(false); - const [running, setRunning] = useState(false); - const [activity, setActivity] = useState(""); - const [input, setInput] = useState(""); - const [error, setError] = useState(""); - const pendingPromptRef = useRef(""); - const bottomRef = useRef(null); - const inputRef = useRef(null); - const agent = useMemo(() => new HttpAgent({ url: "/api/agent", threadId: threadID, fetch: (url, init) => authorizedFetch(url, init) }), [threadID]); - const ready = !agentAvailable || loadedThreadID === threadID; - - useEffect(() => { - let active = true; - setMessages([]); - setMessageTimes({}); - setLoadedThreadID(""); - setThreadMissing(false); - setRunning(false); - setActivity(""); - setError(""); - if (!agentAvailable) { setLoadedThreadID(threadID); return; } - const isDraft = !routeThreadID || draftsRef.current.has(threadID); - if (isDraft) { - agent.setMessages([]); - setLoadedThreadID(threadID); - } else { - authorizedFetch(`/api/agent/threads/${encodeURIComponent(threadID)}`).then(async (response) => { - if (response.status === 404) return null; - if (!response.ok) throw new Error(`Unable to load thread (${response.status})`); - return response.json() as Promise<{ messages?: Message[] }>; - }).then((thread) => { - if (!active) return; - if (thread === null) { setThreadMissing(true); setLoadedThreadID(threadID); return; } - agent.setMessages(thread.messages ?? []); - setMessages([...(thread.messages ?? [])]); - setLoadedThreadID(threadID); - }).catch(() => { - if (!active) return; - pendingPromptRef.current = ""; - setError("This chat could not be restored. Start a new chat or try again."); - }); - } - const subscription = agent.subscribe({ - onEvent: ({ messages: next }) => setMessages([...next] as Message[]), - onRunInitialized: () => { setRunning(true); setError(""); }, - onToolCallStartEvent: ({ event }) => setActivity(activityLabel(event.toolCallName)), - onToolCallEndEvent: () => setActivity(""), - onRunFinalized: ({ messages: next }) => { - const finished = [...next] as Message[]; - setMessages(finished); - setMessageTimes((times) => { - const stamped = { ...times }; - for (const message of finished) if (message.role === "assistant" && !stamped[message.id]) stamped[message.id] = Date.now(); - return stamped; - }); - setRunning(false); - setActivity(""); - draftsRef.current.delete(threadID); - void queryClient.invalidateQueries({ queryKey: threadHistoryQueryKey }); - }, - onRunFailed: (failure) => { - console.error("Agent run failed", failure); - setError("Fanout could not complete this analysis."); - setRunning(false); - setActivity(""); - void queryClient.invalidateQueries({ queryKey: threadHistoryQueryKey }); - }, - }); - return () => { active = false; subscription.unsubscribe(); agent.abortRun(); }; - }, [agent, agentAvailable, queryClient, routeThreadID, threadID]); - - useEffect(() => { bottomRef.current?.scrollIntoView({ behavior: "smooth", block: "end" }); }, [messages, running]); - - useEffect(() => { - if (!agentAvailable) return; - const shortcuts = (event: KeyboardEvent) => { - const target = event.target as HTMLElement | null; - const editing = target?.matches("input, textarea, [contenteditable='true']"); - if (event.key === "/" && !editing) { - event.preventDefault(); - if (routeThreadID) requestAnimationFrame(() => inputRef.current?.focus()); - else void navigate({ to: "/chat" }).then(() => requestAnimationFrame(() => inputRef.current?.focus())); - } - if (event.key === "Escape" && target === inputRef.current) { setInput(""); inputRef.current?.blur(); } - }; - window.addEventListener("keydown", shortcuts); - return () => window.removeEventListener("keydown", shortcuts); - }, [agentAvailable, navigate, routeThreadID]); - - async function run() { - setRunning(true); - setError(""); - try { await agent.runAgent(); } catch (cause) { - console.error("Agent run failed", cause); - setError("Fanout could not complete this analysis."); - setRunning(false); - setActivity(""); - } - } - - async function send(text: string) { - const content = text.trim(); - if (!agentAvailable || !content || running || !ready || threadMissing) return; - if (!routeThreadID) { - draftsRef.current.add(threadID); - void navigate({ to: "/chat/$threadId", params: { threadId: threadID }, replace: true }); - } - const message = { id: createID(), role: "user", content } as Message; - agent.addMessage(message); - setMessages([...agent.messages]); - setMessageTimes((times) => ({ ...times, [message.id]: Date.now() })); - setInput(""); - await run(); - } - - useEffect(() => { - const prompt = pendingPromptRef.current; - if (!ready || !routeThreadID || !prompt) return; - pendingPromptRef.current = ""; - void send(prompt); - }, [ready, routeThreadID, threadID]); - - function submit(event: FormEvent) { event.preventDefault(); void send(input); } - function stop() { agent.abortRun(); setRunning(false); setActivity(""); } - function retry() { if (!running && agent.messages.some((message) => message.role === "user")) void run(); } - function openChat(prompt?: string) { - if (!agentAvailable) return; - const nextThreadID = createID(); - draftsRef.current.add(nextThreadID); - pendingPromptRef.current = prompt ?? ""; - void navigate({ to: "/chat/$threadId", params: { threadId: nextThreadID } }); - } - function newThread() { - agent.abortRun(); - pendingPromptRef.current = ""; - setDraftID(createID()); - void navigate({ to: "/chat" }); - } - function selectThread(selectedThreadID: string) { - void navigate({ to: "/chat/$threadId", params: { threadId: selectedThreadID } }); - } - - return - - ; -} - -export default function App() { - const queryClient = useMemo(() => new QueryClient(), []); - return ; -} -``` - -- [ ] **Step 4: Create a minimal `chat.tsx` so the session compiles** - -Task 6 fills this file in. For now create `ui/host/src/chat.tsx` with the three pieces the session and the routes need, moved verbatim from the old `App.tsx`: `ChatPage`, `Composer`, `Welcome`, `ChatMessage`, `toolTitle`, plus: - -```tsx -const activityLabels: Record = { - observability_overview: "Checking system health…", - service_topology: "Mapping service dependencies…", - service_performance: "Reading performance signals…", - trace_detail: "Inspecting a trace…", - search_logs: "Searching logs…", - intelligence_snapshot: "Reviewing detected anomalies…", -}; - -export function activityLabel(toolName: string): string { - return activityLabels[toolName] ?? "Working on it…"; -} -``` - -Change every `useFanoutApp` import in the moved code to `import { useFanoutApp } from "./app-context";` and change `ChatPage` to read `threadMissing` and `retry` from the context so the file type-checks against the new context shape (the visuals are redone in Task 6). Keep the old `Composer` markup for now but drop its `pos="fixed"` wrapper, replacing it with `…`, because the footer it was offset against is gone. - -- [ ] **Step 5: Update the routes** - -`routes/chat.index.tsx`: - -```tsx -import { createFileRoute } from "@tanstack/react-router"; -import { ChatPage } from "../chat"; - -export const Route = createFileRoute("/chat/")({ - component: ChatPage, -}); -``` - -`routes/chat.$threadId.tsx`: change the import to `import { ChatPage } from "../chat";`. - -`routes/dashboards.index.tsx` and `routes/dashboards.$dashboardId.tsx`: change `import { useFanoutApp } from "../App";` to `import { useFanoutApp } from "../app-context";`. - -`dashboard.tsx`: replace the local `getJSON`, `Envelope`, `DashboardRecord`, `DashboardSummary` declarations with `import { dashboardsQueryKey, getJSON, type DashboardRecord, type DashboardSummary, type Envelope } from "./api";` and use `dashboardsQueryKey` where `["dashboards"]` appears. (The page is rewritten in Task 11; this keeps it compiling.) - -- [ ] **Step 6: Remove the dead CSS** - -In `index.css` delete the `.chat-composer-field` rules' dependence on a fixed footer: keep the two `.chat-composer-field` rules as they are (they only style border and focus ring). Nothing else changes in this step. - -- [ ] **Step 7: Write the session test** - -Create `app.test.tsx`: - -```tsx -import { MantineProvider } from "@mantine/core"; -import { createRootRoute, createRoute, createRouter, Outlet, RouterProvider } from "@tanstack/react-router"; -import { act } from "react"; -import { createRoot } from "react-dom/client"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -declare global { - interface Window { happyDOM: { setURL(url: string): void } } -} - -const agentMocks = vi.hoisted(() => ({ runAgent: vi.fn(async () => undefined), abortRun: vi.fn(), instances: [] as Array<{ threadId: string }> })); - -vi.mock("@ag-ui/client", () => ({ - HttpAgent: class { - threadId: string; - messages: Array<{ id: string; role: string; content?: string }> = []; - constructor(options: { threadId: string }) { this.threadId = options.threadId; agentMocks.instances.push(this); } - subscribe() { return { unsubscribe: () => undefined }; } - setMessages(messages: Array<{ id: string; role: string; content?: string }>) { this.messages = messages; } - addMessage(message: { id: string; role: string; content?: string }) { this.messages = [...this.messages, message]; } - runAgent = agentMocks.runAgent; - abortRun = agentMocks.abortRun; - }, -})); - -vi.mock("./auth", () => ({ - default: ({ children }: { children: React.ReactNode }) => children, - useRuntimeStatus: () => ({ setup_required: false, auth_mode: "local", agent_available: true, smtp_configured: true, self_signup: false }), - useViewer: () => ({ id: "viewer-1", email: "v@example.com", name: "Vee", role: "admin" }), - authorizedFetch: (input: RequestInfo | URL, init?: RequestInit) => fetch(input, init), - logout: vi.fn(async () => undefined), - clearSession: vi.fn(), -})); - -import App from "./App"; -import { ChatPage } from "./chat"; - -const fetchMock = vi.fn(); - -function json(body: unknown, status = 200) { - return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); -} - -function setValue(input: HTMLTextAreaElement, value: string) { - Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")?.set?.call(input, value); - input.dispatchEvent(new InputEvent("input", { bubbles: true, data: value, inputType: "insertText" })); -} - -describe("Session", () => { - beforeEach(() => { - vi.stubGlobal("fetch", fetchMock); - fetchMock.mockReset(); - agentMocks.runAgent.mockClear(); - agentMocks.instances.length = 0; - fetchMock.mockImplementation(async (input) => { - const url = new URL(String(input), "http://localhost"); - if (url.pathname === "/api/dashboards") return json({ dashboards: [] }); - if (url.pathname === "/api/agent/threads") return json({ threads: [], nextCursor: "" }); - if (url.pathname.startsWith("/api/agent/threads/")) return json({ message: "not found" }, 404); - throw new Error(`unexpected request: ${url.pathname}`); - }); - window.happyDOM.setURL("https://fanout.example.com/chat"); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - document.body.innerHTML = ""; - }); - - it("starts a draft without fetching a thread and names the thread on first send", async () => { - const rootRoute = createRootRoute({ component: App }); - const chatIndex = createRoute({ getParentRoute: () => rootRoute, path: "/chat/", component: ChatPage }); - const chatThread = createRoute({ getParentRoute: () => rootRoute, path: "/chat/$threadId", component: ChatPage }); - const router = createRouter({ routeTree: rootRoute.addChildren([chatIndex, chatThread]) }); - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - await act(async () => root.render()); - await vi.waitFor(() => expect(document.querySelector('textarea[aria-label="Message Fanout"]')).not.toBeNull()); - expect(fetchMock.mock.calls.some(([input]) => String(input).includes("/api/agent/threads/"))).toBe(false); - - const composer = document.querySelector('textarea[aria-label="Message Fanout"]') as HTMLTextAreaElement; - await act(async () => setValue(composer, "Summarize system health")); - await act(async () => composer.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }))); - - await vi.waitFor(() => expect(agentMocks.runAgent).toHaveBeenCalledTimes(1)); - await vi.waitFor(() => expect(window.location.pathname).toMatch(/^\/chat\/[0-9a-f-]{36}$/)); - expect(fetchMock.mock.calls.some(([input]) => String(input).includes("/api/agent/threads/"))).toBe(false); - expect(document.body.textContent).not.toContain("no longer exists"); - - await act(async () => root.unmount()); - }); - - it("explains a thread that no longer exists", async () => { - window.happyDOM.setURL("https://fanout.example.com/chat/deleted-thread"); - const rootRoute = createRootRoute({ component: App }); - const chatThread = createRoute({ getParentRoute: () => rootRoute, path: "/chat/$threadId", component: ChatPage }); - const router = createRouter({ routeTree: rootRoute.addChildren([chatThread]) }); - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - await act(async () => root.render()); - await vi.waitFor(() => expect(document.body.textContent).toContain("This chat no longer exists")); - expect(document.body.textContent).toContain("New chat"); - await act(async () => root.unmount()); - }); -}); -``` - -The `Outlet` import is unused; remove it if `tsc` complains. `ChatPage` in Task 6 renders the "This chat no longer exists" alert; the placeholder from Step 4 must already render that text when `threadMissing` is true, so add to the Step 4 `ChatPage`: `if (threadMissing) return ;` reading `newThread` from the context. - -- [ ] **Step 8: Run tests and lint** - -Run: `cd ui/host && bun run lint && bun run test` -Expected: lint clean; every test file passes (auth, auth-session, dashboard-layout, mcp-app-frame, rail, app). - -- [ ] **Step 9: Build the host and check the shell by hand** - -```bash -cd "$(git rev-parse --show-toplevel)" && just ui-host -``` - -Start the local instance (memory note "local UI screenshot loop") and open `http://127.0.0.1:7520/chat`. Confirm: rail on the left with New chat, Chats, Dashboards, Search; header has burger (below 992px), brand, theme toggle, avatar; no footer; avatar menu shows the email and Sign out; Cmd-K focuses search; at 390px the burger opens the drawer and nothing clips. Then discard the built assets: - -```bash -git checkout -- internal/ui/dist -``` - -- [ ] **Step 10: Commit Tasks 4 and 5 together** - -```bash -git add ui/host/src -git commit -m "feat(ui): replace the header toggle and history drawer with a persistent rail - -The rail is the only navigation: New chat, the chat list with rename and -delete, the dashboard list, and search on Cmd-K. The footer, the Live dot and -the Dashboard/Chat toggle are gone; sign out and the external links live in -the account menu. The session no longer fetches a thread it has just named, -which removes the 404 on every new chat." -``` - ---- - -## Phase C: chat pane and embedded views - -### Task 6: The chat pane - -**Files:** -- Modify: `ui/host/src/chat.tsx` (full rewrite of the Task 5 placeholder), `ui/host/src/index.css` -- Test: `ui/host/src/chat.test.tsx` (new) - -**Interfaces:** -- Produces from `./chat`: `ChatPage()`, `activityLabel(toolName: string): string`, `toolTitle(name: string): string`. `Composer` and `Welcome` stay module-private. -- Consumes: `useFanoutApp` from `./app-context`; `MCPAppFrame` and `MCPAppContent` from `./mcp-app-frame`; `BrandMark` from `./brand`. - -- [ ] **Step 1: Write the failing test** - -Create `chat.test.tsx`: - -```tsx -import { MantineProvider } from "@mantine/core"; -import type { Message } from "@ag-ui/client"; -import { act, createRef, type FormEvent } from "react"; -import { createRoot } from "react-dom/client"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { FanoutAppContext, type FanoutAppContextValue } from "./app-context"; -import { ChatPage } from "./chat"; - -function value(overrides: Partial = {}): FanoutAppContextValue { - return { - agentAvailable: true, threadID: "thread-1", threadMissing: false, messages: [], messageTimes: {}, ready: true, running: false, activity: "", - input: "", setInput: vi.fn(), error: "", bottomRef: createRef(), inputRef: createRef(), - send: vi.fn(async () => undefined), submit: vi.fn((event: FormEvent) => event.preventDefault()), stop: vi.fn(), retry: vi.fn(), - openChat: vi.fn(), newThread: vi.fn(), selectThread: vi.fn(), - ...overrides, - }; -} - -async function mount(context: FanoutAppContextValue) { - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - await act(async () => root.render()); - return root; -} - -function button(text: string) { - return Array.from(document.querySelectorAll("button")).find((item) => item.textContent?.trim() === text); -} - -describe("ChatPage", () => { - afterEach(() => { document.body.innerHTML = ""; }); - - it("offers suggestions on an empty draft and sends the chosen one", async () => { - const context = value(); - const root = await mount(context); - expect(document.body.textContent).toContain("What do you want to know about your system?"); - expect(document.body.textContent).not.toContain("See what changed"); - const chip = button("Find the source of elevated errors"); - expect(chip).not.toBeUndefined(); - await act(async () => chip?.click()); - expect(context.send).toHaveBeenCalledWith("Find the source of elevated errors"); - await act(async () => root.unmount()); - }); - - it("renders markdown tables and code blocks with product chrome", async () => { - const messages: Message[] = [ - { id: "u1", role: "user", content: "Show the slowest endpoints" } as Message, - { id: "a1", role: "assistant", content: "| Endpoint | P95 |\n|---|---|\n| GET /orders | 650ms |\n\n```sql\nSELECT 1\n```" } as Message, - ]; - const root = await mount(value({ messages, messageTimes: { u1: Date.UTC(2026, 8, 5, 18, 16) } })); - const table = document.querySelector(".chat-markdown table"); - expect(table).not.toBeNull(); - expect(table?.closest(".chat-table")).not.toBeNull(); - expect(document.querySelector(".chat-markdown pre code")?.textContent).toContain("SELECT 1"); - expect(document.querySelector('button[aria-label="Copy code"]')).not.toBeNull(); - expect(document.querySelector('button[aria-label="Copy message"]')).not.toBeNull(); - expect(document.querySelector(".mantine-Avatar-root")).toBeNull(); - await act(async () => root.unmount()); - }); - - it("shows the activity line and a stop button while running", async () => { - const context = value({ running: true, activity: "Checking system health…", messages: [{ id: "u1", role: "user", content: "hi" } as Message] }); - const root = await mount(context); - expect(document.body.textContent).toContain("Checking system health…"); - const stop = document.querySelector('button[aria-label="Stop"]') as HTMLButtonElement; - expect(stop).not.toBeNull(); - await act(async () => stop.click()); - expect(context.stop).toHaveBeenCalled(); - await act(async () => root.unmount()); - }); - - it("offers retry after a failed run and a new chat for a missing thread", async () => { - const failed = value({ error: "Fanout could not complete this analysis.", messages: [{ id: "u1", role: "user", content: "hi" } as Message] }); - let root = await mount(failed); - await act(async () => button("Retry")?.click()); - expect(failed.retry).toHaveBeenCalled(); - await act(async () => root.unmount()); - document.body.innerHTML = ""; - - const missing = value({ threadMissing: true }); - root = await mount(missing); - expect(document.body.textContent).toContain("This chat no longer exists"); - await act(async () => button("New chat")?.click()); - expect(missing.newThread).toHaveBeenCalled(); - await act(async () => root.unmount()); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cd ui/host && bun run test src/chat.test.tsx` -Expected: FAIL on the first test (`See what changed` still present) and the markdown test (`.chat-table` missing). - -- [ ] **Step 3: Rewrite `chat.tsx`** - -```tsx -import type { Message } from "@ag-ui/client"; -import { ActionIcon, Alert, Box, Button, Center, Container, Group, Loader, Paper, Stack, Table, Text, Textarea, Title, Tooltip, Typography } from "@mantine/core"; -import { Check, Copy, PaperPlaneTilt, Stop } from "@phosphor-icons/react"; -import { lazy, Suspense, useState, type ComponentProps, type ReactNode } from "react"; -import Markdown, { type Components } from "react-markdown"; -import remarkGfm from "remark-gfm"; -import { useFanoutApp } from "./app-context"; -import { BrandMark } from "./brand"; -import type { MCPAppContent } from "./mcp-app-frame"; - -const MCPAppFrame = lazy(() => import("./mcp-app-frame")); - -const suggestions = [ - "Summarize system health for the last hour", - "Find the source of elevated errors", - "Map the current service dependencies", - "Show the slowest endpoints", -]; - -const activityLabels: Record = { - observability_overview: "Checking system health…", - service_topology: "Mapping service dependencies…", - service_performance: "Reading performance signals…", - trace_detail: "Inspecting a trace…", - search_logs: "Searching logs…", - intelligence_snapshot: "Reviewing detected anomalies…", -}; - -export function activityLabel(toolName: string): string { - return activityLabels[toolName] ?? "Working on it…"; -} - -export function toolTitle(name: string) { - return ({ observability_overview: "System health", service_topology: "Service map", service_performance: "Performance", trace_detail: "Trace analysis", search_logs: "Logs" } as Record)[name] ?? "System analysis"; -} - -export function ChatPage() { - const { agentAvailable, messages, messageTimes, ready, running, activity, error, threadMissing, bottomRef, send, retry, newThread } = useFanoutApp(); - if (!agentAvailable) return Optional capabilityChat is not configuredAdd an AI provider key to enable chat. Telemetry ingest, dashboards, traces, logs, and metrics remain available without it.; - const visibleMessages = messages.filter((message) => message.role !== "tool"); - return - - - {threadMissing && It was deleted, or the link is wrong.} - {!threadMissing && !ready &&
Loading chat
} - {!threadMissing && ready && <> - {visibleMessages.length === 0 && } - - {visibleMessages.map((message) => )} - {running && {activity || "Analyzing your system"}} - {error && {error}} -
- - } - - - {!threadMissing && } - ; -} - -function Composer() { - const { input, setInput, inputRef, submit, send, stop, ready, running } = useFanoutApp(); - return - - - -