Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
ea35baa
docs(plans): add the milestone 4 plan for agent-built dashboards
vishr Oct 9, 2026
253719d
docs(benchmarks): open the milestone 4 criteria audit
vishr Oct 9, 2026
e114b2e
docs(plans): apply the milestone 4 plan review
vishr Oct 9, 2026
ccdc370
test(query,panel): reproduce the ingest-backlog deadlines
vishr Oct 10, 2026
395ec4b
fix(query): let analytical writes in during an ingest backlog
vishr Oct 10, 2026
31c5946
fix(panel,ui): show a panel's full error in its Data view
vishr Oct 10, 2026
b714e3d
test(panel): benchmark raw panel queries on replayed demo data
vishr Oct 10, 2026
d84c628
fix(panel,query): resolve schema-qualified columns in SQL panels by s…
vishr Oct 10, 2026
21af3fb
test(e2e): guard first render and refresh requests of a 12-panel board
vishr Oct 10, 2026
a28edc1
fix(intelligence): keep anomaly findings when a write misses the gate
vishr Oct 10, 2026
0c8dfa2
test(query): fail CI when an engine upgrade changes SQL built-ins
vishr Oct 10, 2026
7639c12
docs(agents): review new SQL built-ins when changing the engine pin
vishr Oct 10, 2026
e01cde7
feat(eval): score answer intents and add an optional quality judge
vishr Oct 10, 2026
9601bfa
docs(site): add the guide to building and refining dashboards
vishr Oct 10, 2026
1fd5433
feat(ui): meet WCAG AA in dashboards and chat panels, and check it in CI
vishr Oct 10, 2026
fed0a91
docs(benchmarks): record idle-machine panel and backlog measurements
vishr Oct 10, 2026
752f453
ci: run the browser performance spec in its own job
vishr Oct 10, 2026
744ec61
fix(ui): keep panel cards' rounded bottom corners
vishr Oct 10, 2026
c95edc0
docs(benchmarks): record the first milestone 4 final verification run
vishr Oct 10, 2026
bbfa48d
fix(query,eval): catch edge rollups up quickly and count successful s…
vishr Oct 10, 2026
2d17645
fix(ui): wrap a long empty-panel diagnosis inside its card
vishr Oct 10, 2026
ae10f2f
test(panel): benchmark metric panels on replayed telemetry
vishr Oct 10, 2026
4d4d5eb
docs(benchmarks): record the milestone 4 rerun, metric panels and the…
vishr Oct 10, 2026
4a6e06f
test(e2e): enforce the calibrated 1.8 s render ceiling in CI
vishr Oct 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,24 @@ jobs:
- name: just test-race
run: just test-race

# The performance spec runs in its own job so neither group nears Playwright's
# 12-minute global timeout; one worker keeps its timings uncontended.
e2e:
name: Browser smoke
name: Browser ${{ matrix.group }}
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- group: smoke and accessibility
slug: smoke
skip_performance: "1"
args: ""
- group: performance
slug: performance
skip_performance: ""
args: e2e/performance.spec.ts
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
Expand All @@ -118,12 +132,14 @@ jobs:
- name: just ui-deps
run: just ui-deps
- name: just e2e
run: just e2e
- name: Upload browser failures
if: failure()
run: just e2e ${{ matrix.args }}
env:
FANOUT_E2E_SKIP_PERFORMANCE: ${{ matrix.skip_performance }}
- name: Upload browser evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: browser-smoke-failure
name: browser-evidence-${{ matrix.slug }}
path: |
ui/host/playwright-report/
ui/host/test-results/
Expand Down
4 changes: 3 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@

- Use the pinned DuckDB 2 engine in `internal/duckdb`, built with
`scripts/with-duckdb.sh`. A Go driver version does not identify its bundled
native engine; verify `SELECT version()` when changing the pin.
native engine; verify `SELECT version()` when changing the pin. A pin change
also fails the built-in function snapshot test: review every added or changed
built-in against the telemetry SQL blocklist before regenerating it.
- Build and test on native Linux or macOS, AMD64 or ARM64. The wrapper verifies
each platform archive's checksum and uses its matching headers and statically
linked core, JSON, Parquet, and time-zone extensions. Do not add dynamic
Expand Down
535 changes: 535 additions & 0 deletions docs/benchmarks/2026-10-agent-dashboards-m4.md

Large diffs are not rendered by default.

1,401 changes: 1,401 additions & 0 deletions docs/superpowers/plans/2026-10-09-agent-dashboards-m4.md

Large diffs are not rendered by default.

130 changes: 130 additions & 0 deletions docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,133 @@ ignored by git and uploaded only on CI failure.

For browser-free verification, run `bunx playwright test --list` in `ui/host`.
`bun run test` runs unit tests and excludes Playwright specs.

## Dashboard performance measurements

Correct rendering and fast rendering need separate evidence. The checked-in
smoke suite verifies populated panels, geometry and bounded fragments; it does
not currently enforce a dashboard latency ceiling. A passing native-engine
correctness test is not a timing measurement either.

For a performance run, use a disposable instance with a recorded telemetry
snapshot and a twelve-panel dashboard over an absolute 24-hour window. Record
the candidate source identity, native engine version, platform, memory cap,
query connections, dataset bounds and row/file/byte counts. Keep cold and warm
samples separate and preserve the dashboard spec and variable values.

Measure at least twenty full-dashboard samples, from navigation or refresh to
the last visible panel being painted with its real result. Include requests,
retries and variable resolution in that duration. A completed HTTP response or
a hidden panel does not establish a completed render. Report nearest-rank p95
against the local target of **1500 ms** for the twelve-panel view. CI runner
timings and any calibrated CI ceiling must be reported separately.

For individual panels, collect at least one hundred warm executions per query
shape against the same snapshot and report p95 against **500 ms**. Cover the
headline totals, grouped latency/error measures, time series and logs actually
used by the board. Retain failures and timeout counts alongside latency; do not
make a slow or failed run disappear by retrying until it passes. These targets
are acceptance criteria, not guarantees for arbitrary telemetry volumes.

The current executor compiles ordinary structured aggregates against raw
telemetry. Its disposable read-cache version 4 stores completed-batch trace
candidates and parts, not general minute aggregates. Service map and health
panels use analytical rollups with their own watermark lag and approximate
latency semantics. Do not assume those rollups answer arbitrary raw counts,
rates or quantiles exactly, or report every dashboard query as cached.

The existing native tests exercise whole-window totals, bounded newest buckets
and width-aware 24-hour bucket selection:

```sh
rtk just test ./internal/panel -run '"TestGaugeTotalsMatchStat|TestBudgetKeepsCompleteNewestBuckets|TestWidthAwareCellBuckets"'
```

Use the pinned engine wrapper through `just test`, with the workspace's offline
Go cache/temp environment configured. Those small fixtures verify semantics and
budgets; they do not replace the representative performance run. Record actual
measurements and limitations in the milestone's `docs/benchmarks/` document.

## Dashboard accessibility verification

Run the browser-free host tests and type checks from `ui/host`:

```sh
rtk bun run test
rtk bun run lint
```

The existing theme, chart-unit and row-formatting tests check semantic text and
mark contrast in both schemes. Keyboard tests cover dashboard focus, chart
navigation, range cancellation, full-screen and shortcut preferences. The exact
palette-order test preserves six categorical colour slots; its name alone does
not prove colour-vision-deficiency separation. There is currently no automated
axe scan or CVD validator in the checked-in smoke suite.

For an accessibility review, inspect dashboard and chat panels in both themes
at both smoke widths. Tab through controls and charts; exercise arrows, Enter,
Shift+Left/Right, Cancel, Escape and full-screen return focus. Check the
single-key shortcut switch, typing/menu/dialog exceptions and error inspection
in Data, including an error longer than its Chart preview. Review populated,
empty, failed, stale and truncated states.

Assess WCAG AA text contrast and chart marks against their actual surfaces.
Check all six ordered categorical colours plus Other under colour-vision
deficiency simulation. Browser accessibility scans cannot read canvas text, so
supplement any scan with compiler/token assertions and screenshots.
CI runs the performance spec (`just e2e e2e/performance.spec.ts`) in its own job and
the rest with `FANOUT_E2E_SKIP_PERFORMANCE=1`. `just e2e` runs the axe-core AA scan of dashboards and chat panels in both
themes; `FANOUT_E2E_SCREENSHOTS=1 just e2e` also captures every panel type in
both themes and states for review under `ui/host/test-results/`. Record the
scanner or simulation method and version, findings and manual review results;
an unrun or inconclusive check is not an accessibility pass.

## Agent dashboard benchmark

The evaluation runner tests scoring and transport offline without a provider:

```sh
rtk proxy bun test scripts/dashboard-eval
rtk proxy bun scripts/dashboard-eval/main.ts --mock --no-output
rtk just dashboard-eval-generate-check
```

The mock exercises ten saves and five consecutive edits with synthetic usage.
It is $0 fixture evidence, not an agent-quality or real-data performance result.
The generated server-response fixtures remain generator-owned.

For a real benchmark, the controller supplies private prompts, exact edit
expectations, a fresh replay manifest, authenticated runtime inputs and one
retained cost ledger. Follow [the runner's input and metering instructions](../scripts/dashboard-eval/README.md).
The benchmark requires ten completed dashboard saves, executed checks for every
saved panel, no final invalid spec, median save latency at most 45 seconds and
five exact edits that preserve unrelated authored fields. Save receipts must
identify the immutable versions being scored; reading the latest dashboard is
not a substitute.

Holdout uses explicit dashboard/answer intent denominators instead of the fixed
ten-save/five-edit benchmark. Optional paired judging needs both judge keys and
shares the authoring ledger; missing keys produce an explicit skip and no quality
pass. Keep sealed inputs and their detailed outputs outside the worktree and
do not use them to tune the candidate. The runner exits **0** for a complete pass,
**1** for a measured failure and **2** for invalid, incomplete or budget-blocked
evidence. Unknown cost stops further paid calls pending settlement.

## Documentation checks

Dashboard workflow prose lives in
`site/src/content/docs/guides/build-dashboards.mdx`. HTTP, MCP and role references
are generated from the server; change their generator sources when necessary,
then regenerate and check:

```sh
rtk just docs-generate
rtk just docs-generate-check
rtk just test ./cmd/fanout-docgen
```

With site dependencies already installed, run `rtk bun run build` from `site/`
to check the sidebar and compile pages, then run the table, social-card and
canonical guards included in that script. `just site-build` also installs site
dependencies; use the installed build directly when dependency installation is
outside the verification environment's scope. Neither command publishes the site.
15 changes: 15 additions & 0 deletions internal/agent/dashboard_eval_golden_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ func TestDashboardEvalGoldensCurrent(t *testing.T) {
t.Fatal(err)
}
evalStableSave(savedOutput)
var savedThread []map[string]any
wire := func(terminal StreamEvent) string {
terminal.Usage = &TokenUsage{InputTokens: 2, OutputTokens: 1}
wireSpec := authored
Expand All @@ -166,6 +167,19 @@ func TestDashboardEvalGoldensCurrent(t *testing.T) {
if err := json.Unmarshal([]byte(messageText(message.Content)), &result); err != nil {
t.Fatalf("tool response: %s: %v", messageText(message.Content), err)
}
persisted := evalObject(t, message)
persisted["id"] = "saved_message"
var content map[string]any
if err := json.Unmarshal([]byte(messageText(message.Content)), &content); err != nil {
t.Fatal(err)
}
evalStableSave(content)
raw, err := json.Marshal(content)
if err != nil {
t.Fatal(err)
}
persisted["content"] = string(raw)
savedThread = []map[string]any{persisted}
}
}
if result.Dashboard.ID == "" {
Expand Down Expand Up @@ -204,6 +218,7 @@ func TestDashboardEvalGoldensCurrent(t *testing.T) {
sort.Strings(mutations)
value["mutating_tools"] = mutations
value["tool_errors"] = evalToolErrors(t)
value["saved_thread"] = savedThread
raw, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatal(err)
Expand Down
8 changes: 5 additions & 3 deletions internal/cmd/e2eseed/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ func run(endpoint, tokenFile string) error {
if strings.TrimSpace(string(token)) == "" {
return errors.New("empty token file")
}
data, err := payloads(13, time.Now().UTC())
base := time.Now().UTC()
data, err := payloads(13, base)
if err != nil {
return err
}
Expand All @@ -74,6 +75,7 @@ func run(endpoint, tokenFile string) error {
return fmt.Errorf("read %s response failed", signal)
}
}
fmt.Printf("seed_window=%s,%s\n", base.Add(-24*time.Hour).Format(time.RFC3339Nano), base.Format(time.RFC3339Nano))
return nil
}

Expand All @@ -93,8 +95,8 @@ func payloads(seed int64, base time.Time) (map[string][]byte, error) {
logRequest := &collectorlogs.ExportLogsServiceRequest{}
services := []string{"frontend", "checkout", "payment", "inventory", "shipping", "notifications"}
// One complete distributed trace each minute. Versions split exactly one hour ago.
for minute := range 120 {
at := base.Add(-2*time.Hour + time.Duration(minute)*time.Minute)
for minute := range 1440 {
at := base.Add(-24*time.Hour + time.Duration(minute)*time.Minute)
traceID := identifier(uint64(minute+1), 16)
for i, service := range services {
version := "1.0.0"
Expand Down
Loading
Loading