From d22418e143c9eb6d4ba44b5675ed8d8f4ced1933 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 10 Oct 2026 08:02:24 -0700 Subject: [PATCH 01/37] =?UTF-8?q?docs(dashboards):=20milestone=205=20desig?= =?UTF-8?q?n=20=E2=80=94=20correct=20metrics,=20per-minute=20summaries,=20?= =?UTF-8?q?system=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-10-10-agent-dashboards-m5-design.md | 370 ++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md diff --git a/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md b/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md new file mode 100644 index 000000000..93d775d5d --- /dev/null +++ b/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md @@ -0,0 +1,370 @@ +# Milestone 5: correct metrics, per-minute summaries and the system page + +Status: draft for review · 2026-10-10 · branch `feat/agent-dashboards-m5` + +This extends `2026-10-04-agent-dashboards-design.md`. Milestones 1 to 4 are +merged. Its success criteria S1 to S13 stay in force; this milestone adds S14 +to S22. + +## Goal + +Fanout shows what is happening in a whole system, not only in its traced +services, and its numbers are right at a realistic scale. + +A system is everything whose behaviour decides whether the software works: +the services, what they depend on, where they run, how people reach them, and +what changes. Everything arrives over OTLP through the OpenTelemetry +Collector, using its standard receivers (`hostmetrics`, `docker_stats`, +`httpcheck`) and semantic conventions. Fanout adds no per-source integrations. +It recognises the standard names and presents them. + +The front page answers, in this order: + +1. Is it reachable? +2. Is anything failing or slow? +3. Is anything running out: disk, memory, CPU? +4. What changed? + +Then the user drills into traces and logs, or asks the agent. + +## Why now + +Three facts from milestone 4 decide the scope. + +1. **Metric measures are wrong for counters and histograms.** `rate()` counts + data points per second, so for a counter it reports how often the Collector + exports, not traffic. A histogram's `p95` is taken over each data point's + stored value, not its buckets, so it is not the 95th percentile of the + observations. The front page's CPU, network and disk panels are counters, + so it cannot be built on today's measures. +2. **Ingest discards metric meaning.** It stores a cumulative sum as `sum` + whether it is monotonic or not, so `process.memory.usage` (an up-down + counter) and `system.network.io` (a counter) are indistinguishable. The + demo data holds 213 sum metrics of both kinds. Exponential histograms are + converted with one bound per count instead of one fewer, so every metric + histogram query silently drops them. +3. **The raw path stops being fast enough.** The replayed demo carries about + 15 spans per second. At 10x that volume a twelve-panel request takes 1.1 to + 1.6 s on the server, at or past S6's 1.5 s budget for the whole render. + Milestone 3 removed the per-minute summaries. A system worth watching runs + at 1,000 to 5,000 spans per second. + +## Success criteria + +| ID | Criterion | Target | Evidence | +| --- | --- | --- | --- | +| S14 | Counter rate | `rate()` on a counter equals the per-second increase across resets, export-interval changes, delta temporality and many series | engine-executed golden tests; demo-data cross-check against an independent window query | +| S15 | Histogram percentiles | `pNN`/`quantile` on a metric histogram equals linear interpolation over the summed bucket increases | engine-executed golden tests with known distributions | +| S16 | Metric fidelity | every OTLP metric kind keeps its kind, temporality and monotonicity; exponential histograms keep zero and negative counts | ingest round-trip tests | +| S17 | Route-invariant results | every measure the summaries serve returns identical values from summaries and from raw rows; span quantiles are within 1% relative error of the exact rank value (`quantile_disc`) | equality tests on the pinned engine; demo-data comparison | +| S18 | Scale | at 1,000 spans/s for 24 h with host and container metrics from 20 hosts: S7 (≤ 500 ms p95 per panel) for every routed shape, a twelve-panel request ≤ 1.0 s on the server, S6 (≤ 1.5 s first render) in the browser | Go benchmark and Playwright on generated data | +| S19 | Ingest headroom | sustained ingest at 2x the S18 rate with summaries on; summary upkeep adds ≤ 10% CPU | ingest benchmark | +| S20 | The system page shows real trouble | through one Collector, the page shows a filesystem above 90%, a failing external dependency, an unreachable endpoint and a version change within two refreshes | Playwright on a fixture stack; one local run with a real Collector | +| S21 | System page speed | first render ≤ 1.5 s p95 at the S18 volume | Playwright performance test | +| S22 | The agent uses metrics correctly | on host-metric prompts, counters use `rate()`, up-down counters and gauges use `last`/`avg`, histograms use percentiles; S1–S5 still hold | eval harness (public prompts; the sealed holdout is untouched) | + +Paid model spend for the milestone stays within $5, all of it in the final +verification. + +## What exists today + +- `metric_type` is `gauge`, `sum`, `sum_delta`, `histogram`, + `exp_histogram` or `summary`. Start time and monotonicity are dropped at + ingest. Histogram panels take a `temporality` option because the data does + not say. +- Metric histogram panels (`heatmap`, `histogram`) already group by series + identity and take last-minus-first per bound per period, clamped at zero. + A counter reset contributes 0 for that period. +- The read caches are the trace caches only (`readCacheVersion` 4). The + analytical `service_rollup` and `edge_rollup` tables refresh behind a + watermark. `service_rollup` holds per-minute, per-service span counts, error + rate, p50 and p95 (not mergeable across minutes), and log and metric counts. + Health, overview, dependencies, latency, the alert engine and SQL panels + read it. +- `/` redirects to chat or dashboards. Nothing presents host, container or + reachability data, although the demo data holds `hostmetrics` and + `docker_stats` output for one host and 24 containers. + +## Approaches considered + +**Metric meaning: new columns, or new type names in the existing column.** +Renaming values inside `metric_type` keeps the physical schema, but old format +3 files would carry `sum` with a different meaning, which is silent +misreading. New columns change the physical schema, so the batch format moves +to 4 and format 3 files are refused, which is explicit. The product is +pre-release and the deployed instances already need a wipe, so the cost is +local data only. **Chosen: format 4 with new columns.** + +**Quantiles under summaries: exact raw, approximate summaries, or one sketch +on both paths.** Exact `quantile_cont` on raw rows with sketches in summaries +makes a panel's value depend on which path served it. One logarithmic sketch +used by both paths makes results identical by construction and testable by +equality, at a stated 1% relative error. Prometheus histogram quantiles are far +coarser, and DDSketch-based products use the same bound. **Chosen: one sketch, +both paths.** + +**Where summaries live: per batch, or global per minute.** A global table +needs rewrites when late or compacted data arrives. Per-batch partial +summaries follow the existing trace-cache rules: they are written with the +batch's cache marker, compaction derives output contributions from cached +inputs, and retention drops them with the batch. Uncached batches and partial +edge minutes are read raw. **Chosen: per batch.** + +**Metric summaries.** At the Collector's default 60 s interval each series has +one point per minute, so a per-minute metric summary saves nothing, and a +series' increase spans batches. **Chosen: no metric value summaries.** The +benchmark decides whether metric panels meet S7 raw; if they do not, the fix +is designed then, from numbers. + +**The system page: a fixed React page, or a generated dashboard.** A fixed +page would be a second renderer. A dashboard spec generated on the server and +run through the panel executor reuses everything, including S8's single batch +request. **Chosen: generated dashboard, read-only.** + +## Phase A: metric fidelity and series-aware measures + +### Storage (batch format 4) + +The metrics Parquet schema gains: + +| Column | Type | Meaning | +| --- | --- | --- | +| `temporality` | dict string | `cumulative`, `delta`, or empty for gauges and summaries | +| `monotonic` | boolean | true for monotonic sums; false otherwise | +| `start_time_unix_nano` | int64 | the data point's start time; 0 when absent | + +`metric_type` becomes the OTLP data kind only: `gauge`, `sum`, `histogram`, +`exponential_histogram`, `summary`. `sum_delta` is removed. Format 4 is the +only accepted format; format 3 files are refused like any unsupported format, +with no reader or conversion. + +Exponential histograms convert to explicit bounds correctly: the zero bucket +and the negative buckets become the lowest buckets, and counts have exactly +one more entry than bounds. The conversion keeps every observation's count. + +### The metrics view + +The view's `type` column becomes the semantic kind: `gauge`, `counter` +(monotonic sum), `updown` (non-monotonic sum), `histogram` (both explicit and +exponential) or `summary`. It adds `temporality` and `start_time`. +`get_telemetry_schema` lists each metric's kind and temporality, so the agent +chooses measures from facts. + +### Series and increases + +A series is one combination of namespace, service, name, unit, scope name, +scope version, attributes and resource, as metric histogram panels already +use. Metric measures work in two stages, the way PromQL's +`sum by (x) (rate(m[1m]))` does: first per series and bucket, then across the +series in each group. + +The increase of a cumulative series between consecutive points is +`current - previous`, or `current` when the value decreased or the start time +changed (a reset). Each increase belongs to the bucket of its later point. The +query reads one lookback period before the window, 10 minutes, so the first +bucket has its previous point. A delta series' increase is its value. A series +first seen inside the window contributes from its second point. Histograms +apply the same rule per bound, with a reset detected on the total count. + +### Measures on metrics + +| Measure | gauge, updown | counter | histogram | +| --- | --- | --- | --- | +| `count()` | data points | data points | data points | +| `rate()` | not defined | Σ series increase ÷ bucket seconds | Σ count increase ÷ bucket seconds | +| `sum(value)` | Σ series bucket average | Σ series increase | Σ sum increase | +| `avg(value)` | mean of series bucket averages | not defined | Σ sum increase ÷ Σ count increase | +| `min`/`max(value)` | over data points | over data points | not defined | +| `last(value)` | Σ series latest value | Σ series latest value | not defined | +| `pNN`/`quantile(value, q)` | over data points | not defined | linear interpolation over Σ bound increases | + +A measure that is not defined for the series' kind returns no value, and the +empty-panel diagnosis names the kind and the measures that fit, for example +"rate() needs a counter; system.memory.usage is an updown; use last(value) or +avg(value)". OTLP summary metrics have no aggregate measures, because their +stored quantiles cannot be combined. + +Validation adds one rule: an aggregate metric query other than `count()` must +fix the metric per group, with a `name = ...` filter on one value or `name` +among `by`. Different metrics never sum into one number. + +Histogram quantile interpolation follows Prometheus: the lowest bucket's lower +bound is 0 when its upper bound is positive, and an observation in the +overflow bucket returns the highest finite bound. + +Histogram and heatmap panels use the same increase rule. Their `temporality` +option is removed, because the data now carries it. + +### Phase A changes outside the panel compiler + +- The agent's spec guide and examples teach metric kinds and the measure + table, including one counter-rate and one histogram-percentile example. +- The dashboards guide documents metric kinds and measures. +- The benchmark replay infers monotonicity for the stored demo data from + OpenTelemetry's semantic conventions, because that source lost it. + +## Phase B: the scale benchmark + +Measure before building. A generator writes realistic telemetry through OTLP +ingest: + +- spans and logs at 1,000 spans per second for 24 hours, built by tiling the + demo replay across namespaces and hosts, keeping its shape of services, + operations, routes and errors; +- host and container metrics for 20 hosts with 24 containers each, at the + Collector's 10 s and 60 s intervals, using the demo's `hostmetrics` and + `docker_stats` names; +- a 6-hour informational run at 5,000 spans per second. + +The run checks free disk first and refuses to start below twice its estimated +output. It records, for every M4 benchmark shape plus the new metric shapes: +raw-path p95 per panel, the twelve-panel request time, ingest throughput, and, +for Phase C sizing, the number of distinct summary keys per minute for each +candidate dimension set. + +Results go into `docs/benchmarks/2026-10-agent-dashboards-m5.md`. + +## Phase C: per-minute summaries + +### What is summarised + +Per batch and per minute, three private read tables: + +- **Spans.** Keys: namespace, minute, and the dimension set below. Values: + span count, error count, served-span count, duration sum, minimum and + maximum, and a duration sketch. +- **Logs.** Keys: namespace, minute, service, severity, severity number. + Value: count. +- **Metric counts.** Keys: namespace, minute, service. Value: data points. + +The span dimension set starts as service, operation, kind, status, +`http_method`, `http_status_code`, `http_route`, `rpc_service`, `rpc_method`, +`db_system`, `messaging_system`, `peer_service`, `service_version` and +`deployment_env`. Phase B's key counts confirm it. The rule: summary rows per +minute stay at or below one twentieth of spans per minute at the S18 rate. A +dimension that breaks the rule is dropped, and panels that use it read raw. + +### The sketch + +Durations are bucketed logarithmically with relative accuracy α = 0.01: +γ = (1 + α) / (1 − α), and a positive duration `x` falls in bucket +`ceil(log_γ(x))`. Zero and negative durations share one bucket valued 0; NaN +is excluded, as today. A quantile walks the summed bucket counts and returns +the bucket's representative value, `2γ^i / (γ + 1)`. Raw-path span quantiles +use exactly this computation, so both paths return the same number. + +### Routing + +The compiler serves a query from summaries when all of these hold: + +- the signal is spans or logs, and the panel is an aggregate visualization + (stat, gauge, timeseries, bar or table); +- every `by` field and every column referenced by a filter is a summary key; + attribute lookups always read raw; +- every measure is `count`, `rate`, `error_rate`, `share`, or `avg`, `sum`, + `min`, `max`, `pNN` or `quantile` of `duration_ms`; +- the bucket is a whole number of minutes. + +A routed query reads summaries for whole minutes of cached batches, and raw +rows for uncached batches and for the partial minutes at each end of the +window. The read happens in one transaction under the pinned file snapshot, +as the trace caches do. Panel results report which path served them, for the +Inspect view and for tests. + +### Lifecycle + +Summaries follow the read-cache rules in `AGENTS.md`. They are written with +the batch's cache marker in one transaction. Compaction derives the output +batch's summaries from its inputs' summaries. Retention drops them with the +batch. `readCacheVersion` moves to 5, and a version mismatch rebuilds every +private read table. + +### `service_rollup` retires as a table + +Two per-minute span aggregates would be redundant. `service_rollup` and its +refresh loop are removed. Health, overview, dependencies, latency and the +alert engine read the span, log and metric-count summaries through one Go +reader. The public SQL relation `service_rollup` stays as a view with the same +columns, computed from the summary tables; p50 and p95 come from the merged +sketch. `edge_rollup` stays, because caller and callee pairs join spans across +batches. + +## Phase D: the system page + +`/` opens the system page, and the rail gains a System entry. The page is a +dashboard spec generated on the server from what telemetry exists in the +selected range and namespace, run through the panel executor in one batch +request, and rendered by the dashboard renderer. It is read-only. A section +whose data is absent is replaced by one text panel naming the Collector +receiver that supplies it, linked to the setup guide. + +Sections, in the page's question order: + +1. **Reachable.** From `httpcheck` metrics: a state timeline per endpoint and + a latency timeseries. +2. **Failing or slow.** The health panel and the service map, which already + show external dependencies from client spans. +3. **Running out.** Two tables, worst first, with warning at 80% and bad at + 90%. Hosts: CPU busy from `system.cpu.time` rates by state, memory used + from `system.memory.usage` by state, and the fullest filesystem from + `system.filesystem.usage` by mountpoint. Containers: CPU from + `container.cpu.utilization`, memory as `container.memory.usage.total` over + `container.memory.usage.limit`. These are SQL panels written by the server + over the metrics view, using the Phase A increase and latest-value rules. +4. **What changed.** Deploy markers on the health trend, and a versions table: + service, `service.version` and when it was first seen, runtime name and + version, and SDK language and version, from resource attributes. + +The documentation gains a guide to watching a system: a Collector +configuration with `hostmetrics`, `docker_stats`, `httpcheck` and OTLP +forwarding, and what each section needs. + +### The self-observation test + +The page is right if it would have shown the problems a real stack had. S20's +fixture stack replays the demo plus four scenarios through OTLP: a filesystem +filling past 90%, an external dependency returning errors, an `httpcheck` +target going down, and a new `service.version`. Playwright checks that each +appears in its section within two refreshes. Once, on a development machine, a +real Collector observing real local containers confirms the same page. + +## Phase E: final verification + +S1 to S22 run on the final commit: the eval harness (within $5, with new +public host-metric prompts for S22), the browser collector, the Playwright +performance and accessibility tests, the Go benchmarks at the S18 volume, and +the equality tests. Results complete the M5 benchmark document. + +## Testing + +- Every SQL-producing change is tested by executing on the pinned engine, + not by comparing SQL text. +- Counter, histogram and sketch logic have golden tests with known answers: + resets, interval changes, delta series, late and out-of-order points, + series first seen inside a window, empty buckets and overflow buckets. +- Route equality: each routed shape runs both ways on the same snapshot and + must return identical frames. +- Lifecycle: compaction, retention and late batches never double count or + lose a summary contribution. +- The fixture-stack e2e test runs in CI. The real-Collector run is recorded + in the benchmark document. + +## Out of scope + +Alert rules created from system-page sections, incidents and notifications, +outside reachability checks run by Fanout itself (the Collector's `httpcheck` +does this), cost and usage tracking, vulnerability matching for runtime +versions, and summaries for metric values. Each is designed separately if the +page shows it is needed. + +## Risks + +- **Summary cardinality.** A route-rich service can produce many keys per + minute. Phase B measures it, and the one-twentieth rule drops dimensions + rather than letting summaries grow. +- **Lookback cost.** Reading 10 minutes before the window adds work to every + counter query. The benchmark includes counter shapes at 10 s intervals. +- **Changed numbers.** Span quantiles move from exact to within 1%. The + dashboards guide states it, and Inspect shows the path. +- **Format break.** Format 4 refuses existing local data. Deployed instances + are already on a release whose data needs a wipe before upgrading. From fb6e11937796f64d1949d6f08af0e6a27b852abf Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 10 Oct 2026 08:04:02 -0700 Subject: [PATCH 02/37] docs(dashboards): M5 counter rate uses elapsed time; All-able name filters do not fix the metric --- .../specs/2026-10-10-agent-dashboards-m5-design.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md b/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md index 93d775d5d..bba0e159b 100644 --- a/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md +++ b/docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md @@ -171,13 +171,21 @@ apply the same rule per bound, with a reset detected on the total count. | Measure | gauge, updown | counter | histogram | | --- | --- | --- | --- | | `count()` | data points | data points | data points | -| `rate()` | not defined | Σ series increase ÷ bucket seconds | Σ count increase ÷ bucket seconds | +| `rate()` | not defined | Σ series (increase ÷ elapsed) | Σ series (count increase ÷ elapsed) | | `sum(value)` | Σ series bucket average | Σ series increase | Σ sum increase | | `avg(value)` | mean of series bucket averages | not defined | Σ sum increase ÷ Σ count increase | | `min`/`max(value)` | over data points | over data points | not defined | | `last(value)` | Σ series latest value | Σ series latest value | not defined | | `pNN`/`quantile(value, q)` | over data points | not defined | linear interpolation over Σ bound increases | +A series' rate in a bucket is its increase divided by the time those +increases cover: the sum of `time - previous time` over the deltas whose later +point falls in the bucket. A bucket narrower than the export interval would +otherwise multiply the rate, so a counter exported every 60 s reads the same +per-second rate in 10 s and 1 h buckets. A series with no point in a bucket +contributes nothing to that bucket, and the automatic bucket for metric +panels is never shorter than 1 minute. + A measure that is not defined for the series' kind returns no value, and the empty-panel diagnosis names the kind and the measures that fit, for example "rate() needs a counter; system.memory.usage is an updown; use last(value) or @@ -186,7 +194,9 @@ stored quantiles cannot be combined. Validation adds one rule: an aggregate metric query other than `count()` must fix the metric per group, with a `name = ...` filter on one value or `name` -among `by`. Different metrics never sum into one number. +among `by`. A `name = $var` filter counts only when the variable is +single-valued and has no All option, because All drops the filter. Different +metrics never sum into one number. Histogram quantile interpolation follows Prometheus: the lowest bucket's lower bound is 0 when its upper bound is positive, and an observation in the From a0b8c6be44266ad0782f35e7cc64717d7af1f2c4 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 10 Oct 2026 08:27:03 -0700 Subject: [PATCH 03/37] docs(dashboards): M5 Phase A plan draft (metric fidelity) --- .../plans/2026-10-10-agent-dashboards-m5a.md | 1571 +++++++++++++++++ 1 file changed, 1571 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-10-agent-dashboards-m5a.md diff --git a/docs/superpowers/plans/2026-10-10-agent-dashboards-m5a.md b/docs/superpowers/plans/2026-10-10-agent-dashboards-m5a.md new file mode 100644 index 000000000..69f145ff5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-10-agent-dashboards-m5a.md @@ -0,0 +1,1571 @@ +# Agent dashboards milestone 5, Phase A: metric fidelity Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Establish S14–S16: preserve OTLP metric meaning in format 4, compute increases and elapsed-time rates per series, and derive histogram percentiles from observation weights. +**Architecture:** Extend the canonical metric row, physical Parquet schema and public metrics view together. One private SQL builder supplies deduplicated, ordered series points and increases to structured aggregates and distribution panels. Aggregate within each series before combining series; retain the existing executor, frame contracts and snapshot binder. +**Tech Stack:** Go 1.27.2, the pinned DuckDB 2 native engine through `bash scripts/with-duckdb.sh`, parquet-go, OTLP protobufs, database/sql, existing querytest fixtures, React/TypeScript, Bun/Vitest, and the existing documentation generator. No new dependencies. +**Spec:** docs/superpowers/specs/2026-10-10-agent-dashboards-m5-design.md + +## Global Constraints + +- This is a plan, not implementation or execution evidence. The planning dispatch writes only this file. No git writes, network, browser tools, CodeGraph initialization, questions or extra reports. The seven implementation tasks below describe future edits. The controller reviews and commits; commit steps contain messages only. +- The M5 spec and the Phase A brief explicitly supersede AGENTS.md's format-3-only sentence: Task 1 changes the sole supported format to 4 and updates that sentence. No format-3 reader, schema union, migration, conversion in Fanout, alias or fallback. Reject unsupported active and retired batches before cleanup/schema-batch replacement, preserve their bytes, and never classify unsupported formats or operational failures as corruption. +- Preserve typed VARIANT attributes/resource, literal dotted keys, signed nanoseconds, UTC on every connection, native `TIMESTAMPTZ_NS` predicates, `?::TIMESTAMP_NS::TIMESTAMPTZ_NS` window binds, footer-derived event-time bounds, native Parquet binding and `hive_partitioning=false`. Do not change the engine pin or its I/O defaults. +- No control-schema changes, telemetry value summaries, new caches, summary routing, DDSketch, system page, benchmark generator, paid eval or Phase B–E work. Keep SQLite/modernc, Goose/sqlc and existing read-cache version/semantics unchanged. Existing lifecycle and S1–S13 guards remain required. +- Commands below run from the repository root and use RTK. Before future Go commands, set this environment; the planning dispatch does not run Go or create caches: + + ```bash + rtk mkdir -p .superpowers/gocache .superpowers/gotmp + export GOCACHE="$PWD/.superpowers/gocache" + export GOTMPDIR="$PWD/.superpowers/gotmp" + export GOFLAGS=-mod=readonly + export GOPROXY=off + ``` + + The verified justfile uses `bash scripts/with-duckdb.sh go test`. Targeted commands below use that exact wrapper. **Every task ends with `rtk just check`.** The controller supplies installed dependencies and runs any gate recipe that needs network outside the restricted dispatch; a blocked or skipped gate is not a pass. Do not inherit M4's instruction to omit the aggregate gate. +- All modified/new files use repository-relative paths, snake_case product JSON, plain-English Problems with hints, and no machine names, IPs, credentials or absolute local paths. New file paths below are proposals in existing verified directories; existing modify/test paths were read or listed in this checkout. +- SQL behavior tests execute the actual pinned engine through querytest tables and `Executor.Run`; snapshot tests additionally use the existing Parquet repository helper. No generated-SQL equality assertion is evidence of correctness. Existing span/log compiler tests continue to run. +- Decisions resolving prose ambiguities: (1) cumulative first points are baselines; delta first points contribute their value to sums and get elapsed time only from a usable non-null start time; subsequent deltas use a valid start time when supplied, otherwise the preceding point; (2) elapsed must be positive for rate, and points without elapsed remain eligible for sums; (3) finite scalar values are selected before lagging, so invalid values never poison a baseline; (4) histogram reset applies once to all channels on count decrease, bounds change or two non-null differing start times; a hist_sum decrease alone is not a reset because negative observations are possible; (5) unspecified/unknown OTLP aggregation temporality on sums/histograms is rejected as InvalidArgument, rather than guessed; (6) summary permits point-count operations (`count`, `count_distinct`, `share`), but all value aggregates return NULL; (7) the brief's count_distinct exception to the name rule is binding, while share requires a fixed name or name grouping. +- Undefined kind/measure contributions stay NULL, including all-summary or all-incompatible groups. A real zero increase stays zero. Keep NULLs in frames; when every measure cell is NULL, report empty and run bounded kind diagnosis. Never convert an undefined value to zero or retain an old series in buckets with no point. + +## Review Focus + +1. A counter exported every 60 s viewed with 10 s buckets reads the same per-second rate as with 1 m buckets: Task 4, `TestMetricRateUsesCoveredElapsedTime`. +2. A container restart creates a new series mid-window: no spike from its first point, and the old series stops contributing: Task 3, `TestMetricIncreaseOrdersDeduplicatesAndResets`; Task 4, `TestMetricRestartDoesNotCarryOldSeries`. +3. Two points of one series at the same timestamp (exporter retry) do not create a reset or double the increase: Task 3, `TestMetricIncreaseOrdersDeduplicatesAndResets`. +4. A histogram whose bounds change mid-window (Collector reconfigured) treats the change as a reset instead of subtracting mismatched buckets: Task 3, `TestHistogramIncreaseResetsAllChannels`; Task 5, `TestHistogramPanelsShareResetIncreases`. +5. A dashboard variable for the metric name that has an All option does not satisfy the name rule; selecting All never sums different metrics: Task 4, `TestMetricNameRuleUsesCheckedEqualities` and `TestMetricNameAllGroupsDifferentMetricsSeparately`. + +## Task 1: Format 4 metric columns and ingest kinds + +**Files** + +- Modify: `AGENTS.md`; `internal/telemetry/rows.go`, `internal/telemetry/parquet_rows.go`, `internal/telemetry/parquet.go`, `internal/telemetry/verify.go`; `internal/ingest/server.go`; `internal/query/querytest/tables.go`, `internal/query/views.go`; `internal/panel/testhelpers_test.go`, `internal/panel/panel_bench_data_test.go`, `internal/panel/panel_bench_test.go`; `site/src/content/docs/reference/data-layout.mdx`. +- Create/test: `internal/ingest/metric_fidelity_test.go`, `internal/telemetry/metric_format_test.go`, `internal/panel/metric_view_test.go`. +- Existing regression tests: `internal/telemetry/parquet_test.go`, `internal/telemetry/schema_batch_test.go`, `internal/telemetry/verify_test.go`, `internal/telemetry/store/repair_test.go`, `internal/query/views_schema_test.go`, `internal/query/rollup_test.go`, `internal/panel/distribution_test.go`. Update metric fixtures with accurate stored kind/temporality/monotonicity; do not rename visualization `histogram` to an OTLP kind. + +**Interfaces** + +Consumes (existing): + +```go +func NewServer(cfg config.Config, submitter batchSubmitter) *Server +func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.ExportMetricsServiceRequest) (*collectormetrics.ExportMetricsServiceResponse, error) +func makeMetricParquetRow(r Metric) metricParquetRow +func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error +func OpenParquetStore(dir string) (*ParquetStore, error) +func ValidatePublishedBatch(dir string) (err error) +func CreateTables(db *sql.DB, createCaches func(*sql.DB) error) error // querytest +func CreateViews(db *sql.DB) error // query +func NewExecutor(engine Engine, retentionDays int) *Executor +func (e *Executor) Run(ctx context.Context, req RunRequest) ([]Result, error) +``` + +Produces: `telemetry.Metric.Temporality string`, `Monotonic bool`, `StartTimeUnixNanos int64`; matching Parquet fields; `batchMetadataVersion = 4`; metrics `type` in gauge/counter/updown/histogram/summary, `temporality VARCHAR`, nullable `start_time TIMESTAMPTZ_NS`. New ingest helper `func metricTemporality(t metricspb.AggregationTemporality) (string, error)`. New test-only shared helpers (fully defined below): `m5Fixture`, `m5Insert`, `m5Run`, `m5Values`, `m5Counter`. + +- [ ] **Write the failing ingest fidelity test.** In package ingest, import testing, telemetry/config, collectormetrics and metricspb under the same aliases as server.go. Capture through existing `newTestSubmitter`; this is not a fake SQL engine. + +```go +func TestMetricIngestKeepsKindTemporalityAndStart(t *testing.T) { + at, start := uint64(1790856000000000017), uint64(1790855940000000009) + point := func() *metricspb.NumberDataPoint { + return &metricspb.NumberDataPoint{TimeUnixNano: at, StartTimeUnixNano: start, + Value: &metricspb.NumberDataPoint_AsInt{AsInt: 12}} + } + metrics := []*metricspb.Metric{ + {Name: "g", Data: &metricspb.Metric_Gauge{Gauge: &metricspb.Gauge{DataPoints: []*metricspb.NumberDataPoint{point()}}}}, + {Name: "c", Data: &metricspb.Metric_Sum{Sum: &metricspb.Sum{IsMonotonic: true, AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, DataPoints: []*metricspb.NumberDataPoint{point()}}}}, + {Name: "d", Data: &metricspb.Metric_Sum{Sum: &metricspb.Sum{IsMonotonic: true, AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_DELTA, DataPoints: []*metricspb.NumberDataPoint{point()}}}}, + {Name: "u", Data: &metricspb.Metric_Sum{Sum: &metricspb.Sum{AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, DataPoints: []*metricspb.NumberDataPoint{point()}}}}, + {Name: "h", Data: &metricspb.Metric_Histogram{Histogram: &metricspb.Histogram{AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_DELTA, DataPoints: []*metricspb.HistogramDataPoint{{TimeUnixNano: at, StartTimeUnixNano: start, Count: 2, ExplicitBounds: []float64{1}, BucketCounts: []uint64{1, 1}}}}}}, + {Name: "e", Data: &metricspb.Metric_ExponentialHistogram{ExponentialHistogram: &metricspb.ExponentialHistogram{AggregationTemporality: metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, DataPoints: []*metricspb.ExponentialHistogramDataPoint{{TimeUnixNano: at, StartTimeUnixNano: start, Count: 1, Positive: &metricspb.ExponentialHistogramDataPoint_Buckets{BucketCounts: []uint64{1}}}}}}}, + {Name: "s", Data: &metricspb.Metric_Summary{Summary: &metricspb.Summary{DataPoints: []*metricspb.SummaryDataPoint{{TimeUnixNano: at, StartTimeUnixNano: start, Count: 1}}}}}, + } + ch := make(chan telemetry.Metric, len(metrics)) + srv := NewServer(config.Config{DefaultNamespace: "test"}, newTestSubmitter(nil, nil, ch)) + _, err := srv.exportMetrics(t.Context(), &collectormetrics.ExportMetricsServiceRequest{ + ResourceMetrics: []*metricspb.ResourceMetrics{{ScopeMetrics: []*metricspb.ScopeMetrics{{Metrics: metrics}}}}, + }) + if err != nil { t.Fatal(err) } + kinds := []string{"gauge", "sum", "sum", "sum", "histogram", "exponential_histogram", "summary"} + temps := []string{"", "cumulative", "delta", "cumulative", "delta", "cumulative", ""} + for i := range metrics { + got := <-ch + if got.Type != kinds[i] || got.Temporality != temps[i] || got.Monotonic != (i == 1 || i == 2) || got.StartTimeUnixNanos != int64(start) { + t.Fatalf("point %d: %+v", i, got) + } + } +} +``` + +- [ ] **Add format rejection tests, including byte preservation.** Package telemetry imports encoding/json, errors, os, filepath, testing. These tests deliberately modify only temporary fixtures. The existing `readOneParquetRow[T any](t *testing.T, path string) T` helper in parquet_test.go is reused for the physical-row round-trip. + +```go +func TestMetricPhysicalRowKeepsFidelityColumns(t *testing.T) { + store,err:=OpenParquetStore(t.TempDir());if err!=nil {t.Fatal(err)} + defer store.Close() + metric:=Metric{Name:"net",Type:"sum",Temporality:"delta",Monotonic:true, + StartTimeUnixNanos:1790855940000000017,TimeUnixNanos:1790856000000000009,IngestedAt:1790856000000000010} + if err:=store.CommitBatch(t.Context(),BatchMetadata{ID:"fidelity"},nil,nil,[]Metric{metric});err!=nil {t.Fatal(err)} + got:=readOneParquetRow[metricParquetRow](t,filepath.Join(store.BatchPath("fidelity"),"metrics.parquet")) + if got.MetricType!="sum"||got.Temporality!="delta"||!got.Monotonic||got.StartTimeUnixNano!=metric.StartTimeUnixNanos {t.Fatalf("physical=%+v",got)} + if err:=ValidatePublishedBatch(store.BatchPath("fidelity"));err!=nil {t.Fatal(err)} + if store.BatchMetadata()[0].Version!=4 {t.Fatal("metadata is not format 4")} +} + +func TestFormatThreeIsRefusedBeforeCleanup(t *testing.T) { + root := t.TempDir() + store, err := OpenParquetStore(root) + if err != nil { t.Fatal(err) } + err = store.CommitBatch(t.Context(), BatchMetadata{ID: "old"}, nil, nil, + []Metric{{Name: "n", Type: "gauge", TimeUnixNanos: 1, IngestedAt: 1}}) + if err != nil { t.Fatal(err) } + if err := store.Close(); err != nil { t.Fatal(err) } + metaPath := filepath.Join(root, "batches", "old.batch", "metadata.json") + raw, err := os.ReadFile(metaPath) + if err != nil { t.Fatal(err) } + var meta BatchMetadata + if err := json.Unmarshal(raw, &meta); err != nil { t.Fatal(err) } + meta.Version = 3 + old, err := json.Marshal(meta) + if err != nil { t.Fatal(err) } + if err := os.WriteFile(metaPath, old, 0o644); err != nil { t.Fatal(err) } + staging := filepath.Join(root, "staging", "keep") + if err := os.MkdirAll(filepath.Dir(staging), 0o755); err != nil { t.Fatal(err) } + if err := os.WriteFile(staging, []byte("preserve"), 0o644); err != nil { t.Fatal(err) } + schemaPath := filepath.Join(root, "batches", SchemaBatch, "metrics.parquet") + schemaBefore, err := os.ReadFile(schemaPath) + if err != nil { t.Fatal(err) } + if _, err := OpenParquetStore(root); !errors.Is(err, ErrUnsupportedBatchFormat) { t.Fatalf("open: %v", err) } + if err := ValidatePublishedBatch(filepath.Dir(metaPath)); !errors.Is(err, ErrUnsupportedBatchFormat) || IsBatchCorrupt(err) { t.Fatalf("verify: %v", err) } + for path, want := range map[string]string{metaPath: string(old), staging: "preserve", schemaPath: string(schemaBefore)} { + got, err := os.ReadFile(path) + if err != nil || string(got) != want { t.Fatalf("changed rejected input: %v", err) } + } +} + +func TestFormatFourRequiresNewMetricPhysicalSchema(t *testing.T) { + root := t.TempDir() + store, err := OpenParquetStore(root) + if err != nil { t.Fatal(err) } + if err := store.CommitBatch(t.Context(), BatchMetadata{ID: "schema"}, nil, nil, + []Metric{{Name: "n", Type: "sum", Monotonic: true, Temporality: "cumulative", TimeUnixNanos: 1, IngestedAt: 1}}); err != nil { t.Fatal(err) } + dir := store.BatchPath("schema") + if err := store.Close(); err != nil { t.Fatal(err) } + // A column with the wrong physical type must not be accepted even with version 4. + if err := writeTypedParquet(filepath.Join(dir, "metrics.parquet"), + []struct { Temporality int64 `parquet:"temporality"` }{{1}}, 1024); err != nil { t.Fatal(err) } + if err := ValidatePublishedBatch(dir); !errors.Is(err, ErrUnsupportedBatchFormat) || IsBatchCorrupt(err) { t.Fatalf("schema: %v", err) } +} +``` + +- [ ] **Define the shared engine-executed fixture helpers and metrics view test.** Add helpers to existing `testhelpers_test.go`; imports needed are database/sql, encoding/json and querytest, in addition to its existing imports. `query.Duck{DB: db}` has no repository and therefore executes these tables without Parquet rebinding; production snapshot coverage is a separate Task 3 test. The duckdb driver is already registered through query's imports. + +```go +func m5Fixture(t *testing.T) (*Executor, *sql.DB) { + t.Helper() + db, err := sql.Open("duckdb", "?threads=2&memory_limit=256MB") + if err != nil { t.Fatal(err) } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec("SET TimeZone='UTC'"); err != nil { t.Fatal(err) } + if err := querytest.CreateTables(db, query.CreateCacheTables); err != nil { t.Fatal(err) } + if err := query.CreateViews(db); err != nil { t.Fatal(err) } + e := NewExecutor(&query.Duck{DB: db}, 30) + e.now = func() time.Time { return fixtureStart.Add(time.Hour) } + return e, db +} + +func m5Insert(t *testing.T, db *sql.DB, m telemetry.Metric) { + t.Helper() + encode := func(v map[string]any) string { + if v == nil { return "{}" } + b, err := json.Marshal(v) + if err != nil { t.Fatal(err) } + return string(b) + } + _, err := db.Exec(`INSERT INTO telemetry.metrics + (namespace,service,name,unit,metric_type,temporality,monotonic,start_time_unix_nano, + metric_time,time_unix_nano,value,hist_bounds_json,hist_counts_json,hist_count,hist_sum, + scope_name,scope_version,attributes,resource,ingested_at,ingested_unix_nano) + VALUES (?,?,?,?,?,?,?,?,?::TIMESTAMP_NS,?,?,?,?,?,?,?, ?,?::JSON::VARIANT,?::JSON::VARIANT,?::TIMESTAMP_NS,?)`, + m.Namespace, m.ServiceName, m.Name, m.Unit, m.Type, m.Temporality, m.Monotonic, m.StartTimeUnixNanos, + time.Unix(0, m.TimeUnixNanos).UTC(), m.TimeUnixNanos, m.Value, m.HistBoundsJSON, m.HistCountsJSON, m.HistCount, m.HistSum, + m.ScopeName, m.ScopeVersion, encode(m.Attributes), encode(m.Resource), time.Unix(0, m.IngestedAt).UTC(), m.IngestedAt) + if err != nil { t.Fatal(err) } +} + +func m5Counter(at time.Time, value float64) telemetry.Metric { + return telemetry.Metric{Namespace: "test", ServiceName: "svc", Name: "net", Unit: "By", + Type: "sum", Monotonic: true, Temporality: "cumulative", Value: value, + TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), IngestedAt: at.UnixNano(), + ScopeName: "scope", ScopeVersion: "1"} +} + +func m5Run(t *testing.T, e *Executor, p Panel, start, end time.Time) Result { + t.Helper() + e.now = func() time.Time { return end } + if p.ID == "" { p.ID = "metric" } + if p.Title == "" { p.Title = "Metric" } + results, err := e.Run(t.Context(), RunRequest{Dashboard: Dashboard{ + Name: "Metrics", Time: Time{From: &start, To: &end}, Panels: []Panel{p}, + }}) + if err != nil { t.Fatal(err) } + if len(results) != 1 || results[0].Status == StatusError { t.Fatalf("result: %+v", results) } + return results[0] +} + +func m5Values(t *testing.T, r Result, column string) []any { + t.Helper() + if r.Frame == nil { t.Fatalf("missing frame: %+v", r) } + for i, c := range r.Frame.Columns { if c.Name == column { return r.Frame.Values[i] } } + t.Fatalf("missing column %s", column) + return nil +} + +func TestMetricViewMapsKindsAndKeepsNanosecondStart(t *testing.T) { + e, db := m5Fixture(t) + for i, stored := range []string{"gauge", "sum", "sum", "histogram", "exponential_histogram", "summary"} { + m := m5Counter(fixtureStart.Add(time.Duration(i)*time.Second), 1) + m.Type, m.Monotonic = stored, i == 1 + m.Name = fmt.Sprintf("metric%d", i) + m.StartTimeUnixNanos = fixtureStart.Add(-time.Minute).UnixNano()+17 + if i == 0 { m.StartTimeUnixNanos = 0; m.Temporality = "" } + if i == 5 { m.Temporality = "" } + m5Insert(t, db, m) + } + r := m5Run(t, e, Panel{Viz: "table", SQL: `SELECT name,type,temporality, + epoch_ns(start_time)::DOUBLE AS start_nanos,typeof(start_time) AS start_type + FROM metrics WHERE $__window(time) ORDER BY name`}, fixtureStart, fixtureStart.Add(time.Minute)) + if got := m5Values(t, r, "type"); !reflect.DeepEqual(got, []any{"gauge","counter","updown","histogram","histogram","summary"}) { t.Fatalf("kinds: %v", got) } + if m5Values(t, r, "start_nanos")[0] != nil { t.Fatal("zero start must be NULL") } + for _, got := range m5Values(t, r, "start_type") { if got != "TIMESTAMPTZ_NS" { t.Fatalf("start type: %v", got) } } + var exact int64 + if err := db.QueryRow("SELECT epoch_ns(start_time) FROM metrics WHERE name='metric1'").Scan(&exact); err != nil || exact != fixtureStart.Add(-time.Minute).UnixNano()+17 { t.Fatalf("nanoseconds=%d err=%v", exact, err) } +} +``` + +- [ ] **Run red.** The new fidelity tests initially fail to compile because the canonical fields do not exist. After adding the fields only, rerun to observe missing metadata/incorrect kinds; the legacy rejection test fails because version 3 is accepted. + +```bash +rtk bash scripts/with-duckdb.sh go test ./internal/ingest ./internal/telemetry ./internal/panel -run 'TestMetricIngestKeepsKindTemporalityAndStart|TestMetricPhysicalRowKeepsFidelityColumns|TestFormatThreeIsRefusedBeforeCleanup|TestFormatFourRequiresNewMetricPhysicalSchema|TestMetricViewMapsKindsAndKeepsNanosecondStart' -count=1 +``` + +- [ ] **Implement the format and mapping.** Add the three fields to Metric and the physical row, assign them in `makeMetricParquetRow`, and add these definitions to querytest's table and query's view respectively: + +```go +// metricParquetRow additions: +Temporality string `parquet:"temporality,dict"` +Monotonic bool `parquet:"monotonic"` +StartTimeUnixNano int64 `parquet:"start_time_unix_nano"` +``` + +```sql +-- querytest table additions +temporality VARCHAR, +monotonic BOOLEAN, +start_time_unix_nano BIGINT, +-- replace metric_type AS type in viewMetrics; add temporality and start_time +CASE WHEN metric_type='sum' AND monotonic THEN 'counter' + WHEN metric_type='sum' THEN 'updown' + WHEN metric_type='exponential_histogram' THEN 'histogram' + ELSE metric_type END AS type, +temporality, +CASE WHEN start_time_unix_nano=0 THEN NULL + ELSE make_timestamp_ns(start_time_unix_nano)::TIMESTAMPTZ_NS END AS start_time, +``` + +Set `batchMetadataVersion` to 4. `loadStoredBatch` already compares the complete `telemetrySchema[metricParquetRow]()` and `BatchVerifier.Validate` calls `loadRegisteredBatch` before decoding: extend the row contract, retain this one verification path and the native all-column decode, rather than invent another schema copy. Update the comments in parquet.go/views.go and AGENTS.md/data-layout to format 4. + +```go +func metricTemporality(t metricspb.AggregationTemporality) (string, error) { + switch t { + case metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE: + return "cumulative", nil + case metricspb.AggregationTemporality_AGGREGATION_TEMPORALITY_DELTA: + return "delta", nil + default: + return "", fmt.Errorf("sum and histogram metrics require cumulative or delta temporality") + } +} +``` + +In all five point loops set `StartTimeUnixNanos: int64(dp.StartTimeUnixNano)`. Gauges and summaries leave temporality empty and monotonic false. For sums call metricTemporality once before the point loop, return `status.Error(codes.InvalidArgument, err.Error())` on error, store `Type: "sum"`, `Temporality: temporality`, `Monotonic: d.Sum.IsMonotonic`; delete the sum_delta branch. For explicit/exponential histograms call the helper on their own aggregation temporality, set Temporality and keep Monotonic false; exponential kind is exactly `exponential_histogram`. + +Task 1's mechanical reader audit uses the brief's search, plus all stored-type/format literals. `metricHistogramRaw` still selects semantic `histogram`; that selector now includes exponential points. Change readbench metadata checks/outputs to 4, selectors/maps from `sum` to `counter` (include updown when choosing gauge-like measures), histogram benchmark from `p95(hist_sum)` to `p95(value)`, and shift `start_time_unix_nano` only when nonzero in `tilePanelParquet`. Do not infer monotonicity inside a production reader. Controller conversion of its lost-fidelity source is addressed in Task 7. + +```bash +rtk proxy rg -n "sum_delta|'histogram'|\"histogram\"" internal ui +rtk proxy rg -n 'metric_type|exp_histogram|Version: 3|Version != 3|format-3|format 3' internal ui site/src/content/docs/reference/data-layout.mdx AGENTS.md +``` + +Every stored/view type consumer found by these reads must use the new distinction in this task; negative tests and historical migration descriptions may name rejected versions. Keep existing test fixture kind and temporality truthful even while Task 5 still uses the old panel option. + +- [ ] **Run green and the complete gate.** The targeted command above must pass; verify readbench setup also compiles/runs after the version change. + +```bash +rtk bash scripts/with-duckdb.sh go test ./internal/telemetry/... ./internal/ingest ./internal/query ./internal/panel -count=1 +rtk bash scripts/with-duckdb.sh go test -tags=readbench ./internal/panel -run 'TestPanelBenchmarkReplay' -count=1 +rtk just check +``` + +- [ ] **Controller commit message:** `feat(telemetry): preserve metric semantics in batch format 4` + +## Task 2: Exponential histogram conversion + +**Files:** Modify `internal/ingest/server.go`, `internal/ingest/server_test.go`; create/test `internal/ingest/exponential_histogram_test.go`. Retain Task 1's ingest round-trip test. + +**Interfaces** + +Consumes (existing): `func expHistBuckets(dp *metricspb.ExponentialHistogramDataPoint) []float64`, `func expHistCounts(dp *metricspb.ExponentialHistogramDataPoint) []uint64`, and `func toJSON(v interface{}) string`. + +Produces (new): `func explicitExpHistogram(dp *metricspb.ExponentialHistogramDataPoint) ([]float64, []uint64)`. Delete both old helpers and their callers after introducing this single paired conversion; exportMetrics calls it once per point and serializes both results. + +- [ ] **Write the failing count-preservation test.** Package ingest; imports math, reflect, testing and metricspb. + +```go +func TestExponentialHistogramRetainsEveryObservation(t *testing.T) { + for _, tc := range []struct { + name string; scale, offset int32; positive, negative []uint64; zero uint64 + bounds []float64; counts []uint64 + }{ + {"scale_zero_negative_offset", 0, -1, []uint64{2,3}, []uint64{4,1}, 6, []float64{.5,1,2}, []uint64{11,2,3,0}}, + {"scale_three_positive_offset", 3, 2, []uint64{2,3}, []uint64{4}, 1, []float64{math.Pow(2,.25),math.Pow(2,.375),math.Pow(2,.5)}, []uint64{5,2,3,0}}, + {"scale_negative_two", -2, 1, []uint64{7,9}, nil, 0, []float64{16,256,4096}, []uint64{0,7,9,0}}, + {"zero_only", 0, 0, nil, nil, 9, []float64{0}, []uint64{9,0}}, + {"negative_only", 3, -2, nil, []uint64{3,5}, 0, []float64{0}, []uint64{8,0}}, + {"empty", 0, 0, nil, nil, 0, []float64{0}, []uint64{0,0}}, + } { + t.Run(tc.name, func(t *testing.T) { + dp := &metricspb.ExponentialHistogramDataPoint{Scale: tc.scale, ZeroCount: tc.zero, + Positive: &metricspb.ExponentialHistogramDataPoint_Buckets{Offset: tc.offset, BucketCounts: tc.positive}, + Negative: &metricspb.ExponentialHistogramDataPoint_Buckets{BucketCounts: tc.negative}} + dp.Count = tc.zero + for _, n := range tc.positive { dp.Count += n } + for _, n := range tc.negative { dp.Count += n } + bounds, counts := explicitExpHistogram(dp) + if len(counts) != len(bounds)+1 || !reflect.DeepEqual(counts, tc.counts) { t.Fatalf("bounds=%v counts=%v", bounds, counts) } + if len(bounds) != len(tc.bounds) { t.Fatalf("bounds=%v", bounds) } + for i, want := range tc.bounds { if math.Abs(bounds[i]-want) > math.Abs(want)*1e-12 { t.Fatalf("bound %d=%g want %g", i, bounds[i], want) } } + var total uint64 + for _, n := range counts { total += n } + if total != dp.Count { t.Fatalf("total=%d count=%d", total, dp.Count) } + }) + } +} +``` + +- [ ] **Run red:** the new paired helper is undefined. Defining its signature with the previous conversion exposes wrong lengths and lost zero/negative counts. + +```bash +rtk bash scripts/with-duckdb.sh go test ./internal/ingest -run TestExponentialHistogramRetainsEveryObservation -count=1 +``` + +- [ ] **Implement the paired conversion and replace the nil-output tests.** No interpolation of individual negative buckets is promised: the brief explicitly folds negatives and zero into the first bucket. + +```go +func explicitExpHistogram(dp *metricspb.ExponentialHistogramDataPoint) ([]float64, []uint64) { + low := dp.ZeroCount + if dp.Negative != nil { for _, n := range dp.Negative.BucketCounts { low += n } } + if dp.Positive == nil || len(dp.Positive.BucketCounts) == 0 { + return []float64{0}, []uint64{low, 0} + } + base := math.Pow(2, math.Pow(2, -float64(dp.Scale))) + bounds := make([]float64, len(dp.Positive.BucketCounts)+1) + for i := range bounds { bounds[i] = math.Pow(base, float64(dp.Positive.Offset)+float64(i)) } + counts := make([]uint64, len(bounds)+1) + counts[0] = low + copy(counts[1:], dp.Positive.BucketCounts) + return bounds, counts +} +``` + +In the exponential point loop use `bounds, counts := explicitExpHistogram(dp)` and `toJSON(bounds)`, `toJSON(counts)`. Remove `TestExpHistBuckets`/`TestExpHistCounts` (which currently expect nil for zero-only data) in favor of the full conversion test. Task 5 executes the converted shape through percentile and distribution panels. + +- [ ] **Run green:** the conversion and full ingest tests pass; run the gate. + +```bash +rtk bash scripts/with-duckdb.sh go test ./internal/ingest -run 'TestExponentialHistogramRetainsEveryObservation|TestMetricIngestKeepsKindTemporalityAndStart' -count=1 +rtk just check +``` + +- [ ] **Controller commit message:** `fix(ingest): retain all exponential histogram counts` + +## Task 3: Shared series-increase SQL + +**Files:** Create `internal/panel/metric_series.go`, `internal/panel/metric_series_test.go`; modify `internal/panel/exec.go`, `internal/panel/table_trends.go`. Test with existing `internal/panel/testhelpers_test.go` and `internal/query/file_snapshot_test.go` behavior through a real panel request. + +**Interfaces** + +Consumes: `func buildWhere(sig *signal, filters []Filter, scope Scope) (string, []any, error)`, `func sqlFloat(v float64) string`, existing `Scope`, `Compiled`, `Engine` and `queryrows.WithWindow`. + +Produces (new): + +```go +const metricLookback = 10 * time.Minute +const metricSeriesKey = "namespace,service,name,unit,scope_name,scope_version,attributes,resource" +func metricReadScope(p *Panel, scope Scope) Scope +func metricSeriesCTE(filters []Filter, scope Scope) (string, []any, error) +``` + +The CTE defines `metric_input` (raw lookback rows), `metric_points` (finite, valid, unique timestamp rows), and `metric_increases` (in-window increases). Output retains identity/grouping fields and exposes `_bucket BIGINT`, `increase DOUBLE`, `elapsed_s DOUBLE`, `count_increase DOUBLE`, `sum_increase DOUBLE`, `bound_increases HUGEINT[]`, `_bounds DOUBLE[]`. A cumulative first point stays in metric_points for latest-value operations but contributes no increase. Delta first points are retained with nullable elapsed. No public API or dependency is added. New test-only helpers are `func m5BindTimes(t *testing.T, text string, args []any) string` and `func m5IncreaseResult(t *testing.T, e *Executor, scope Scope, selectSQL string) Result`. + +- [ ] **Write the failing builder tests.** Package panel imports encoding/json, fmt, reflect, strings, testing, time, telemetry, telemetrystore and queryrows. Execute builder results through SQL panels on the querytest engine; test SQL is the builder plus a projection, not an assertion about its text. Helpers are defined here. + +```go +func m5BindTimes(t *testing.T, text string, args []any) string { + t.Helper() + // Builder tests have no authored filters; their four arguments are times. + // Render only these trusted fixture instants into a guarded SQL panel. + for _,arg:=range args { + at,ok:=arg.(time.Time);if !ok {t.Fatalf("fixture argument is %T",arg)} + text=strings.Replace(text,"?",fmt.Sprintf("make_timestamp_ns(%d)",at.UnixNano()),1) + } + if strings.Contains(text,"?") {t.Fatal("unbound fixture instant")} + return text +} + +func m5IncreaseResult(t *testing.T, e *Executor, scope Scope, selectSQL string) Result { + t.Helper() + cte, args, err := metricSeriesCTE(nil, scope) + if err != nil { t.Fatal(err) } + return m5Run(t,e,Panel{Viz:"table",SQL:m5BindTimes(t,cte+selectSQL,args)}, + scope.Start.Add(-metricLookback),scope.End) +} + +func TestMetricIncreaseOrdersDeduplicatesAndResets(t *testing.T) { + e, db := m5Fixture(t) + at := fixtureStart + for _, p := range []struct{ seconds int; value float64; ingest int64; start int64 }{ + {60,160,60,1}, {-60,40,-60,1}, {0,100,0,1}, + {60,10,59,1}, // earlier retry at the same timestamp must lose + {120,5,120,1}, // value decrease: reset contributes 5 + {180,20,180,2}, // start change without decrease: reset contributes 20 + {240,50,240,0}, // absent start does not by itself reset + } { + m := m5Counter(at.Add(time.Duration(p.seconds)*time.Second),p.value) + m.IngestedAt = at.Add(time.Duration(p.ingest)*time.Second).UnixNano() + m.StartTimeUnixNanos = p.start + m5Insert(t,db,m) + } + // New resource identity is a new container series; its first point is not traffic. + for i, value := range []float64{9000,9060} { + m := m5Counter(at.Add(time.Duration(150+i*60)*time.Second),value) + m.Resource = map[string]any{"container.id":"new"} + m5Insert(t,db,m) + } + r := m5IncreaseResult(t,e,Scope{Start:at,End:at.Add(5*time.Minute),Interval:time.Minute}, + " SELECT epoch_ms(time)::BIGINT AS time,increase,elapsed_s FROM metric_increases ORDER BY time") + if got := m5Values(t,r,"increase"); !reflect.DeepEqual(got,[]any{60.0,60.0,5.0,20.0,60.0,30.0}) { t.Fatalf("increase=%v",got) } + if got := m5Values(t,r,"elapsed_s"); !reflect.DeepEqual(got,[]any{60.0,60.0,60.0,60.0,60.0,60.0}) { t.Fatalf("elapsed=%v",got) } +} + +func TestDeltaFirstPointNeedsElapsedOnlyForRate(t *testing.T) { + e,db := m5Fixture(t) + for i := range 3 { + m := m5Counter(fixtureStart.Add(time.Duration(i)*time.Minute),12) + m.Temporality = "delta" + if i == 2 { m.StartTimeUnixNanos = fixtureStart.Add(90*time.Second).UnixNano() } + m5Insert(t,db,m) + } + r := m5IncreaseResult(t,e,Scope{Start:fixtureStart,End:fixtureStart.Add(3*time.Minute)}, + " SELECT epoch_ms(time)::BIGINT AS time,increase,elapsed_s FROM metric_increases ORDER BY time") + if !reflect.DeepEqual(m5Values(t,r,"increase"),[]any{12.0,12.0,12.0}) || + !reflect.DeepEqual(m5Values(t,r,"elapsed_s"),[]any{nil,60.0,30.0}) { t.Fatalf("delta=%+v",r.Frame) } +} + +func TestMetricSeriesIdentitySeparatesEveryField(t *testing.T) { + for _,field:=range []string{"namespace","service","name","unit","scope_name","scope_version","attributes","resource"} { + t.Run(field,func(t *testing.T) { + e,db:=m5Fixture(t) + m5Insert(t,db,m5Counter(fixtureStart.Add(-time.Minute),0)) + for i,value:=range []float64{9000,9060} { + m:=m5Counter(fixtureStart.Add(time.Duration(i)*time.Minute),value) + switch field { + case "namespace":m.Namespace="other" + case "service":m.ServiceName="other" + case "name":m.Name="other" + case "unit":m.Unit="other" + case "scope_name":m.ScopeName="other" + case "scope_version":m.ScopeVersion="other" + case "attributes":m.Attributes=map[string]any{"literal.dot":int64(1)} + case "resource":m.Resource=map[string]any{"literal.dot":true} + } + m5Insert(t,db,m) + } + r:=m5IncreaseResult(t,e,Scope{Start:fixtureStart,End:fixtureStart.Add(2*time.Minute)}, + " SELECT epoch_ms(time)::BIGINT AS time,increase,elapsed_s FROM metric_increases ORDER BY time") + if !reflect.DeepEqual(m5Values(t,r,"increase"),[]any{60.0}) {t.Fatalf("identity=%+v",r.Frame)} + }) + } +} + +func TestHistogramIncreaseResetsAllChannels(t *testing.T) { + e,db := m5Fixture(t) + for i,p := range []struct{ bounds,counts string; count int64; sum float64; start int64 }{ + {"[10,20]","[10,20,30]",60,100,1}, + {"[10,20]","[12,23,35]",70,130,1}, + {"[5,20]","[20,30,40]",90,200,1}, // bounds change while total increases + {"[5,20]","[1,2,3]",6,12,1}, // count decrease + {"[5,20]","[2,3,4]",9,18,2}, // start change + } { + m:=m5Counter(fixtureStart.Add(time.Duration(i)*time.Minute),0) + m.Type,m.Monotonic="histogram",false + m.HistBoundsJSON,m.HistCountsJSON,m.HistCount,m.HistSum,m.StartTimeUnixNanos=p.bounds,p.counts,p.count,p.sum,p.start + m5Insert(t,db,m) + } + cte,args,err:=metricSeriesCTE(nil,Scope{Start:fixtureStart,End:fixtureStart.Add(5*time.Minute),Interval:time.Minute}) + if err!=nil { t.Fatal(err) } + r:=m5Run(t,e,Panel{Viz:"table",SQL:m5BindTimes(t,cte+" SELECT count_increase,sum_increase,to_json(bound_increases) AS weights FROM metric_increases ORDER BY time",args)},fixtureStart.Add(-metricLookback),fixtureStart.Add(5*time.Minute)) + wantCounts:=[]float64{10,90,6,9}; wantSums:=[]float64{30,200,12,18} + wantBounds:=[][]float64{{2,3,5},{20,30,40},{1,2,3},{2,3,4}} + if r.Frame.Rows!=4 {t.Fatalf("rows=%d",r.Frame.Rows)} + for i,weights:=range m5Values(t,r,"weights") { + var got []float64;if err:=json.Unmarshal([]byte(weights.(string)),&got);err!=nil {t.Fatal(err)} + if m5Values(t,r,"count_increase")[i]!=wantCounts[i]||m5Values(t,r,"sum_increase")[i]!=wantSums[i]||!reflect.DeepEqual(got,wantBounds[i]) {t.Fatalf("row %d: %+v",i,r.Frame)} + } +} +``` + +The builder tests above execute real guarded SQL panels through Executor.Run on querytest tables. The integration test below additionally exercises the production file snapshot; Tasks 4–5 add the full structured-panel use of these CTEs. + +- [ ] **Add the lookback snapshot regression.** This test is written now and becomes green when Task 4 connects rate compilation; until then, its Task 3 assertion uses a SQL panel with an explicit extended input window and original output window, plus the `metricReadScope` result. Do not leave the test red across task commits. + +```go +func TestMetricReadScopeIncludesSeparateLookbackBatch(t *testing.T) { + engine,repo:=newTestEngine(t) + at:=fixtureStart + for i,p:=range []struct{offset time.Duration; value float64}{{-time.Minute,100},{0,160},{time.Minute,220}} { + if err:=repo.Commit(t.Context(),telemetrystore.Batch{ID:fmt.Sprintf("lookback%d",i),Metrics:[]telemetry.Metric{m5Counter(at.Add(p.offset),p.value)}});err!=nil {t.Fatal(err)} + } + p:=Panel{ID:"rate",Title:"Rate",Viz:"timeseries",Query:&Query{From:"metrics",Where:[]string{"name='net'"},Measures:[]string{"rate()"},Bucket:"1m"}} + original:=Scope{Start:at,End:at.Add(2*time.Minute),Interval:time.Minute} + read:=metricReadScope(&p,original) + if !read.Start.Equal(at.Add(-10*time.Minute))||!original.Start.Equal(at) {t.Fatalf("read=%+v original=%+v",read,original)} + e:=NewExecutor(engine,30) + cte,args,err:=metricSeriesCTE(nil,original);if err!=nil {t.Fatal(err)} + built:=m5Run(t,e,Panel{Viz:"table",SQL:m5BindTimes(t,cte+" SELECT _bucket AS time,increase,elapsed_s FROM metric_increases ORDER BY time",args)},read.Start,read.End) + if built.Frame.Rows!=2||m5Values(t,built,"increase")[0]!=60.0 {t.Fatalf("lookback frame=%+v",built.Frame)} + // A real executor SQL panel independently confirms baseline visibility. + r:=m5Run(t,e,Panel{Viz:"table",SQL:`WITH prior AS ( + SELECT time,value,lag(value) OVER (ORDER BY time) AS previous FROM metrics + WHERE time >= $__from::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND name='net') SELECT value-previous AS increase FROM prior + WHERE time >= CAST('2026-10-01 12:00:00' AS TIMESTAMPTZ_NS) ORDER BY time`},read.Start,read.End) + if !reflect.DeepEqual(m5Values(t,r,"increase"),[]any{60.0,60.0}) {t.Fatalf("snapshot=%+v",r.Frame)} +} +``` + +Once Task 4 lands, append a structured `m5Run` with panel p and original bounds expecting rates `[1,1]`; retain the builder SQL panel snapshot test. queryrows is needed by production code, not by these test helpers; do not leave an unused import in the test file. + +- [ ] **Run red:** builder/read-scope symbols do not exist; then run again with their signatures to observe missing dedup/reset/lookback behavior. + +```bash +rtk bash scripts/with-duckdb.sh go test ./internal/panel -run 'TestMetricIncreaseOrdersDeduplicatesAndResets|TestMetricSeriesIdentitySeparatesEveryField|TestDeltaFirstPointNeedsElapsedOnlyForRate|TestHistogramIncreaseResetsAllChannels|TestMetricReadScopeIncludesSeparateLookbackBatch' -count=1 +``` + +- [ ] **Implement the single builder.** Package panel imports fmt, strings and time. Use validated filters through buildWhere; do not concatenate author text. Only bucket expressions cast time for datetime functions; both input and output predicates remain nanosecond instants. + +```go +const metricLookback = 10 * time.Minute +const metricSeriesKey = "namespace,service,name,unit,scope_name,scope_version,attributes,resource" + +func metricReadScope(p *Panel, scope Scope) Scope { + if p.Query != nil && p.Query.From == "metrics" { scope.Start = scope.Start.Add(-metricLookback) } + return scope +} + +func metricSeriesCTE(filters []Filter, scope Scope) (string, []any, error) { + read := scope + read.Start = read.Start.Add(-metricLookback) + where,args,err := buildWhere(signals["metrics"],filters,read) + if err != nil { return "",nil,err } + bucket := "0::BIGINT" + if scope.Interval > 0 { + bucket = fmt.Sprintf("epoch_ms(time_bucket(INTERVAL '%d seconds',time::TIMESTAMP_NS))::BIGINT",int64(scope.Interval/time.Second)) + } + text := `WITH metric_input AS (SELECT * FROM metrics WHERE `+where+`), + metric_decoded AS (SELECT *, + TRY(from_json(hist_bounds_json,'["DOUBLE"]')) AS _bounds, + TRY(from_json(hist_counts_json,'["UBIGINT"]')) AS _counts + FROM metric_input), + metric_valid AS (SELECT * FROM metric_decoded WHERE + (type<>'histogram' AND isfinite(value)) OR + (type='histogram' AND isfinite(hist_sum) AND hist_count>=0 + AND _bounds IS NOT NULL AND _counts IS NOT NULL + AND len(_counts)=len(_bounds)+1 + AND coalesce(list_bool_and(list_transform(_bounds,x->isfinite(x))),true))), + metric_unique AS (SELECT * EXCLUDE (_retry) FROM ( + SELECT *,row_number() OVER (PARTITION BY `+metricSeriesKey+`,time ORDER BY + ingested_unix_nano DESC,value DESC,hist_count DESC,hist_sum DESC, + hist_counts_json DESC,hist_bounds_json DESC,start_time DESC NULLS LAST) AS _retry + FROM metric_valid) WHERE _retry=1), + metric_points AS (SELECT *,`+bucket+` AS _bucket FROM metric_unique), + metric_previous AS (SELECT *, + lag(time) OVER w AS _previous_time, + lag(value) OVER w AS _previous_value, + lag(start_time) OVER w AS _previous_start, + lag(hist_count) OVER w AS _previous_count, + lag(hist_sum) OVER w AS _previous_sum, + lag(_bounds) OVER w AS _previous_bounds, + lag(_counts) OVER w AS _previous_counts + FROM metric_points WINDOW w AS (PARTITION BY `+metricSeriesKey+` ORDER BY time,ingested_unix_nano)), + metric_reset AS (SELECT *, + ((start_time IS NOT NULL AND _previous_start IS NOT NULL AND start_time<>_previous_start) + OR CASE WHEN type='histogram' THEN hist_count<_previous_count OR _bounds<>_previous_bounds + ELSE value<_previous_value END) AS _reset, + CASE WHEN temporality='delta' AND start_time IS NOT NULL AND start_time