diff --git a/.env.example b/.env.example index ef34df6a0..770958379 100644 --- a/.env.example +++ b/.env.example @@ -81,7 +81,7 @@ # FANOUT_DUCKDB_MEMORY= # FANOUT_DUCKDB_THREADS; integer -# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +# Caps DuckDB's global query worker pool. Zero selects one worker per CPU the process may run on (the affinity mask and cgroup quota, as GOMAXPROCS resolves them) rather than DuckDB's count of the machine's hardware threads. Set it to leave cores free for ingest on a query-heavy co-tenant host. # Default: 0 # FANOUT_DUCKDB_THREADS= @@ -182,6 +182,11 @@ # Default: (empty) # FANOUT_PUBLIC_URL= +# FANOUT_QUERY_CACHE_TTL; duration +# The oldest a dashboard panel, exemplar or trace detail result served from memory may be, measured from the end of the time window it covers (for traces and exemplars, from when their run started). A result younger than half the TTL is served as is; one younger than the TTL is served while one background run refreshes it; an older one is rerun. A request with Cache-Control: no-cache, such as a dashboard's manual refresh, always reruns. Zero disables the cache; concurrent identical requests still share one execution. +# Default: 10s +# FANOUT_QUERY_CACHE_TTL= + # FANOUT_RETENTION_DAYS; integer # Sets telemetry retention in days; zero disables expiry. # Default: 30 diff --git a/AGENTS.md b/AGENTS.md index 659b88172..1cfc90850 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,10 +93,15 @@ Analytical service/edge watermark lag is separate. Completed-batch writes have their own gate and write-pool slot; analytical writes use disjoint tables. Maintenance must hold both gates for checkpointing. -- Batch format 4, including its physical schema, is the only accepted telemetry - format. Schema changes require a format version change. Disable schema unioning - and Hive partition inference; use native Parquet binding so VARIANT extracts - can reach the scan. Pre-project hot messaging fields into scalar read-view +- Batch format 5, including its physical schema, is the only accepted telemetry + format. Schema changes require a format version change. Ingest copies standard + semconv keys into typed dedicated columns, converting values of another type + by fixed rules; a value with no conversion leaves the column NULL and is + counted as a spill. The map keeps every value as sent, and the column is the + key's read value. No storage column holds JSON: repeated + values are typed lists. Every signal's files keep their declared sort order. + Disable schema unioning and Hive partition inference; use native Parquet + binding so VARIANT extracts can reach the scan. Pre-project hot messaging fields into scalar read-view columns; this preview does not push general VARIANT extracts across views. Reject unsupported metadata before cleanup or schema rewriting, preserve those files, and add no legacy reader or format fallback. @@ -105,8 +110,9 @@ cast returned timestamps and datetime-function arguments only. Every engine connection uses UTC. Do not change Parquet UTC-instant semantics for speed. - Offline verification reuses one bounded, non-spilling native engine to decode - all Parquet columns and checks physical schemas, complete span sort tuples, - and exact index ranges. Operational errors and unsupported formats never + all Parquet columns and checks physical schemas, every signal's complete sort + tuples, exact index ranges, dedicated columns against their converted map + values and spill counts, and list lengths. Operational errors and unsupported formats never authorize quarantine. Discarded hashes force decoding; they are not checksums. - Keep DuckDB 2's memory-governed asynchronous I/O defaults unless measurements justify tuning them. Do not force an unbounded read-ahead depth. diff --git a/README.md b/README.md index a41bb48f7..793e86f89 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ client, built-in agent, MCP tools, dashboards, and alerts are preview capabiliti ### Docker ```sh -docker run --name fanout -p 7520:7520 \ +docker run --name fanout --restart unless-stopped -p 7520:7520 \ -v fanout-data:/var/lib/fanout/data \ -e FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) \ labstack/fanout:latest diff --git a/cmd/fanout-docgen/routes.go b/cmd/fanout-docgen/routes.go index 8e0998374..1a67f5672 100644 --- a/cmd/fanout-docgen/routes.go +++ b/cmd/fanout-docgen/routes.go @@ -265,7 +265,7 @@ func renderRoutes(routes []api.RouteDoc) []byte { b.WriteString("## Interface stability\n\nThe `/api/*` routes serve the browser client and can change without migration\npaths. [MCP](/docs/guides/mcp) is the agent interface. OTLP ingest and\n[operational endpoints](/docs/reference/endpoints) have their own contracts.\n\nManage alert rules through the [HTTP API](/docs/guides/alerts). Administrators\nrotate ingest tokens on **Connect telemetry** or with\n`POST /api/settings/ingest/token/rotate`.\n\n") - b.WriteString("## Errors\n\nApplication API errors, including unknown routes and wrong methods, return objects shaped like\n`{\"code\":\"bad_request\",\"message\":\"Safe explanation\"}`. Codes are stable\nsnake_case identifiers; clients branch on `code` rather than message text.\nInvalid dashboard or panel specs also include `problems` with field paths,\nmessages and optional hints. Every error response uses `Cache-Control: no-store`;\nHEAD errors have no body. Server failures return a short, safe message; the\nunderlying error is logged and never returned.\n\nDefault codes by HTTP status:\n\n| Status | Code |\n|---|---|\n| 400 | `bad_request` |\n| 401 | `unauthorized` |\n| 403 | `forbidden` |\n| 404 | `not_found` |\n| 405 | `method_not_allowed` |\n| 409 | `conflict` |\n| 413 | `payload_too_large` |\n| 415 | `unsupported_media_type` |\n| 422 | `unprocessable` |\n| 429 | `rate_limited` |\n| 500 | `internal` |\n| 502 | `bad_gateway` |\n| 503 | `unavailable` |\n| 504 | `timeout` |\n\nOther 4xx statuses use `bad_request`; other 5xx statuses use `internal`.\nSpecific conditions use `already_current`, `dashboard_version_not_found`,\n`dashboard_version_conflict`, `invalid_spec` and `last_active_administrator`.\nTool failures carry `error: {code, message}` inside the tool result.\nPanel execution results keep their own status and error fields.\nOAuth, OTLP/gRPC, MCP JSON-RPC and AG-UI run errors keep their protocol spelling.\nOperational responses at `/healthz`, `/readyz` and `/metrics` keep their bodies.\n\n") + b.WriteString("## Errors\n\nApplication API errors, including unknown routes and wrong methods, return objects shaped like\n`{\"code\":\"bad_request\",\"message\":\"Safe explanation\"}`. Codes are stable\nsnake_case identifiers; clients branch on `code` rather than message text.\nInvalid dashboard or panel specs also include `problems` with field paths,\nmessages and optional hints. Every error response uses `Cache-Control: no-store`;\nHEAD errors have no body. Server failures return a short, safe message; the\nunderlying error is logged and never returned.\n\nDefault codes by HTTP status:\n\n| Status | Code |\n|---|---|\n| 400 | `bad_request` |\n| 401 | `unauthorized` |\n| 403 | `forbidden` |\n| 404 | `not_found` |\n| 405 | `method_not_allowed` |\n| 409 | `conflict` |\n| 413 | `payload_too_large` |\n| 415 | `unsupported_media_type` |\n| 422 | `unprocessable` |\n| 429 | `rate_limited` |\n| 500 | `internal` |\n| 502 | `bad_gateway` |\n| 503 | `unavailable` |\n| 504 | `timeout` |\n\nOther 4xx statuses use `bad_request`; other 5xx statuses use `internal`.\nSpecific conditions use `already_current`, `dashboard_version_not_found`,\n`dashboard_version_conflict`, `invalid_spec` and `last_active_administrator`.\n`GET /api/telemetry/schema` returns 503 `schema_busy` with `Retry-After` when a\nnew window and namespace cannot start a discovery yet, or when the discovery is\nstill running after 15 seconds; it keeps running, so a retry finds it cached.\nCached ones are always served. An unknown window is 400 `bad_request`.\nTool failures carry `error: {code, message}` inside the tool result.\nPanel execution results keep their own status and error fields.\nOAuth, OTLP/gRPC, MCP JSON-RPC and AG-UI run errors keep their protocol spelling.\nOperational responses at `/healthz`, `/readyz` and `/metrics` keep their bodies.\n\n") b.WriteString("[Roles](/docs/reference/roles) maps capabilities to roles.\n\n") @@ -286,5 +286,7 @@ func renderRoutes(routes []api.RouteDoc) []byte { b.WriteString(" session, which is how a scraper reaches the metrics endpoint.\n\n") b.WriteString("`Any` registers every HTTP method with the same authorization requirement.\n`GET /*` serves the single-page application for browser paths. Unknown `/api/*`\nand OTLP paths return errors.\n") + b.WriteString("\n## OTLP overload responses\n\nOTLP admission counts protobuf spans, log records or metric points before\nconverting them into storage rows. Under compaction debt, interactive starvation\n(refusals or queue-wait timeouts), or the decoded-byte budget, the entire export\nis refused retryably. Panel queueing alone does not trigger ingest throttling.\ngRPC responds with `RESOURCE_EXHAUSTED` and `google.rpc.RetryInfo`; HTTP responds\nwith 429 and `Retry-After` in whole seconds rounded up. The message identifies\nthe pressure reason. Rate refusals request a one-to-five-second delay; the hard\ndebt limit requests two seconds. Byte-budget refusals retain their existing\nretry delay. Enable exporter retries and honor these delays; clients with\nretries disabled can drop refused telemetry. An acknowledgement still means\nthe accepted export was durably published.\n\nExemplars and trace detail use a priority read lane with reserved capacity.\nPanels and interactive reads have separate bounded queues; a refused read\nretains the existing 503 busy response with `Retry-After`.\n") + return []byte(b.String()) } diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 249a3402d..a3dbf7068 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -120,6 +120,14 @@ func main() { os.Exit(1) } + // A failed run exits nonzero after every deferred close has run, so a + // supervisor restarts it. + failed := false + defer func() { + if failed { + os.Exit(1) + } + }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -139,23 +147,20 @@ func main() { slog.Error("duckdb init failed", "err", err) os.Exit(1) } - defer q.Close() + // A stalled engine may still be running native code, so its handles stay + // open; the process exits instead. + stalled := false + defer func() { + if !stalled { + _ = q.Close() + } + }() writer := telemetrystore.NewWriter(repository, cfg.IngestBatchSize) writer.SetInFlightBudget(cfg.IngestMaxInFlightBytes) writerResult := make(chan error, 1) - go func() { - err := writer.Run(ctx) - // Publish the result before notifying the process-wide error channel. The - // shutdown path waits on writerResult after Writer.Wait, so a final-commit - // failure cannot be lost in the close(done) -> goroutine-send scheduling gap. - writerResult <- err - if err != nil { - errCh <- fmt.Errorf("telemetry writer: %w", err) - } - }() - go q.RunRollups(ctx) + go q.Watch(ctx) // Start intelligence detector detector := intelligence.NewDetector(q, intelligence.DefaultDetectorConfig()) @@ -276,11 +281,25 @@ func main() { // raw *sql.DB here bypassed the Telemetry maintenance-race protection. queries := observability.New(q, q, cfg.RetentionDays) panels := query.NewExecutor(q, cfg.RetentionDays) + writer.SetAdmission(repository.CompactionDebt, panels.InteractiveRefusals) + go func() { + err := writer.Run(ctx) + // Publish the result before notifying the process-wide error channel. The + // shutdown path waits on writerResult after Writer.Wait, so a final-commit + // failure cannot be lost in the close(done) -> goroutine-send scheduling gap. + writerResult <- err + if err != nil { + errCh <- fmt.Errorf("telemetry writer: %w", err) + } + }() + q.SetReadPressure(panels.Pressure) + go q.RunRollups(ctx) panels.SetRollupReader(queries) + panels.SetCacheTTL(cfg.QueryCacheTTL) api.RegisterQueryRoutes(e, panels) api.RegisterFanQLRoutes(e, q, panels) api.RegisterAnnotationRoutes(e, annotations.New(q)) - api.RegisterTraceRoutes(e, queries, cfg.RetentionDays) + api.RegisterTraceRoutes(e, panels.CachedTraces(queries), cfg.RetentionDays) api.RegisterIntelligenceRoutes(e, detector) dashboards := dashboard.New(sqlite.DB, panels) api.RegisterDashboardRoutes(e, dashboards) @@ -415,7 +434,15 @@ func main() { case <-stop: slog.Info("shutting down") case err := <-errCh: + failed = true slog.Error("fatal error, shutting down", "err", err) + case err := <-q.Fatal(): + // DuckDB invalidated its database, or Fanout's own work on it stalled. + // Flush ingest and exit nonzero so the supervisor restarts Fanout, + // which rebuilds the query cache from Parquet. + failed = true + stalled = errors.Is(err, engine.ErrStalled) + slog.Error("query engine failed, shutting down for a restart", "err", err) } // Stop every surface and finish in-flight exports before stopping the writer. @@ -427,6 +454,8 @@ func main() { _ = httpSrv.Close() } shutdownCancel() + // Schema discoveries outlive their requests; stop them before the engine. + panels.Close() grpcSrv.Stop() cancel() writer.Wait() diff --git a/docs/adr/0008-batch-format-5.md b/docs/adr/0008-batch-format-5.md new file mode 100644 index 000000000..1a448c5da --- /dev/null +++ b/docs/adr/0008-batch-format-5.md @@ -0,0 +1,49 @@ +# 0008. Batch format 5 + +Status: Accepted +Date: 2026-11-01 + +## Context + +Format 4 kept every OpenTelemetry attribute in a VARIANT map. Filters and +groupings on common keys such as `http.route` or `k8s.pod.name` decoded the +map on every row and could not use Parquet statistics. Some values were +stored as JSON strings, and log and metric files had no declared row order, +so their row groups covered every service and time. Proposal 0005 records +the analysis. + +## Decision + +Format 5 replaces format 4 and supersedes the format half of ADR 0003. + +- A fixed rule over semantic conventions v1.44.0 selects 47 standard keys: + 18 span attribute keys and 29 resource keys. Each has a typed dedicated + column. Spans carry all 47; logs and metrics carry the 29 resource + columns. A different key list needs a new format version. +- Ingest converts each key's value to the column type. Integer keys take + integers, integral floats within range and base-10 integer strings, so + `"200"` is stored as 200. String keys take strings and the text of + integers, booleans and floats. Any other value leaves the column NULL and + is counted in the footer as a spill. The map keeps every value as sent. +- No storage column holds JSON. Span events and links, metric exemplars, + histogram buckets and summary quantiles are typed lists. +- Every signal's files keep a declared sort order. Spans sort by trace + hash, start time and span ID. Logs sort by five-minute bucket, namespace, + service and time. Metrics sort by namespace, service, name, hour, series + and time. +- Reads rewrite `attributes['k']` and `resource['k']` for a standard key to + its column on the parsed SQL, so FanQL, structured panels and SQL share one + rule and read the converted value. Users never name the columns, and + result stars leave them out. +- SQL rejects `GROUP BY ALL`, because the pinned engine groups its keys + without VARIANT collation and splits equal attribute values. + +## Consequences + +- Filters and groupings on standard keys read one typed column and skip + row groups by statistics. +- A standard key has one type in schema discovery, completion and results, + whatever type senders used. +- Format 4 directories are refused and preserved; operators delete them + before running format 5. +- Adding a key, changing a type or changing a sort order is a format change. diff --git a/docs/adr/0009-ingest-admission-at-saturation.md b/docs/adr/0009-ingest-admission-at-saturation.md new file mode 100644 index 000000000..c598071dd --- /dev/null +++ b/docs/adr/0009-ingest-admission-at-saturation.md @@ -0,0 +1,88 @@ +# 0009. Ingest admission at saturation + +Status: Accepted +Date: 2026-11-03 + +## Context + +Blocking the ingest writer at the compaction debt limit leaves decoded exports +queued until their client deadlines. Under sustained overload this can kill +exporters and consume CPU needed to retire debt. A shared FIFO for panels, +exemplars and trace detail also lets long panel queues exhaust drill deadlines. + +## Decision + +Reject whole exports before converting protobuf items into storage rows. A +rows-per-second token bucket adapts every 250 ms. At half the unchanged 1,024 +reclaimable-batch limit, rising debt reduces the rate by 15%; falling debt +increases it by 10%, bounded by 1.5 times the recent 60-second peak. The entry +rate uses the preceding second and the recent measured peak, so retry gaps +cannot seed a warm store at one row per second. The band cap stops decreasing +at one normal batch per pressure window (50,000 rows/s), or the measured entry +rate if lower. Below that, further row cuts shrink files without reducing their +creation frequency. The hard debt stop remains unconditional. With no history, the +first export seeds the bucket so cold pressure cannot strand it. Rising interactive +refusals (including queue-wait timeouts) indicate starvation for three seconds +after the latest refusal. Panel queueing alone never reduces ingest. Starvation +reduces a separate read cap once per second to 50% of the measured peak before +starvation; recovery increases it by 10% per second. Freeze that reference during +the incident so a long incident cannot erase its own floor. Half the healthy +rate bounds the throughput sacrificed to read feedback while leaving room for +reads to recover. Continued starvation at that floor requires measuring read +cost; a lower ingest rate requires compaction debt, not just read demand. Apply +the smaller of the independent debt and read caps so the read floor cannot +defeat debt protection. +Read-driven refusals are bounded in time as well: after the first refusal, +allow them for at most one second, then suspend the read cap for ten seconds. +Both the [Collector exporter helper](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.157.0/exporter/exporterhelper/internal/retry_sender.go) +and Go OTLP exporters wait for the greater of the server delay and exponential +backoff. Their default initial backoff is five seconds with 50% jitter. Every +first retry from the entire refusal window therefore lands in the open interval, +with 2.5 seconds of scheduling/transport margin, within telemetrygen's configured +30-second export timeout. Reducing RetryInfo alone does not shorten that client +backoff. A percentage rate floor alone also +cannot prevent the same exporter losing repeated token-bucket races. The +independent debt cap, hard debt stop and in-flight byte budget still apply during +the open interval. This favors exporter progress over prolonged read-driven +ingest reduction; admission priority must protect interactive latency. + +Below the debt band admission is uncapped unless reads require a cap. + +At the debt limit refuse every new export with a two-second retry delay. Other +rate refusals carry the estimated token wait, clamped to one through five +seconds. OTLP/gRPC returns RESOURCE_EXHAUSTED with RetryInfo; OTLP/HTTP returns +429 with Retry-After. The existing decoded-byte budget remains a separate bound. +Already admitted exports drain and preserve durable acknowledgement semantics. + +Reserve max(1, slots/6) read slots for interactive requests, with interactive +waiters served first on releases. Under contention retain only one panel +progress slot: a running panel satisfies it, so its sibling cannot prevent an +interactive replacement taking the next released slot. A refresh runs at most +two siblings concurrently and holds a bounded lifetime ticket while accepted; +idle or coalesced work holds no global slot. Panels queue within that bounded +refresh population; interactive requests queue up to twice the slot count. A one-slot pool has no reservation, +so panels remain usable. The engine still retains its two separate connections +for other readers where the pool permits. No CPU reservation is introduced. + +Under debt pressure group commits target 200,000 rows and a one-second window, +with the existing 64 MiB group bound. Sparse retries below 50,000 total +rows keep the full bounded window rather than closing after an idle gap, so +throttling does not multiply small files and their read and compaction overhead. +Once a normal-size 50,000-row batch is available, a 20 ms idle gap releases +closed-loop exporters to send again without waiting the full second. Physical row groups +remain 50,000 rows. Compaction continues under read pressure without algorithmic +changes or a higher debt limit. + +## Consequences + +Senders need retries enabled and must honor the protocol's delay. Retry-disabled +exporters can drop refused work. Admission is feedback control, not a guarantee +that arbitrary offered load or arbitrary-size exports meet a client deadline. +The hard stop remains a safety valve; bounded backlog and read latency require +load measurements. Larger groups can retain more decoded memory, still within +the in-flight byte budget. Interactive floods can reduce panel throughput. + +The backpressure-seconds metric now measures time with an admission cap active, +not time blocked in the writer. Historical values across this change are not +directly comparable. Throttling reasons, current rate, compaction pass duration +and retired inputs expose whether the controller and compactor are keeping up. diff --git a/docs/adr/README.md b/docs/adr/README.md index 27202757a..0d9b92238 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -32,3 +32,5 @@ Date: YYYY-MM-DD | 0005 | [CalVer releases](0005-calver-releases.md) | Accepted | | 0006 | [One structured query path](0006-one-structured-query-path.md) | Accepted | | 0007 | [A hand-written FanQL parser](0007-hand-written-fanql-parser.md) | Accepted | +| 0008 | [Batch format 5](0008-batch-format-5.md) | Accepted | +| 0009 | [Ingest admission at saturation](0009-ingest-admission-at-saturation.md) | Accepted | diff --git a/docs/authentication.md b/docs/authentication.md index 071cad0bd..9fa660717 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -98,7 +98,8 @@ login links are unavailable in that mode. All user-management routes require an administrator. User projections, including `GET /api/auth/me` and `GET /api/users`, expose `display_name` and `status`. Status is either `active` or `suspended`; roles remain `viewer`, `operator`, -and `admin`. +and `admin`. `GET /api/users` returns `{"users": [...]}`, including an empty array +when there are no users. - `POST /api/users` provisions an account with `email`, `display_name`, and `role`. New accounts have status `active`. diff --git a/docs/operations.md b/docs/operations.md index 991ab92fb..8becbb864 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -44,7 +44,7 @@ Unselected AI providers' value and file forms are ignored. For Docker, mount a provisioned secret and inject its path: ```sh -docker run --name fanout -p 7520:7520 \ +docker run --name fanout --restart unless-stopped -p 7520:7520 \ -v fanout-data:/var/lib/fanout/data \ --mount type=bind,src="$AUTH_SECRET_PATH",dst=/run/secrets/auth_code,readonly \ -e FANOUT_AUTH_CODE_SECRET_FILE=/run/secrets/auth_code \ @@ -186,11 +186,43 @@ normally. Healthcheck reads only these two settings and does not require valid secrets. An explicit `fanout healthcheck ` uses default proxy and certificate verification behaviour. -- `/healthz` is the liveness check. -- `/readyz` includes storage readiness and resolved runtime sizing. +- `/healthz` is the liveness check. It fails only when Fanout's own work on + the query engine stalls: a Parquet publication holds reads for 10 minutes, + or an idle engine does not answer a ping while no read holds it. An + overloaded engine stays alive, and so does one running a query past its + deadline. +- `/readyz` includes storage readiness and resolved runtime sizing. A busy + engine reports `degraded` with 200; a stalled one reports 503. A query still + running 25 seconds past its deadline reports `queries` as `degraded`, logs + a warning with a hash of the query (never its text), and increments + `fanout_duckdb_statement_overruns_total`. - `/metrics` exposes Prometheus metrics. Keep it private or configure `FANOUT_METRICS_TOKEN`; do not make it public merely to simplify scraping. +Monitor admission and compaction together: + +- `fanout_ingest_throttled_total{reason}` counts refused whole exports by `debt`, + `reads`, or `budget`. Exporters need retries enabled. +- `fanout_ingest_admission_rows_per_second` is the current row-rate cap; zero + means uncapped. +- `fanout_ingest_backpressure_seconds_total` measures time with an admission cap + active. Earlier releases measured blocked-writer time; do not compare them + directly. +- `fanout_compaction_backlog_batches` is total generation-0 inventory, including + batches excluded from reclaimable debt. +- `fanout_compaction_pass_seconds` measures pass duration, and + `fanout_compaction_retired_batches_total` counts input batches retired by + successful publications. Compare its rate with backlog growth. + +Run Fanout under a supervisor that restarts it on exit: Docker or Compose with +`restart: unless-stopped` (`--restart unless-stopped`), systemd with +`Restart=on-failure`, or Kubernetes. Docker's `HEALTHCHECK` only marks a +container unhealthy; it does not restart it. Fanout exits with status 1 when +the query engine fails beyond recovery, for example when it runs out of memory +while writing its cache, or when a stall that fails `/healthz` lasts another +25 seconds. It first finishes writing received telemetry; the restart +rebuilds the query cache from stored telemetry. + Alert on repeated restarts, readiness failures, ingest authentication failures, telemetry drops, sustained query latency, and available disk space. Retention is controlled by `FANOUT_RETENTION_DAYS`; maintenance removes expired data on diff --git a/docs/proposals/0002-agent-dashboards.md b/docs/proposals/0002-agent-dashboards.md index 80cc10753..4896e3e20 100644 --- a/docs/proposals/0002-agent-dashboards.md +++ b/docs/proposals/0002-agent-dashboards.md @@ -120,7 +120,7 @@ agent / chat / MCP client browser dashboard │ │ ▼ ▼ MCP tools ──────────────┐ /api/panels/query (batch) - get_telemetry_schema │ /api/annotations + get_telemetry_schema │ /api/annotations/query preview_panels │ /api/variables/resolve create/replace/edit │ │ _dashboard ▼ ▼ @@ -381,7 +381,7 @@ New product routes follow AGENTS.md naming: | `POST /api/panels/query` | run inline panels for a time range and variable values; used by dashboards, previews and chat | | `POST /api/panels/exemplars` | traces behind one bucket, bar or row of a panel | | `POST /api/variables/resolve` | options for the query variables of a spec | -| `GET /api/annotations` | deploys and anomalies for a range, namespace and services | +| `POST /api/annotations/query` | deploys and anomalies for a range, namespace and services | | `GET /api/telemetry/schema` | columns, units, attribute keys with types and cardinality, metric names, services | `/api/observability/*` stays for the chat views until those views render diff --git a/docs/proposals/0004-fanql.md b/docs/proposals/0004-fanql.md index 61c3edf46..f157183fc 100644 --- a/docs/proposals/0004-fanql.md +++ b/docs/proposals/0004-fanql.md @@ -164,7 +164,7 @@ Rules, each with fixtures in the corpus: 8. **Value classification.** In value position the classifier tries duration, then number, then boolean, then word. In field position a number-shaped token is `a filter needs a field before >`. 9. **Calls.** An identifier immediately followed by `(` is a call. In `where` the name must be a filter function (`lower upper starts_with contains regexp_matches length`); `spans where P95(Duration)` is `function p95 is not allowed in a filter`. Arguments are typed by signature: `starts_with(field, string)`, `contains(field, string)`, `regexp_matches(field, string)`, `lower(field)`, `upper(field)`, `length(field)`. A bare word in a `string` slot is `starts_with needs a quoted string`. `length`, `lower`, `upper` must be compared (`length(...) needs a comparison`); `starts_with`, `contains`, `regexp_matches` are predicates alone. A space before `(` is `write starts_with(...) without a space`. 10. **Optional `where`.** `where` may be omitted after the source; canonical prints it whenever a filter is present. Search bars never contain it and reject it. -11. **Foreign syntax** gets a hint, never an alias, and only on unmistakable syntax: a known foreign stage command directly after `|` (`stats summarize select project extend eval fields rename take head top order sort by`), `order by`/`group by`/`facet` at a clause position, a foreign call shape at stage position (`bin( span( percentile( perc95( pct( count_over_time( quantile_over_time( avg( mean( median( dc( distinct_count( cardinality(`), `from`, `index=`, `source=`, `search` as the first token, `ago(`, `now(`, `earliest=`, `_time:`, `since`/`until` at clause position, the operators `== =~ |= |~ ::`, `{` at term start, `@word:`, `!word:`. A bare foreign word in filter position is free text. Messages name the FanQL form: `"summarize" is not FanQL; write the measures after |, e.g. | count() by service`; `== is not FanQL; use key:value or =`; `=~ is not FanQL; ~ is contains; use regexp_matches(field, 're') for a regular expression`; `{ selectors are not FanQL; write labels as key:value, e.g. service:api`; `"@http.target" is not a field name; write the attribute without @: http.target`; `! negation is not FanQL; write -service:api`; `the time range is not part of FanQL; use the time picker (since is reserved)`. +11. **Foreign syntax** gets a hint, never an alias, and only on unmistakable syntax: a known foreign stage command directly after `|` (`stats summarize select project extend eval fields rename take head top order sort by`), `head N`/`take N` and `order by`/`group by`/`facet` at a clause position (`group by` after measures has the fix-it `by`), a foreign call shape at stage position (`bin( span( percentile( perc95( pct( count_over_time( quantile_over_time( avg( mean( median( dc( distinct_count( cardinality(`), `from`, `index=`, `source=`, `search` as the first token, `ago(`, `now(`, `earliest=`, `_time:`, `since`/`until` at clause position, the operators `== =~ |= |~ ::`, `{` at term start, `@word:`, `!word:`. A bare foreign word in filter position is free text. Messages name the FanQL form: `"summarize" is not FanQL; write the measures after |, e.g. | count() by service`; `== is not FanQL; use key:value or =`; `=~ is not FanQL; ~ is contains; use regexp_matches(field, 're') for a regular expression`; `{ selectors are not FanQL; write labels as key:value, e.g. service:api`; `"@http.target" is not a field name; write the attribute without @: http.target`; `! negation is not FanQL; write -service:api`; `the time range is not part of FanQL; use the time picker (since is reserved)`. 12. **Quotes, backticks, brackets.** `"` or `'` with doubling as the only escape (no backslash escapes). Unclosed quote: `unclosed quote; close the value with "`, running to the end of the text. Backticks open a raw entry (0.8); `unclosed raw filter; close it with `` ` ``. Parentheses are preserved in the token stream: `unmatched ); remove it or add the opening (` and `unclosed (; add ) before |, sort, limit or the end` with the opening span. Recovery after a bad term consumes to the next whitespace at the term's depth. 13. **`*` and `$`.** `*` is legal as `k:*`, `-k:*`, `count(*)`; elsewhere in an unquoted value it is `* wildcards are not in FanQL`, hint `use ~ for contains, or quote "api*"`. `$ident` is a variable; `$` followed by anything else is `unexpected $`; `$__` names are rejected. 14. **Second `|`.** `| sort x` and `| limit n` after the stage are accepted and canonicalised without the pipe. Any other second stage: `FanQL runs one aggregation stage`, hint `remove the second |; more stages come in a later version`. @@ -191,7 +191,7 @@ Field position means a `where` term read as a field (0.3 rule 4), a measure argu - Registry invariant, tested: column names contain no `.`, so a dotted key is never a column. A segment that is not an identifier (`x-y.z`) is `"x-y.z" needs brackets`; in measure context, where `-` is an operator, `count_distinct(x-y)` is `unexpected - in count_distinct; write attributes['x-y']`. - `duration` is `duration_ms` in filters, measures and `sort`; `time`, `start_time`, `end_time` are time columns (`"time" is a time column; compare it with > or <`). - In measure context a dotted word is one `Path` token (`count_distinct(user.id)`, `avg(attributes.http.response_size)`, `p95(resource.k)`); a token starting with a digit is a number (`count() * 1.5`). After `by` the lexer is back in filter context, so `by resource.k8s.pod.name` is a word read as a field. -- In value position a dotted word is a value (`name:http.server.duration`). +- In value position a dotted word is a value (`name:http.server.request.duration`). #### 0.6 Entries and variables @@ -269,7 +269,7 @@ The reference says beside `!=`: "rows without `k` are excluded; `-k:v` includes | Timestamp | `"2026-10-29T10:00:00Z"`, `"2026-10-29 10:00:00"` (quoted; optional fraction and offset; no offset means UTC) | Only on time columns; lowered to `'…'::TIMESTAMPTZ_NS`. Elsewhere a string. | | Variable | `$service` | 0.6 | | Existence | `k:*` | `IS NOT NULL`; `-k:*` is `IS NULL`. The only accepted `*` in a value. | -| Free text | `timeout`, `"connection reset"` | `contains(lower(body), lower(text))` on logs, `operation` on spans; metrics: `free text needs a field on metrics; use name:http.server.duration`. | +| Free text | `timeout`, `"connection reset"` | `contains(lower(body), lower(text))` on logs, `operation` on spans; metrics: `free text needs a field on metrics; use name:http.server.request.duration`. | | Raw | `` `expr` `` | below | **`http_status_code`** is a text column. Comparisons `< <= > >=` and `between` on it lower through `TRY_CAST(http_status_code AS INTEGER)` so `>=500` is numeric; equality and `in` stay on the text column (`= '500'`) so the standalone-equality shape survives. One catalog flag (`compareAs: integer`) on one field. @@ -399,7 +399,7 @@ The parse response reports shape problems with spans mapped from the lowered fie | `scatter` | 2 measures, `by` 1-2 | | | `state_timeline` | 1 measure, `by` 1, bucket; `requires thresholds` | sort | -**Traces.** The `traces` viz does not list matching spans: candidates are `(namespace, trace_id)` pairs with at least one span matching the filters in the window, ranked by the matching spans' `max(duration_ms)` (or `min(start_time)` for `sort +start`), limited to `limit` (at most 1,001); each candidate returns one row, the trace's root span (or its earliest span in the window when the root is outside it), with `status` set to error if any span in the trace errored. The reference says exactly this; the Explore trace list labels columns "root service" and "root duration". Trace sort keys: `duration` (canonical for `duration_ms`), `errors`, `+start`; `sort +duration` and `sort start` are `traces sort descending by duration, by errors, or ascending by start only`. Logs: `time`, `+time`. `log_patterns`: `count`. +**Traces.** The `traces` viz does not list matching spans: candidates are `(namespace, trace_id)` pairs with at least one span matching the filters in the window, ranked by the matching spans' `max(duration_ms)` (or `min(start_time)` for `sort +start`), limited to `limit` (at most 1,001); each candidate returns one row, the trace's root span (or its earliest span in the window when the root is outside it), with `status` set to error if any span in the trace errored. A whole entry that is one prefix-negated term (`-service:x`, lowered per 0.7) instead excludes every trace with a span in the window that the positive term matches, so excluding a listed root's service removes that row; SQL negatives and negation inside `or` stay per span. The reference says exactly this; the Explore trace list labels columns "root service" and "root duration". Trace sort keys: `duration` (canonical for `duration_ms`), `errors`, `+start`; `sort +duration` and `sort start` are `traces sort descending by duration, by errors, or ascending by start only`. Logs: `time`, `+time`. `log_patterns`: `count`. **`sort errors` stays.** It is valid stored state (appends `status = 'STATUS_CODE_ERROR'` and orders by duration). FanQL accepts and prints it with the notice `sort errors also filters to error spans; new queries: write status:error … sort duration`; completion after `sort` under traces offers `duration`, `+start` and `errors` (detail "filters to errors, orders by duration"); the reference teaches `status:error … sort duration` and lists `errors` as the legacy combined key. @@ -487,11 +487,11 @@ When `editable` is false the envelope must carry `query`; consumers open the bui #### 0.14 Completion -Inputs: `GET /api/telemetry/schema` returns low-cardinality column values (12 per column, 200 for `service`), attribute keys sampled from 5,000 rows with at most 200 keys per signal (each with type, count and up to three service names), metric names; windows are limited to 24h and relative to now, cached one minute per `(window, namespace)`. Rules that follow: +Inputs: `GET /api/telemetry/schema` returns low-cardinality column values (12 per column, 200 for `service`), `operation` values, attribute keys sampled from 1,000 rows with at most 200 keys per signal (each with type, every scalar type when rows disagree, count and up to three service names), the sampled values of the attributes behind `http_route`, `http_status_code`, `rpc_service` and `messaging_system` as those columns' values, and metric names; windows are limited to 24h and relative to now, cached one minute per `(window, namespace)`. A discovery waits at most 60 seconds for one of two slots, runs at most 25 seconds, and keeps running after its callers leave, until the server stops, so the next request finds it cached; a request waits at most 15 seconds and then gets `503 schema_busy` with `Retry-After`. Rules that follow: 1. **Window clamp and recency, disclosed.** Completion requests `window = min(explore range, 24h)`. `schema.recent` is true whenever the Explore window is absolute or longer than 24h, and the popup header then says "keys seen in the last 24h (recent), not in the selected interval". 2. **Sampled metadata ranks; it never hides.** Every key seen is offered; keys not seen are still legal. Service lists only boost keys whose list contains a `service:` literal already in the text. -3. **Cold cache never blocks, and never stampedes.** With no schema for the cache key, completion answers from registries alone with `schema.warm: false` and requests one coalesced background fetch per `(window, namespace)` key, at most 4 concurrent process-wide, on a detached context with the existing 20-second schema deadline. A failed fetch is cached negatively for 10 s and reported as `warm: false, error: true`; the UI re-requests on the next keystroke and shows "schema unavailable" after two failures. +3. **Cold cache never blocks, and never stampedes.** With no schema for the cache key, completion answers from registries alone with `schema.warm: false` and requests one coalesced background fetch per `(window, namespace)` key, at most 4 concurrent process-wide, on a detached context that waits 20 seconds for the discovery. A failed fetch is cached negatively for 10 s and reported as `warm: false, error: true`; the UI re-requests on the next keystroke and shows "schema unavailable" after two failures. 4. **Types rank, not gate.** In a numeric argument slot, number-typed attributes list first, then everything else. `count_distinct` lists every field. `quantile`'s second slot offers `0.5 0.9 0.95 0.99`. Zero-argument functions insert `()` and move on. Algorithm: lex the whole text (never fails); the current token is the one containing the cursor or ending at it, and `replace` is its whole span (`sev|erity:error` replaces `severity`). Inside a `String` or `Raw` token: no completion. Inside a `BracketKey` token (`attributes['htt|`): key completion for that scope, the replace span is the key text inside the quotes and the insert closes the bracket (`http.target']`); an unterminated bracket key is still a `BracketKey`. Parse the prefix in completion mode, recording the set of alternatives the parser would accept next (`source`, `clause`, `field`, `operator`, `value(field)`, `measure`, `marg(func, index)`, `bucket`, `sort_key`, `integer`, `variable`, `bracket_key(scope)`; the term-start class is the union of field, free text, `-`, `(`, `$`, `not` and clause words). Map classes to candidates ranked by exact prefix match, substring match, class order and schema count; at most 50 items; the viz and variables shape the sets: @@ -612,7 +612,7 @@ Rather than separate renderers, the rail gets **Logs** and **Traces** entries th 1. **No empty dashboards.** The spec keeps its 1–40 panel invariant (`internal/panel/validate.go`). "New dashboard" asks for a name, then opens the Explore editor in a drawer; the dashboard is created with that first panel. Removing the last panel stays refused. 2. **No template gallery yet.** Only `DefaultSpec()` exists and every owner already has it. Add a "Duplicate dashboard" action instead (new name required); templates become a later content task. 3. **AI keys stay env-only in Fanout.** Self-hosted: Settings explains `FANOUT_AI_PROVIDER` and the key variables, and how to connect an MCP client. Hosted fanout.run: the customer enters a key on the fanout.run account page, and the control plane renders it into the tenant's environment and redeploys (Onebox Cloud design). Fanout gains no secret storage. -4. **UI saves follow the MCP rule that panels must return data,** enforced in the UI: Save and Add panel are enabled only after a run returned rows without error. The HTTP API and the existing SQL editor keep their current behaviour. +4. **UI saves follow the MCP rule that panels must return data,** enforced in the UI: Add panel and Create dashboard are enabled only after a run returned rows without error. Editing an existing panel needs only a run without error, since a healthy panel such as errors by service can be empty right now; this keeps the old SQL editor's behaviour now that the panel editor replaces it. The HTTP API keeps its current behaviour. 5. **Variables.** The Add-panel drawer opened from a dashboard can reference that dashboard's variables (for example `$service`), and the FanQL editor and the builder's where-editor understand `$name` with the All rule of section 0.6. When saving from Explore into a dashboard that has a variable on the same field, the save dialog offers to replace the literal filter with the variable. ## Constraints diff --git a/docs/proposals/0005-dedicated-attribute-columns.md b/docs/proposals/0005-dedicated-attribute-columns.md new file mode 100644 index 000000000..0c93b0498 --- /dev/null +++ b/docs/proposals/0005-dedicated-attribute-columns.md @@ -0,0 +1,620 @@ +# Dedicated attribute columns and batch format 5 + +Status: Accepted (amended 2026-11-01) +Date: 2026-10-31 +Pull request: none yet + +## Summary + +Batch format 5 copies a fixed set of standard OpenTelemetry attributes into +typed Parquet columns. A mechanical rule over the stable semantic conventions +at v1.44.0 chooses the set of 47 keys, and the rule is fixed before any +measurement. Users keep writing `attributes['k']` or `k`. The compiler reads +the column instead of the VARIANT map, and the map stays complete. + +The same format also changes log and metric storage, so there is one format +break and one data wipe. Metric files store the user-visible metric type as a +column and keep histogram buckets and summary quantiles as typed lists instead +of JSON strings. Exponential histograms keep their native scale and offsets. +Log and metric files gain a declared sort order, taken from peer practice, so +their row-group statistics can skip work. The 29 rule C resource columns +appear in span, log and metric files alike. Span events and links and metric +exemplars become typed lists, so no storage column holds JSON. + +## Decisions + +Recorded on 2026-10-31: + +1. Existing format-4 data is deleted on the lab and production deployments. + There is no converter and no legacy reader. +2. The seven merged format-4 columns are removed with no aliases. +3. The key rule is A and C below. An earlier draft had a rule B for required + span attributes, such as `url.full`. It is dropped. +4. The rule reads semantic conventions v1.44.0. The evaluation thresholds + stand as written. +5. No compatibility path is kept for keys the rule excludes. `rpc_method` is + removed although `rpc.method` is still Release Candidate; RPC keys gain + columns in the format that follows their Stable release. + +Recorded on 2026-11-01: + +6. Format 5 also carries the log and metric storage changes in the section + "Logs, metrics, events and links" below, so there is one format break and one wipe. There + is no backward compatibility: the old metric columns go without aliases. + Sort keys and layouts come from standards and peer practice, thresholds + are fixed before measuring, and the evaluation uses independent data, + never the OpenTelemetry Demo. +7. Rule C resource columns apply to every signal: spans, logs and metrics + get the same 29 keys under the same `_r_` names. Each signal's file + has its own footer spill counts. Rule A stays on spans. +8. Exemplars become typed list columns, following the ClickHouse exporter's + exemplar layout. +9. The L1 thresholds stand as written: at most half of format 4's time at + 100M log rows a day, and within ±10% at 10M. +10. The new metric columns keep the `hist_` and `exp_` prefixes. +11. Span events and links become typed list columns, following the + ClickHouse exporter's layout, and `events_json` and `links_json` are + removed with no aliases. With decision 8, no storage column of any + signal holds JSON. +12. The L4 and M4 resource-filter queries join the evaluation with the same + target as M1 to M3: at most half of format 4's time. + +The rule comes from the published standard alone. No key is added or removed +because of what Fanout's own test or lab data contains. + +## Motivation + +Span attributes live in the VARIANT `attributes` map and resource attributes +in `resource`. DuckDB decodes every row of each row group it reads. Span +files sort by trace hash, so each 50,000-row group covers the file's whole +time range. A filter on `attributes['http.target']` therefore costs time in +proportion to file size, not to the query window. At 2 threads and 1 GB it +took 0.17 s on 54k rows, 3.4 s on 2.1M rows and 10 s on 8.3M rows. The scalar +`http_route` column took under 0.07 s. Hourly compaction bounds narrow +windows, but 24-hour filters still time out. + +Format 4 already has scalar columns: `http_method`, `http_status_code`, +`http_route`, `db_system`, `rpc_method`, `rpc_service`, `peer_service`, +`service_version`, `deployment_env`, `exception_type` and `exception_message` +(`internal/telemetry/parquet_rows.go`). Most of them merge a stable key with +its deprecated spelling at ingest (`internal/ingest/server.go`). The read +views project `messaging_system` and `messaging_destination` from shredded +VARIANT fields (`internal/engine/views.go`). + +## Goals and non-goals + +Goals: + +- Filters and group-bys on standard keys cost in proportion to the window. +- One user-visible name per key. +- A key list that anyone can rederive from public sources. +- Metric type filters and histogram quantiles read typed columns, with no + per-query JSON parsing. +- Service, metric name and time filters on logs and metrics skip row groups + where the data allows it. + +Non-goals: + +- Speeding up arbitrary custom keys. +- Changing the span sort order, or the row-group size of any signal. +- Applying rule A to log record or metric point attributes. Rule A is + derived from span duration metrics; rule C resource columns cover every + signal. + +## Peer survey + +| System | Columns | How keys are chosen | +|---|---|---| +| Tempo vParquet4 | Fixed: `http.method`, `http.url`, `http.status_code`, `service.name`, `k8s.{cluster,namespace,pod,container}.name` and unprefixed `cluster`, `namespace`, `pod`, `container`. Also 10 configurable string slots for each of span and resource. [schema.go](https://github.com/grafana/tempo/blob/main/tempodb/encoding/vparquet4/schema.go) | A fixed list, plus user configuration | +| Tempo vParquet5 | No fixed span keys; only `service.name` keeps a resource column. 20 string and 5 integer slots for each of span, resource and event. Tempo 3.1 writes vParquet5 by default. [schema.go](https://github.com/grafana/tempo/blob/main/tempodb/encoding/vparquet5/schema.go) | The user configures each tenant. The docs advise choosing the attributes that contribute most to block size, and `tempo-cli analyse block` reports candidates. [docs](https://grafana.com/docs/tempo/latest/operations/dedicated_columns/) | +| OTel Collector ClickHouse exporter | `Map(LowCardinality(String), String)` columns for span and resource attributes, with bloom-filter skip indexes on their keys and values. Only `service.name` has its own column, `ServiceName`. [traces_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/traces_table.sql) | None | +| SigNoz | `LowCardinality` columns filled by a `DEFAULT` expression from the attribute maps, each with an `_exists` flag: `service.name`, `http.route`, `messaging.system`, `messaging.operation`, `db.system`, `rpc.system`, `rpc.service`, `rpc.method`, `peer.service`, and later `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.agent.name` and `gen_ai.tool.name`. [traces_migrations.go](https://github.com/SigNoz/signoz-otel-collector/blob/main/cmd/signozschemamigrator/schema_migrator/traces_migrations.go). For logs, marking a field as selected adds the same kind of column and a skip index. [reader.go](https://github.com/SigNoz/signoz/blob/main/pkg/query-service/app/clickhouseReader/reader.go) | A fixed list, plus user selection for logs | +| Jaeger v2 ClickHouse | Typed nested key/value arrays. No dedicated columns. [create_spans_table.sql](https://github.com/jaegertracing/jaeger/blob/main/internal/storage/v2/clickhouse/sql/create_spans_table.sql) | None | +| Honeycomb | Every field is its own column, and new fields add columns. [blog](https://www.honeycomb.io/blog/why-observability-requires-distributed-column-store) | Automatic, all keys | +| OTel-Arrow | Separate attribute record batches with typed value columns. No promoted keys. [data_model.md](https://github.com/open-telemetry/otel-arrow/blob/main/docs/data_model.md) | None | + +Peers that promote keys use a fixed list, a configurable one, or both. None +of them builds its fixed list from observed frequency. Tempo leaves the choice +to the operator and advises sizing it from their own blocks. Fixed lists age: +Tempo vParquet4 hard-coded pre-1.21 names such as `http.method`, and +vParquet5 dropped them. SigNoz still uses the deprecated `db.system`, +`rpc.service` and `peer.service`. A fixed list must therefore follow semconv +versions. + +## Key selection rule (fixed before measuring) + +The input is the semantic-conventions YAML model at tag +[v1.44.0](https://github.com/open-telemetry/semantic-conventions/tree/v1.44.0). +A committed script applies the rule, so the list cannot be edited by hand. + +- **Span rule A.** Take every attribute at Required, Conditionally Required + or Recommended level on a Stable duration metric among + `http.server.request.duration`, `http.client.request.duration`, + `db.client.operation.duration`, `rpc.server.call.duration` and + `rpc.client.call.duration`. Semconv bounds the cardinality of metric + attributes, so these are its own common group-by dimensions. +- **Resource rule C.** Take every Stable attribute in the `service`, + `deployment`, `k8s`, `container`, `host` and `cloud` namespaces. +- **All rules.** Each key must be Stable and scalar. `` templates are + excluded. `service.name` and `service.namespace` are skipped because the + existing `service` and `namespace` identity columns already store them. + +Applied to v1.44.0, the rules give 47 keys. Every key is Stable at that tag. +Keys marked int are integers, and all others are strings. + +- **Rule A, 18 keys.** From + [model/http/metrics.yaml](https://github.com/open-telemetry/semantic-conventions/blob/v1.44.0/model/http/metrics.yaml): + `http.request.method`, `http.response.status_code` (int), `http.route`, + `url.scheme`, `server.address`, `server.port` (int), `error.type`, + `network.protocol.name` and `network.protocol.version`. Added by + [model/db/metrics.yaml](https://github.com/open-telemetry/semantic-conventions/blob/v1.44.0/model/db/metrics.yaml): + `db.system.name`, `db.namespace`, `db.collection.name`, + `db.operation.name`, `db.response.status_code`, `db.query.summary`, + `db.stored_procedure.name`, `network.peer.address` and + `network.peer.port` (int). +- **Rule C, 29 keys.** From the `registry.yaml` file of each namespace under + [model/](https://github.com/open-telemetry/semantic-conventions/tree/v1.44.0/model): + `service.version`, `service.instance.id`, `deployment.environment.name`, + `container.id`, `container.image.name` and 24 `k8s.*` keys. These are the + `name` and `uid` of cluster, node, deployment, replicaset, statefulset, + daemonset, job and cronjob, plus `namespace.name`, + `pod.{name,uid,hostname,ip,start_time}`, `container.name` and + `container.restart_count` (int). + +The generated tables in +[http-metrics](https://github.com/open-telemetry/semantic-conventions/blob/v1.44.0/docs/http/http-metrics.md), +[database-metrics](https://github.com/open-telemetry/semantic-conventions/blob/v1.44.0/docs/db/database-metrics.md), +[rpc-metrics](https://github.com/open-telemetry/semantic-conventions/blob/v1.44.0/docs/rpc/rpc-metrics.md) +and +[docs/registry/attributes](https://github.com/open-telemetry/semantic-conventions/tree/v1.44.0/docs/registry/attributes) +show the same levels and stability. + +Excluded by the rule at v1.44.0: + +- RPC. Both RPC duration metrics and every `rpc.*` attribute are Release + Candidate. A later semconv release that makes them Stable adds + `rpc.system.name`, `rpc.method` and `rpc.response.status_code` through a + new format version. +- Messaging, because none of its keys is Stable. +- `db.query.text`, `url.template` and `user_agent.synthetic.type`, which are + Opt-In on every rule A metric. +- All `host.*` and `cloud.*` keys, `container.name` and the other + non-Stable keys in the rule C namespaces. +- `container.image.tags` and `container.image.repo_digests`, which are + string arrays. +- The `k8s..label.` and `k8s..annotation.` templates. + +The `exception.*` keys sit on span events and remain the existing +`exception_type` column. + +**Renamed keys.** The v1.44.0 registry lists these renames: + +- `http.method` → `http.request.method` +- `http.status_code` → `http.response.status_code` +- `http.target` → `url.path` + `url.query` +- `http.url` → `url.full` +- `http.scheme` → `url.scheme` +- `db.system` → `db.system.name` +- `db.name` → `db.namespace` +- `db.operation` → `db.operation.name` +- `db.statement` → `db.query.text` +- `rpc.system` → `rpc.system.name` +- `rpc.service`: folded into a fully qualified `rpc.method` +- `rpc.grpc.status_code` → `rpc.response.status_code`, as a string +- `deployment.environment` → `deployment.environment.name` +- `peer.service` → `service.peer.name`, which is Development + +Fanout stores keys exactly as they are sent. Old spellings stay in the map +and can still be queried, but slowly. Only keys in the list get a column. +Producers that still send old names convert them upstream, either with the +Collector's +[schema processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/schemaprocessor) +(alpha for traces) or with a transform processor recipe in the docs. Fanout +itself carries no alias table. The ingest fallbacks for old spellings in +`internal/ingest/server.go` are removed together with the columns they fill. +A semconv upgrade reruns the script. A changed list is a schema change and +needs a new format version. + +## Design + +**Format 5.** Each key gets a nullable, typed column, with NULL meaning the +key is absent. Strings use dictionary encoding. Physical names are hidden: +`_a_` for span keys and `_r_` for resource keys, with dots turned +into underscores. The VARIANT maps still contain every key, so the canonical +map rule in AGENTS.md holds and trace detail is unchanged. Writing a copy +instead of moving the value avoids rebuilding maps in the read views. + +Span files carry all 47 columns. Log and metric files carry the 29 rule C +columns under the same `_r_` names, so `resource['k8s.pod.name']` +resolves the same way on every signal. Metric `series_id` still hashes the +complete resource map, so series identity does not change. + +**Other types.** Ingest converts a value of another OTLP type to the key's +semconv type, and the column holds the result. An integer key takes an +integer, a double that is integral and within int64 range, and a string that +Go's `strconv.ParseInt` reads in base 10 exactly as sent, with no trimming. +For example, the string `"200"` for `http.response.status_code` becomes 200 +and `503.0` becomes 503. A string key takes a string and the text of an +integer, a boolean or a double, written by Go's `strconv` (`503.0` becomes +`"503"`). Any other value, such as `503.5` or `true` for an integer key or +bytes, maps and lists for any key, has no conversion: the column is NULL and +the file footer counts it for the key as a spill. The map always keeps the +value as sent. + +The following format-4 columns are removed with no aliases: `http_method`, +`http_status_code`, `http_route`, `db_system`, `rpc_method`, +`service_version` and `deployment_env`. Six of them map to keys that now have +dedicated columns. `rpc_method` does not, because `rpc.method` is Release +Candidate at v1.44.0. Filters on `rpc.method` read the VARIANT map until a +later format adds the key. `peer_service` and `rpc_service` contain Fanout +logic and stay as derived columns, which is open question 1. + +**Resolution.** FanQL already reads `http.route` as `attributes['http.route']` +(0004, section 0.5). The compiler maps a key in the list to its hidden column. +The SQL guard does the same on the parsed AST for every +`attributes['']` and `resource['']` subscript. The +rewritten expression has the column's type, not VARIANT, so the existing +`::INTEGER` casts still work. + +The column is the key's read value everywhere, so queries never read the map +for a key in the list and every file keeps its statistics and filter +pushdown. Users cannot write underscore-prefixed names. Catalog, suggestion +and `get_telemetry_schema` show the keys as attributes with an "indexed" +mark. + +The read path implements this as follows (`internal/engine/sql_keys.go`): + +- Snapshot binding rewrites every query, so FanQL, structured panels, panel + SQL and product SQL share one rule. A subscript becomes the column when its + map binds to a signal relation directly, or through subqueries and CTEs + that pass it on with `*`. Rewritten select items keep their original + result names. +- Every other subscript reads the map unchanged, which is the value as sent: + keys outside the list, log and metric attributes, subscripts whose relation + the AST does not determine, and subscripts of a map that a subquery or CTE + projects without its columns. In that last case a value that ingest + converted reads in its original type, as any key outside the list does. +- Unpinned views read the files' columns as they are. +- Variables compared with an integer key bind as numbers. The SQL guard and + the filter gate reject the `_a_*`, `_r_*` and `_trace_hash` names and name + the key to write instead. + +**Shredding.** Format 4 shreds `messaging.*` and four `service.*` and +`deployment.*` resource keys. Format 5 stops shredding the keys that now have +dedicated columns, since shredding them would store a third copy, and keeps +shredding `messaging.*`. + +**Views, rollups, compaction and verification.** + +- The spans view exposes the hidden columns. +- Rollups, default dashboards and FanQL reference examples move from the old + columns to keys. +- The read-cache schema version changes, so caches rebuild. +- Compaction merges format-5 inputs column for column and sums the spill + counts. Its existing schema check already refuses mixed formats. +- Offline verification also checks, on every row, that each column equals + the conversion of its map value, restated in SQL, and that the footer + counts each key's values without a conversion. + +**Existing format-4 data.** AGENTS.md and ADR 0004 forbid a legacy reader. +Fanout keeps refusing format-4 directories and preserves them, as it does +today. The operator deletes them before starting format 5. There is a +precedent: the data reset on 2026-10-11. + +**Configurable columns.** Tempo-style per-install slots would help with +`app.*` and tenant keys. They would also make the physical schema vary per +install, which conflicts with the rule that a schema change needs a format +version. Footers would have to record the mapping, and compaction and query +rewriting would depend on that per-file mapping. They are deferred until +format 5 has been measured, which is open question 3. + +## Logs, metrics, events and links + +### Metric points today + +Format 4 writes one row per OTLP data point +(`internal/ingest/server.go`, `internal/telemetry/parquet_rows.go`): + +| OTLP point | Format 4 storage | +|---|---| +| Gauge | `metric_type` `gauge` and `value`. Integer points become DOUBLE. | +| Sum | `metric_type` `sum`, a `monotonic` flag, `temporality` and `value`. | +| Histogram | `hist_bounds_json` and `hist_counts_json` as JSON strings, `hist_count`, and `hist_sum`, which is 0 when the optional sum is absent. `min`, `max` and `flags` are dropped. | +| Exponential histogram | Converted at ingest to explicit bounds and counts in the same JSON strings. `scale`, the offsets and the zero threshold are not kept. `min`, `max` and `flags` are dropped. | +| Summary | Quantiles in `hist_bounds_json` and their values in `hist_counts_json`, with `hist_count` and `hist_sum`. | + +Gauges, sums and both histogram kinds also store their exemplars as one JSON +string, `exemplars_json`. + +Two costs follow. The metrics view computes the user-visible `type` with a +CASE over `metric_type` and `monotonic` (`internal/engine/views.go`), so a +`type` filter cannot reach the Parquet scan. Every histogram query parses +both JSON strings on every row with `from_json` +(`internal/query/metric_series.go`) before it can compute a quantile. + +### Peer storage of metric points + +| System | Type | Histogram buckets | Exponential histogram | Summary | +|---|---|---|---|---| +| OTel Collector ClickHouse exporter | One table per point kind; sums keep `IsMonotonic` and `AggregationTemporality` | `ExplicitBounds Array(Float64)`, `BucketCounts Array(UInt64)`, plus `Count`, `Sum`, `Min`, `Max`, `Flags`. [metrics_histogram_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/metrics_histogram_table.sql) | Native: `Scale`, `ZeroCount`, `PositiveOffset`, `PositiveBucketCounts Array(UInt64)`, `NegativeOffset`, `NegativeBucketCounts`. [metrics_exp_histogram_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/metrics_exp_histogram_table.sql) | `ValueAtQuantiles Nested(Quantile Float64, Value Float64)`, two parallel arrays. [metrics_summary_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/metrics_summary_table.sql) | +| Prometheus native histograms, also stored by Mimir | Series metadata | Classic histograms are one `_bucket` series per `le` bound | `schema`, `zero_threshold`, `zero_count`, and positive and negative bucket spans with deltas. The OTLP translator maps `scale` to `schema` and merges buckets above schema 8. [types.proto](https://github.com/prometheus/prometheus/blob/main/prompb/types.proto), [histograms.go](https://github.com/prometheus/prometheus/blob/main/storage/remote/otlptranslator/prometheusremotewrite/histograms.go), [spec](https://prometheus.io/docs/specs/native_histograms/), [Mimir](https://grafana.com/docs/mimir/latest/send/native-histograms/) | Separate `quantile`-labelled series | +| SigNoz | `LowCardinality` `type`, `temporality` and `is_monotonic` columns on the series table | One `.bucket` series per bound with an `le` label | A DDSketch aggregate in `exp_hist`, with `count`, `sum`, `min` and `max` | One `.quantile` series per quantile. [squashed_metrics_migrations.go](https://github.com/SigNoz/signoz-otel-collector/blob/main/cmd/signozschemamigrator/schema_migrator/squashed_metrics_migrations.go), [exporter.go](https://github.com/SigNoz/signoz-otel-collector/blob/main/exporter/signozclickhousemetrics/exporter.go) | + +Tempo and Pyroscope store neither logs nor metrics. The ClickHouse exporter +is the only peer that stores OTLP points in their own shape, with typed +arrays and the exponential layout as sent. Format 5 follows it. + +### Metric points in format 5 + +The physical `type` column holds Fanout's vocabulary, `gauge`, `counter`, +`updown`, `histogram` and `summary` (`internal/query/vocabulary.go`), as a +dictionary-encoded string. Each value names one OTLP point kind, except that +`histogram` covers both histogram kinds; a non-null `exp_scale` marks the +exponential one. The view projects `type` unchanged, so a `type` filter +reaches the scan. `metric_type` and `monotonic` are removed, since `type` +carries both. `temporality` is NULL for gauges and summaries. + +| OTLP point | Format 5 columns | +|---|---| +| Gauge | `value DOUBLE` | +| Sum | `value DOUBLE`, `temporality` | +| Histogram | `hist_bounds LIST`, `hist_counts LIST` with one more entry than the bounds, `hist_count UBIGINT`, and nullable `hist_sum`, `hist_min` and `hist_max` DOUBLE | +| Exponential histogram | `exp_scale INTEGER`, `exp_zero_count UBIGINT`, `exp_zero_threshold DOUBLE`, `exp_positive_offset INTEGER`, `exp_positive_counts LIST`, `exp_negative_offset INTEGER`, `exp_negative_counts LIST`, and the four `hist_` scalars | +| Summary | `quantiles LIST`, `quantile_values LIST`, `hist_count`, `hist_sum` | + +Every kind also gains `flags UINTEGER` from the OTLP `DataPointFlags` +([metrics.proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto)). +Columns that do not apply to a point are NULL. An absent histogram sum is +NULL, not 0. Gauges and sums keep DOUBLE values for integer points, as the +ClickHouse exporter does. + +**Exemplars.** The ClickHouse exporter stores exemplars as +`Exemplars Nested(FilteredAttributes Map(LowCardinality(String), String), +TimeUnix DateTime, Value Float64, SpanId String, TraceId String)`, which +ClickHouse keeps as five parallel arrays +([metrics_histogram_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/metrics_histogram_table.sql)). +Format 5 uses the same five parallel lists, one entry per exemplar: +`exemplar_times LIST`, `exemplar_values LIST`, +`exemplar_trace_ids LIST`, `exemplar_span_ids LIST` and +`exemplar_attributes LIST`. Filtered attributes are typed VARIANT +maps like `attributes`, not string maps, so the canonical map rule holds. +Integer exemplar values become DOUBLE, as point values do. Summaries have no +exemplars, and their lists are NULL. `exemplars_json` is removed. + +### Span events and links + +Format 4 stores span events and links as the JSON strings `events_json` and +`links_json` (`eventsToJSON` and `linksToJSON` in `internal/ingest/server.go`). +Event and link attribute values are converted to strings there, so their +types are lost. Dropped attribute, event and link counts are not stored. + +The ClickHouse exporter stores `Events Nested(Timestamp DateTime64(9), Name +LowCardinality(String), Attributes Map(...))` and `Links Nested(TraceId +String, SpanId String, TraceState String, Attributes Map(...))` +([traces_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/traces_table.sql)). +Format 5 uses the same parallel lists, one entry per event or link: + +- `event_times LIST`, `event_names LIST` and + `event_attributes LIST`. +- `link_trace_ids LIST`, `link_span_ids LIST`, + `link_trace_states LIST` and `link_attributes LIST`. + +Attributes are typed VARIANT maps, as for exemplars, so event and link +values keep their OTLP types. Format 5 adds no dropped counts, since format 4 +stores none. The span's own `trace_state` stays a scalar column, and +`exception_type` and `exception_message` are still derived from events. +`events_json` and `links_json` are removed with no aliases. Verification +checks that the event lists of a row have equal lengths, and so do the link +lists. + +**No JSON remains.** After these changes no column in span, log or metric +files holds JSON. The remaining strings are scalar text such as `body`, +`status_message` and `trace_state`; maps are VARIANT and repeated values are +typed lists. The query layer still converts nested results to JSON at the +client boundary, as AGENTS.md requires. + +Ingest no longer converts exponential histograms. The quantile path derives +the upper bound of each bucket from `exp_scale` and the bucket index with the +formula `explicitExpHistogram` uses today, including its merge of bounds that +overflow, and then runs the existing union-of-bounds quantile. The lists are +native, so this is list arithmetic, not parsing. The checks on bound order and +list lengths in `metric_series.go` move to ingest, which rejects a malformed +point with `InvalidArgument` as it does for a bad zero threshold. + +The format-4 columns `metric_type`, `monotonic`, `hist_bounds_json` and +`hist_counts_json` are removed with no aliases. Public SQL that reads them +fails and must move to the new columns. + +### Sort order + +Spans keep `_trace_hash, start_unix_nano, span_id`. Trace locality and the +trace index depend on it, and the alternative is rejected below. + +Logs and metrics have no declared order today. Ingest writes them in arrival +order (`internal/telemetry/parquet.go`), and compaction concatenates inputs +(`internal/telemetry/parquet_merge.go`), so every row group spans every +service and metric of its file. Peers order these tables as follows: + +| System | Logs | Metrics | +|---|---|---| +| OTel Collector ClickHouse exporter | `ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)`, daily partitions. [logs_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/logs_table.sql) | `ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)`, daily partitions, for every point kind. [metrics_sum_table.sql](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/clickhouseexporter/internal/sqltemplates/metrics_sum_table.sql) | +| SigNoz | `logs_v2`: `ORDER BY (ts_bucket_start, resource_fingerprint, severity_text, timestamp, id)` with 30-minute buckets. [squashed_logs_migrations.go](https://github.com/SigNoz/signoz-otel-collector/blob/main/cmd/signozschemamigrator/schema_migrator/squashed_logs_migrations.go), [exporter.go](https://github.com/SigNoz/signoz-otel-collector/blob/main/exporter/clickhouselogsexporter/exporter.go) | `samples_v4`: `ORDER BY (env, temporality, metric_name, fingerprint, unix_milli)` | +| Loki, Prometheus, Mimir | Loki compresses each stream, one label set, into its own time-ordered chunks. [architecture](https://grafana.com/docs/loki/latest/get-started/architecture/) | Prometheus TSDB and Mimir store each series' samples in time-ordered chunks inside time-bounded blocks. [storage](https://prometheus.io/docs/prometheus/latest/storage/) | + +All of them put either a coarse time bucket or the stream or series identity +first, and time last. Format 5 adopts the ClickHouse exporter's keys, the +OTel-native peer, with Fanout's `namespace` placed before `service`: + +- **Logs:** five-minute bucket of `log_time`, `namespace`, `service`, + `log_time`. +- **Metrics:** `namespace`, `service`, `name`, hour of `metric_time`, + `series_id`, `metric_time`. `series_id` already hashes the series identity + (`internal/telemetry/metric_series.go`), as `cityHash64(Attributes)` does + for the exporter. + +The bucket values are computed from the time column and not stored. The +Parquet footer's `sorting_columns` can name only physical columns, so log +and metric files leave it empty. The order is a format-5 invariant that the +writer, compaction and offline verification enforce. + +**Row groups.** The ceiling stays at 50,000 rows for every signal. DuckDB +skips a row group whose min and max statistics exclude a filter, and the sort +makes those ranges narrow. Metric files then separate services and names. +Log files separate time buckets, and separate services inside a bucket only +when a bucket holds more rows than a row group. This is the expected result +of the peer order, not a tuning target, and the evaluation reports it. + +**Interactions.** + +- *Compaction.* The log and metric merges become k-way merges on these keys, + like the span merge, reading only the key columns to build the input-row + schedule. Hour merges and day merges both take sorted inputs and produce a + sorted output. The merge refuses repeated columns today, so the column + copier must advance list columns by whole rows. +- *Late data.* A late row sorts by its event time within its ingest-hour + file. It lands in its own bucket at the front of a log file, so the other + row groups keep tight time ranges. +- *Footer bounds.* Dashboard file snapshots keep reading event-time bounds + from row-group statistics (`parquetTimeRange`). Sorting changes no file's + bounds, and an unknown statistic still includes the file. +- *Read cache.* For logs and metrics it holds only per-batch resource + inventory, an aggregate that does not depend on row order. The cache + version changes with the schema, and caches rebuild. +- *Verification.* Offline verification checks the complete log and metric + sort tuples, as it does for spans, that list lengths match their kind, + and that the five exemplar lists of a row have equal lengths. + +## Evaluation plan + +1. **Fix the criteria first.** Commit the rule, the key list, the queries and + these thresholds before any run: + - Rule A and C equality filters and group-bys over 24 hours on 8.3M-row + files complete within 2× of today's `http_route` time. + - Keys outside the list are no slower than format 4, within ±10%. + - Ingest throughput regresses by no more than 10%. + - Commit memory regresses by no more than 10%. + - Bytes per span increase by no more than 15%. + - Compaction time and bytes written increase by no more than 15%. + - Every query returns rows identical to the same query on format 4. +2. **Fix the log and metric queries and criteria first.** Each query runs + over 24 hours against format 4 on the same data. Logs run at 10M and + 100M rows a day; only the larger volume puts more than 50,000 rows in a + five-minute bucket. Metrics run at 10M points a day. + - L1, service filter: log count for one service per 5-minute bucket. + - L2, severity filter: `severity_number >= 17`, counted by service. + - L3, body search: a substring match on `body`, over 1 hour and 24 hours. + - M1, metric by name: one counter's rate per service. + - M2, type filter: every `type = 'histogram'` series, counted by name. + - M3, histogram quantile: p95 of one duration histogram per service, once + with explicit and once with exponential buckets. + - L4 and M4, resource filter: an equality filter on + `resource['k8s.namespace.name']`, counted by service, on logs and on + metrics. + + The criteria: + - M1 to M4, L4, and L1 at 100M rows, complete in at most half of + format 4's time. + - L1 at 10M rows, L2 and L3 stay within ±10% of format 4. Below a row + group per bucket the log sort cannot separate services, and neither + severity nor body is a sort key. + - Log and metric ingest throughput and commit memory regress by no more + than 10%, sort included. + - Bytes per log row and per metric point, and compaction time and bytes + written, increase by no more than 15%. + - Every query returns rows identical to format 4, except where format 4 + was wrong: an absent histogram sum reads NULL, not 0, and event and + link attribute values keep their OTLP types instead of strings. +3. **Use data the list was not derived from.** + - The scratch generator emits a fixed attribute set. Extend it with a + seeded mode that samples attribute sets from the v1.44.0 registry + itself: stable, development and deprecated keys, keys outside the list, + and values sent with the wrong type, such as its current string + `"200"`. + - Add a recorded corpus from an independent instrumentation set, such as + Jaeger HotROD. + - The same seeded mode emits metrics from the v1.44.0 metric definitions: + each Stable metric with its instrument kind, unit and attribute list, + histograms with the explicit bucket boundaries semconv advises and, + with the same seed, as exponential histograms at several scales, + and a fixed share of summaries, which semconv does not define. It + covers both temporalities, integer and double points, absent sums and + malformed points. + - It emits logs with every severity number, including unset, and with + resources sampled from the rule C namespaces. Bodies come from a + public log corpus such as LogHub, so body search runs on real text. + A fixed share arrives late, with event times hours before ingest. + - Add a recorded metric corpus from the Collector's hostmetrics receiver. + - Do not use the OpenTelemetry Demo or the bench query set. +4. **Report regressions next to wins.** The report covers ingest CPU per span, + allocation ratio, file size, compaction write amplification, startup and + verification time, and query time on wrong-typed data, each with variance across + seeds. For logs and metrics it adds sort time at ingest and the share of + row groups each query skipped. + +## Alternatives + +- **Promote every key, as Honeycomb does.** That is VARIANT shredding plus + extract pushdown, which DuckDB does not do across views today. +- **Bloom filters on map keys and values, as the ClickHouse exporter does.** + Parquet bloom filters cover leaf columns, not VARIANT residuals, and do not + help group-by. +- **Sort spans by time inside files.** This would break trace locality and + the trace index. +- **Keep converting exponential histograms at ingest.** It keeps one bucket + layout for queries, but it loses `scale` and the offsets, which the + ClickHouse exporter and Prometheus both keep. Rejected. +- **Prometheus spans and deltas for buckets.** They compress sparse buckets + in a row store. Parquet's dictionary, run-length and delta encodings + already do that for dense lists, and OTLP sends dense lists. Rejected. +- **One row per bucket, as SigNoz and classic Prometheus histograms do.** It + multiplies metric rows by the bucket count and moves quantiles into a + join. Rejected. +- **Lists inside VARIANT.** The merge would keep only non-repeated columns, + but every query would cast VARIANT to a list on every row. Rejected. +- **Rule B: required span attributes.** It added `url.path`, `url.query`, + `url.full`, `http.request.method_original` and `rpc.method_original`. These + are unbounded strings that roughly double their bytes when copied, and + semconv does not use them as metric dimensions. Rejected. +- **Convert format-4 data instead of deleting it.** A one-time rewrite at + startup is a format-4 reader inside the binary, which AGENTS.md forbids. A + converter kept outside the release must reproduce the exact format-5 + schema and footer. Either way, conversion decodes every row and blocks + ingest while it runs. Rejected. + +## Risks + +- Span files grow by 47 columns, and log and metric files by 29 resource + columns that are mostly null, plus the new metric and exemplar columns. + Columns that are mostly null cost little, but the write path touches all + of them on every signal. +- `rpc.method` loses its scalar column until RPC conventions are Stable, so + RPC filters and group-bys get slower than on format 4. +- `attributes['k']` changes from VARIANT to a typed value, so + `typeof(...)` results change. +- The ingest fallbacks for old spellings are removed, so producers that send + old names lose fast paths until they convert upstream. +- Exemplar, event and link attributes need a list of VARIANT values, a nesting the writer + and DuckDB's reader have not yet carried in Fanout. It must be proven + before the format ships. +- List columns break the merge's rule of one physical value per row. The + column copier must handle repetition levels, and a mistake there corrupts + histograms silently. Verification checks list lengths for this reason. +- Sorting logs and metrics costs ingest CPU and memory, and turns their + compaction from concatenation into a k-way merge. +- Service filters on logs skip row groups only when a five-minute bucket + holds more than 50,000 rows. Small installs gain little on L1. +- Exponential quantiles compute bounds per query instead of once at ingest. +- Public SQL and saved dashboards that read `metric_type`, `monotonic`, + `exemplars_json`, `events_json`, `links_json` or the `hist_*_json` + columns break. The trace detail view reads `events_json` for span details + (`ui/host/src/dashboards/trace/span-extras.ts`) and moves to the event + lists in the same change. + +## Open questions + +None of these blocks format 5. + +1. Should `peer_service` and `rpc_service` stay as derived columns? Their + sources, `peer.service` and `rpc.service`, are deprecated. +2. Should the messaging columns stay, given that the rule excludes them? +3. Should configurable columns be built later, and with how many slots? diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 33cef7ab1..fd0adfbbd 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -36,5 +36,6 @@ Earlier proposals predate the template and keep their original structure. | 0002 | [Agent-built dashboards](0002-agent-dashboards.md) | Implemented | | 0003 | [Milestone 5: correct metrics and the system page](0003-agent-dashboards-m5.md) | Implemented | | 0004 | [Full UI without an LLM key, and FanQL](0004-fanql.md) | Accepted | +| 0005 | [Dedicated attribute columns and batch format 5](0005-dedicated-attribute-columns.md) | Accepted | Settled decisions that a proposal leads to are recorded in [../adr](../adr/README.md). diff --git a/go.mod b/go.mod index 5d8f40266..144c12b70 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,12 @@ require ( github.com/wneessen/go-mail v0.8.1 github.com/zeebo/xxh3 v1.1.0 go.opentelemetry.io/proto/otlp v1.11.1 + golang.org/x/net v0.61.0 golang.org/x/oauth2 v0.37.0 golang.org/x/sync v0.24.0 golang.org/x/sys v0.49.0 golang.org/x/time v0.16.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20261005182115-fad411399dd8 google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 modernc.org/sqlite v1.61.0 @@ -74,12 +76,10 @@ require ( golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20261011100332-faecfb24eb49 // indirect golang.org/x/mod v0.42.0 // indirect - golang.org/x/net v0.61.0 // indirect golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a // indirect golang.org/x/text v0.43.0 // indirect golang.org/x/tools v0.52.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20261005182115-fad411399dd8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20261005182115-fad411399dd8 // indirect modernc.org/libc v1.79.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect diff --git a/internal/agent/answer_only_test.go b/internal/agent/answer_only_test.go index 7a3061e73..73dff003a 100644 --- a/internal/agent/answer_only_test.go +++ b/internal/agent/answer_only_test.go @@ -184,7 +184,7 @@ func TestAnswerOnlyRealToolClassificationGolden(t *testing.T) { defer registry.Close() expected := map[string]bool{ "create_dashboard": false, "edit_dashboard": false, "replace_dashboard": false, "restore_dashboard_version": false, - "get_observability_overview": true, "get_service_topology": true, "get_service_dependencies": true, "get_service_performance": true, "inspect_trace": true, "search_logs": true, "get_intelligence_snapshot": true, "get_telemetry_schema": true, "query_telemetry": true, "preview_panels": true, "list_dashboards": true, "get_dashboard": true, "list_dashboard_versions": true, + "get_overview": true, "get_service_topology": true, "get_service_dependencies": true, "get_service_performance": true, "inspect_trace": true, "search_logs": true, "get_insights": true, "get_telemetry_schema": true, "query_telemetry": true, "preview_panels": true, "list_dashboards": true, "get_dashboard": true, "list_dashboard_versions": true, } for _, tool := range registry.Definitions() { want, ok := expected[tool.Name] diff --git a/internal/agent/dashboard_eval_golden_test.go b/internal/agent/dashboard_eval_golden_test.go index a2f592731..94ba720b6 100644 --- a/internal/agent/dashboard_eval_golden_test.go +++ b/internal/agent/dashboard_eval_golden_test.go @@ -15,6 +15,7 @@ import ( "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/dashboard" "github.com/labstack/fanout/internal/engine" + "github.com/labstack/fanout/internal/engine/querytest" fanoutmcp "github.com/labstack/fanout/internal/mcp" "github.com/labstack/fanout/internal/query" controlstore "github.com/labstack/fanout/internal/store" @@ -41,6 +42,9 @@ func TestDashboardEvalGoldensCurrent(t *testing.T) { if err = repo.Commit(t.Context(), telemetrystore.Batch{ID: "eval_fixture", Spans: []telemetry.Span{{Namespace: "shop", ServiceName: "checkout", TraceID: "trace", SpanID: "span", Name: "request", Kind: "SPAN_KIND_SERVER", StartUnixNanos: from.UnixNano(), EndUnixNanos: from.Add(time.Millisecond).UnixNano(), DurationMS: 1, StatusCode: "STATUS_CODE_OK", IngestedAt: from.UnixNano()}}}); err != nil { t.Fatal(err) } + if err := querytest.RefreshSpanRollup(engine.DB); err != nil { + t.Fatal(err) + } executor := query.NewExecutor(engine, 30) db, err := controlstore.NewSQLite(":memory:") if err != nil { diff --git a/internal/agent/runtime.go b/internal/agent/runtime.go index a7b314baa..e3c07f3d3 100644 --- a/internal/agent/runtime.go +++ b/internal/agent/runtime.go @@ -26,7 +26,7 @@ import ( "github.com/labstack/fanout/internal/toolerror" ) -const systemPrompt = `You are Fanout's observability assistant. When a view is attached to your reply it is the picture: never draw diagrams, trees, or charts in text, never use code fences to draw boxes, arrows, or trees, and never add a table or list that restates what an attached view already shows; your prose adds only what the view omits. Use get_observability_overview first for broad health questions, get_intelligence_snapshot for the latest precomputed anomalies and recurring log patterns, get_service_topology for direct dependency edges, get_service_dependencies for bounded upstream or downstream reachability from a service, get_service_performance for activity/latency/operations/comparisons, inspect_trace for trace or root-cause inspection, and search_logs for log questions. Treat structured outputs as authoritative. You build and change the user's dashboards. Build one whenever the user asks for an overview, asks why something is slow, failing or changing, asks to compare, break down or track telemetry, or asks for anything they would want to look at again; answer a single factual question with a view instead. To build one, read get_telemetry_schema, draft a complete spec of panels that answer the request, run preview_panels, fix every invalid panel, replace or drop every empty one, and only then call create_dashboard. Cover every part of the request: when a part has no data, state that finding in the answer and use the signals the service actually has. Title each panel with exactly what it measures. Prefer a few precise panels over many vague ones: headline stats first, then the time series that explain them, then a table of the worst offenders. After saving, reply in two or three sentences with what the dashboard shows and what stands out. Use filter values exactly as the schema lists them. To change a dashboard, get_dashboard first and use edit_dashboard so unrelated panels stay as they are; replace only when the user asks for a redesign. State the time window you used, distinguish missing data from healthy behavior, and never invent services, metrics, or causal claims. Do not present empty results. Dashboards contain only panels that return data; text panels are exempt. Keep answers concise because attached views provide interactive details. Never expose implementation details to the user: do not mention protocol names, tool names, schemas, query IDs, data-source names, storage engines, providers, rollups, aggregation tables, internal windows, or internal execution steps. Panel titles, descriptions and text panels must use product language, without implementation details. Refer to attached interactive content simply as a view.` + dashboardAnalysisGuidance +const systemPrompt = `You are Fanout's observability assistant. When a view is attached to your reply it is the picture: never draw diagrams, trees, or charts in text, never use code fences to draw boxes, arrows, or trees, and never add a table or list that restates what an attached view already shows; your prose adds only what the view omits. Use get_overview first for broad health questions, get_insights for the latest precomputed anomalies and recurring log patterns, get_service_topology for direct dependency edges, get_service_dependencies for bounded upstream or downstream reachability from a service, get_service_performance for activity/latency/operations/comparisons, inspect_trace for trace or root-cause inspection, and search_logs for log questions. Treat structured outputs as authoritative. You build and change the user's dashboards. Build one whenever the user asks for an overview, asks why something is slow, failing or changing, asks to compare, break down or track telemetry, or asks for anything they would want to look at again; answer a single factual question with a view instead. To build one, read get_telemetry_schema, draft a complete spec of panels that answer the request, run preview_panels, fix every invalid panel, replace or drop every empty one, and only then call create_dashboard. Cover every part of the request: when a part has no data, state that finding in the answer and use the signals the service actually has. Title each panel with exactly what it measures. Prefer a few precise panels over many vague ones: headline stats first, then the time series that explain them, then a table of the worst offenders. After saving, reply in two or three sentences with what the dashboard shows and what stands out. Use filter values exactly as the schema lists them. To change a dashboard, get_dashboard first and use edit_dashboard so unrelated panels stay as they are; replace only when the user asks for a redesign. State the time window you used, distinguish missing data from healthy behavior, and never invent services, metrics, or causal claims. Do not present empty results. Dashboards contain only panels that return data; text panels are exempt. Keep answers concise because attached views provide interactive details. Never expose implementation details to the user: do not mention protocol names, tool names, schemas, query IDs, data-source names, storage engines, providers, rollups, aggregation tables, internal windows, or internal execution steps. Panel titles, descriptions and text panels must use product language, without implementation details. Refer to attached interactive content simply as a view.` + dashboardAnalysisGuidance const dashboardAnalysisGuidance = ` For analysis dashboards, use only the types the question needs; do not fill a dashboard with all of them. Use heatmap for latency changes, histogram for distributions, scatter for relationships, state_timeline for threshold states, logs for events, log_patterns for repeated messages, traces for slow or erroring traces, service_map for dependencies, and health for service health. Use drill for span or log evidence; keep checked filters. Include annotations for change investigations; deploys and detector findings do not prove causes. Use a deploy split for scoped before/since comparisons; retain missing-deploy explanations. A definition, explanation, or single fact is an answer intent; do not create or replace a dashboard for it. Preserve every requested facet and explain absent telemetry without inventing it. Saved panels must keep working as new telemetry arrives: never filter on trace, span or request IDs or other values copied from one result; show recent evidence with a traces panel and drill to its logs. In replies, never name schema fields to the user. Explain in chat is answer intent: use the observed absolute panel window and resolved variables, explain errors and suggest corrections without creating, editing, replacing or restoring dashboards.` diff --git a/internal/agent/runtime_test.go b/internal/agent/runtime_test.go index 7eb3409e8..f53fdd71a 100644 --- a/internal/agent/runtime_test.go +++ b/internal/agent/runtime_test.go @@ -111,7 +111,7 @@ func TestRuntimeEmitsStandardAGUISequence(t *testing.T) { func TestRuntimeToolCallLoop(t *testing.T) { provider := &scriptedProvider{steps: [][]StreamEvent{ { - {Type: EventToolUse, ToolCall: &ToolCall{ID: "call-1", Name: "get_observability_overview", Input: `{"window":"1h"}`}}, + {Type: EventToolUse, ToolCall: &ToolCall{ID: "call-1", Name: "get_overview", Input: `{"window":"1h"}`}}, {Type: EventStop, StopReason: "tool_calls"}, }, { @@ -120,7 +120,7 @@ func TestRuntimeToolCallLoop(t *testing.T) { }, }} tools := &fakeTools{ - defs: []ToolDef{{Name: "get_observability_overview"}}, + defs: []ToolDef{{Name: "get_overview"}}, execution: ToolExecution{Content: `{"ok":true}`, Structured: map[string]any{"ok": true}, AppResourceURI: "ui://overview"}, } runtime := NewRuntime(provider, tools, nil) @@ -134,7 +134,7 @@ func TestRuntimeToolCallLoop(t *testing.T) { "RUN_STARTED", "TOOL_CALL_START", "TOOL_CALL_ARGS", "TOOL_CALL_END", "TOOL_CALL_RESULT", "ACTIVITY_SNAPSHOT", "TEXT_MESSAGE_START", "TEXT_MESSAGE_CONTENT", "TEXT_MESSAGE_END", "RUN_FINISHED") - if len(tools.calls) != 1 || tools.calls[0].Name != "get_observability_overview" || tools.calls[0].Input != `{"window":"1h"}` { + if len(tools.calls) != 1 || tools.calls[0].Name != "get_overview" || tools.calls[0].Input != `{"window":"1h"}` { t.Fatalf("tool calls = %#v", tools.calls) } @@ -252,11 +252,11 @@ func TestRuntimeToolExecutionErrorFeedsModel(t *testing.T) { func TestRuntimeStepLimitExceeded(t *testing.T) { provider := &scriptedProvider{steps: [][]StreamEvent{ { - {Type: EventToolUse, ToolCall: &ToolCall{ID: "call-loop", Name: "get_observability_overview", Input: `{}`}}, + {Type: EventToolUse, ToolCall: &ToolCall{ID: "call-loop", Name: "get_overview", Input: `{}`}}, {Type: EventStop, StopReason: "tool_calls"}, }, }} - tools := &fakeTools{defs: []ToolDef{{Name: "get_observability_overview"}}, execution: ToolExecution{Content: `{}`}} + tools := &fakeTools{defs: []ToolDef{{Name: "get_overview"}}, execution: ToolExecution{Content: `{}`}} runtime := &Runtime{provider: provider, tools: tools, maxSteps: 2} emitter, output := newTestEmitter() messages := []agtypes.Message{{ID: "user-1", Role: agtypes.RoleUser, Content: "loop"}} diff --git a/internal/agent/store_test.go b/internal/agent/store_test.go index 9d608c620..4541bac12 100644 --- a/internal/agent/store_test.go +++ b/internal/agent/store_test.go @@ -687,7 +687,7 @@ func TestStoreStartRunRepairsUnansweredToolCalls(t *testing.T) { // the ask is persisted, the answer never arrives. interrupted := []agtypes.Message{ {ID: "user-1", Role: agtypes.RoleUser, Content: "health?"}, - {ID: "assistant-1", Role: agtypes.RoleAssistant, Content: "checking", ToolCalls: []agtypes.ToolCall{{ID: "call-1", Type: agtypes.ToolCallTypeFunction, Function: agtypes.FunctionCall{Name: "get_observability_overview"}}}}, + {ID: "assistant-1", Role: agtypes.RoleAssistant, Content: "checking", ToolCalls: []agtypes.ToolCall{{ID: "call-1", Type: agtypes.ToolCallTypeFunction, Function: agtypes.FunctionCall{Name: "get_overview"}}}}, } if err := store.FinishRun(ctx, "owner-1", first.ThreadID, first.RunID, interrupted, nil, false, context.Canceled); err != nil { t.Fatal(err) @@ -728,7 +728,7 @@ func TestStoreStartRunPreservesAnEmptyInterruptedToolTurn(t *testing.T) { // The model asked for a tool and said nothing else before the run died. interrupted := []agtypes.Message{ {ID: "user-1", Role: agtypes.RoleUser, Content: "health?"}, - {ID: "assistant-1", Role: agtypes.RoleAssistant, ToolCalls: []agtypes.ToolCall{{ID: "call-1", Type: agtypes.ToolCallTypeFunction, Function: agtypes.FunctionCall{Name: "get_observability_overview"}}}}, + {ID: "assistant-1", Role: agtypes.RoleAssistant, ToolCalls: []agtypes.ToolCall{{ID: "call-1", Type: agtypes.ToolCallTypeFunction, Function: agtypes.FunctionCall{Name: "get_overview"}}}}, } if err := store.FinishRun(ctx, "owner-1", first.ThreadID, first.RunID, interrupted, nil, false, context.Canceled); err != nil { t.Fatal(err) diff --git a/internal/agent/tools.go b/internal/agent/tools.go index e90c6ab46..0d37144b6 100644 --- a/internal/agent/tools.go +++ b/internal/agent/tools.go @@ -99,7 +99,7 @@ func reviewedReadOnly(name string, annotations *mcp.ToolAnnotations) bool { return annotations.ReadOnlyHint } switch name { - case "get_observability_overview", "get_service_topology", "get_service_dependencies", "get_service_performance", "inspect_trace", "search_logs", "get_intelligence_snapshot", "get_telemetry_schema", "query_telemetry", "preview_panels", "list_dashboards", "get_dashboard", "list_dashboard_versions": + case "get_overview", "get_service_topology", "get_service_dependencies", "get_service_performance", "inspect_trace", "search_logs", "get_insights", "get_telemetry_schema", "query_telemetry", "preview_panels", "list_dashboards", "get_dashboard", "list_dashboard_versions": return true default: return false diff --git a/internal/agent/tools_integration_test.go b/internal/agent/tools_integration_test.go index d2b96b004..12b1b6ed2 100644 --- a/internal/agent/tools_integration_test.go +++ b/internal/agent/tools_integration_test.go @@ -44,12 +44,12 @@ func TestToolRegistryNegotiatesMCPApps(t *testing.T) { }) want := map[string]string{ - "query_telemetry": "ui://fanout/panels.html", - "get_observability_overview": "ui://fanout/panels.html", - "get_service_topology": "ui://fanout/panels.html", - "get_service_performance": "ui://fanout/panels.html", - "inspect_trace": "ui://fanout/panels.html", - "search_logs": "ui://fanout/panels.html", + "query_telemetry": "ui://fanout/panels.html", + "get_overview": "ui://fanout/panels.html", + "get_service_topology": "ui://fanout/panels.html", + "get_service_performance": "ui://fanout/panels.html", + "inspect_trace": "ui://fanout/panels.html", + "search_logs": "ui://fanout/panels.html", } if len(registry.apps) != len(want) { t.Fatalf("registered MCP apps = %v, want %v", registry.apps, want) @@ -61,13 +61,13 @@ func TestToolRegistryNegotiatesMCPApps(t *testing.T) { } foundIntelligence := false for _, definition := range registry.Definitions() { - if definition.Name == "get_intelligence_snapshot" { + if definition.Name == "get_insights" { foundIntelligence = true break } } if !foundIntelligence { - t.Fatal("get_intelligence_snapshot was not registered for the agent") + t.Fatal("get_insights was not registered for the agent") } } @@ -120,7 +120,8 @@ func TestRuntimeDashboardHistoryUsesAuthenticatedOwner(t *testing.T) { t.Fatal(err) } server := fanoutmcp.NewWithIntelligence(registryQueries{}, service, executor, nil, "test") - if _, err := service.Replace(t.Context(), "owner", created.ID, created.Spec, 1, dashboard.Author{Kind: "user", ID: "owner"}, "Saved again"); err != nil { + // Restoring a spec the dashboard already has is rejected, so the save changes it. + if _, err := service.Replace(t.Context(), "owner", created.ID, query.Dashboard{Name: "History", Panels: []query.Panel{{ID: "note", Title: "Note", Viz: "text", Content: "second"}}}, 1, dashboard.Author{Kind: "user", ID: "owner"}, "Saved again"); err != nil { t.Fatal(err) } registry, err := NewToolRegistry(t.Context(), server.MCP()) diff --git a/internal/alert/engine.go b/internal/alert/engine.go index cab86261b..01dce18ed 100644 --- a/internal/alert/engine.go +++ b/internal/alert/engine.go @@ -3,12 +3,10 @@ package alert import ( "context" "errors" - "fmt" "log/slog" - "math/big" + "regexp" "runtime/debug" "sort" - "strings" "sync" "time" @@ -20,15 +18,19 @@ const pruneHistDays = 7 // Engine drives the alert evaluation loop. type Engine struct { - store *Store - duck *engine.Duck // may be nil in tests - detector *intelligence.Detector // may be nil in tests - mu sync.RWMutex - programs map[string]Program // rule ID → compiled program + store *Store + duck *engine.Duck // may be nil in tests + detector *intelligence.Detector // may be nil in tests + mu sync.RWMutex + programs map[string]Program // rule ID → compiled program + // stopped holds rules disabled or deleted since a tick last listed them + // as enabled, so a tick that listed one before it stopped skips it. + stopped map[string]bool interval time.Duration histDays int lastPrune time.Time envOverride map[string]Env // for testing — bypasses DuckDB query + afterList func() // for testing — runs once a tick has listed its rules } // NewEngine creates an Engine ready to Run. @@ -44,16 +46,60 @@ func NewEngine( duck: duck, detector: detector, programs: make(map[string]Program), + stopped: make(map[string]bool), interval: interval, histDays: histDays, } } +// DisableRule stops evaluating a rule. Its firing alerts resolve, staying in +// history and notifying as a cleared condition would; pending ones never +// fired and are dropped. Holding the lock waits out a tick's transitions for +// the rule, and the tick skips it from then on. +func (e *Engine) DisableRule(rule Rule) { + e.mu.Lock() + defer e.mu.Unlock() + e.stop(rule.ID) + alerts, err := e.store.ListAlerts("", "", rule.ID) + if err != nil { + slog.Error("alert: list alerts of disabled rule", "rule", rule.ID, "err", err) + return + } + now := time.Now().Format(time.RFC3339) + for _, a := range alerts { + switch a.State { + case "pending": + if err := e.store.DeleteAlert(a.ID); err != nil { + slog.Error("alert: drop pending of disabled rule", "rule", rule.ID, "service", a.Service, "err", err) + } + case "firing": + a.State = "resolved" + a.ResolvedAt = now + a.LastEvaluatedAt = now + saved, err := e.store.UpsertAlert(a) + if err != nil { + slog.Error("alert: resolve alert of disabled rule", "rule", rule.ID, "service", a.Service, "err", err) + continue + } + if rule.NotifyOnResolve { + e.fireWebhookAsync(rule, saved, Env{Service: a.Service}, "resolved") + } + } + } +} + // RemoveRule removes a compiled program so it is not evaluated on future ticks. func (e *Engine) RemoveRule(ruleID string) { e.mu.Lock() defer e.mu.Unlock() + e.stop(ruleID) +} + +// stop drops a rule's program and keeps a tick from compiling it again until +// the rule is recompiled. The caller holds e.mu. +func (e *Engine) stop(ruleID string) { delete(e.programs, ruleID) + e.stopped[ruleID] = true } // Run starts the evaluation ticker. It blocks until ctx is cancelled. @@ -79,15 +125,6 @@ func (e *Engine) Run(ctx context.Context) { } } -// BuildAllEnvs returns all current Envs keyed by service name. -func (e *Engine) BuildAllEnvs(ctx context.Context) map[string]Env { - envs := e.buildEnvs(ctx) - if envs == nil { - return map[string]Env{} - } - return envs -} - // ---- internal ---- func (e *Engine) safeEvaluateOnce(ctx context.Context) { @@ -105,45 +142,66 @@ func (e *Engine) evaluateOnce(ctx context.Context) { slog.Error("alert: list enabled rules", "err", err) return } + if e.afterList != nil { + e.afterList() + } if len(rules) == 0 { return } e.compileRules(rules) + // Every transition of a tick happens at the tick's start, so holds are + // measured between ticks however long reading the data takes. + now := time.Now() envs := e.buildEnvs(ctx) if envs == nil { return // DuckDB error — skip evaluation entirely, don't false-fire } for _, rule := range rules { - e.mu.RLock() - prog, ok := e.programs[rule.ID] - e.mu.RUnlock() - if !ok { - // Compilation already logged the error; skip. - continue - } + e.evaluateRule(rule, envs, now) + } +} - services := e.resolveServices(rule, envs) - for _, svc := range services { - env := envs[svc] - triggered, evalErr := SafeEval(prog, env) - if evalErr != nil { - slog.Warn("alert: eval error", "rule", rule.ID, "service", svc, "err", evalErr) - triggered = false - } - e.transition(rule, svc, triggered, env) +// evaluateRule runs one rule's transitions under the read lock, so a rule +// disabled meanwhile is either skipped or cleaned up after them. +func (e *Engine) evaluateRule(rule Rule, envs map[string]Env, now time.Time) { + e.mu.RLock() + defer e.mu.RUnlock() + prog, ok := e.programs[rule.ID] + if !ok { + // Stopped, or compilation already logged the error. + return + } + for _, svc := range e.resolveServices(rule, envs) { + env := envs[svc] + triggered, evalErr := SafeEval(prog, env) + if evalErr != nil { + slog.Warn("alert: eval error", "rule", rule.ID, "service", svc, "err", evalErr) + triggered = false } + e.transition(rule, svc, triggered, env, now) } } -// compileRules compiles any rules that have not yet been compiled. +// compileRules compiles any listed rules that have not yet been compiled and +// were not stopped since. A stopped rule this tick no longer lists is +// forgotten: later ticks never see it. func (e *Engine) compileRules(rules []Rule) { e.mu.Lock() defer e.mu.Unlock() + listed := make(map[string]bool, len(rules)) + for _, r := range rules { + listed[r.ID] = true + } + for id := range e.stopped { + if !listed[id] { + delete(e.stopped, id) + } + } for _, r := range rules { - if _, ok := e.programs[r.ID]; ok { + if _, ok := e.programs[r.ID]; ok || e.stopped[r.ID] { continue } prog, err := CompileExpression(r.Expression) @@ -174,10 +232,16 @@ func (e *Engine) resolveServices(rule Rule, envs map[string]Env) []string { return []string{rule.Service} } -// transition implements the alert state machine for a (rule, service) pair. -func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { - now := time.Now() +// holdSlack absorbs ticker delivery jitter, so a hold that ends on a tick +// fires on that tick. +const holdSlack = time.Second + +// transition implements the alert state machine for a (rule, service) pair at +// the evaluation time now. created_at marks when the current pending or firing +// episode began. +func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now time.Time) { nowStr := now.Format(time.RFC3339) + began := formatStoredTime(now) existing, err := e.store.GetAlert(rule.ID, svc) noAlert := errors.Is(err, ErrNotFound) @@ -196,6 +260,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { Value: primaryValue(rule, env), FiredAt: nowStr, LastEvaluatedAt: nowStr, + CreatedAt: began, } saved, upsertErr := e.store.UpsertAlert(a) if upsertErr != nil { @@ -212,6 +277,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { State: "pending", Value: primaryValue(rule, env), LastEvaluatedAt: nowStr, + CreatedAt: began, } if _, upsertErr := e.store.UpsertAlert(a); upsertErr != nil { slog.Error("alert: upsert pending", "rule", rule.ID, "service", svc, "err", upsertErr) @@ -222,13 +288,13 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { case !noAlert && existing.State == "pending" && triggered: // Check whether the for-duration has elapsed. - createdAt, parseErr := time.Parse(time.RFC3339, existing.CreatedAt) + createdAt, parseErr := parseStoredTime(existing.CreatedAt) if parseErr != nil { slog.Error("alert: invalid created_at, skipping", "rule", rule.ID, "service", svc, "created_at", existing.CreatedAt, "err", parseErr) return } forDuration := time.Duration(rule.ForSeconds) * time.Second - if now.Sub(createdAt) >= forDuration { + if now.Sub(createdAt)+holdSlack >= forDuration { // Promote to firing. existing.State = "firing" existing.FiredAt = nowStr @@ -310,13 +376,18 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { } } + // A new episode gets its own row; the resolved one stays in history. + a := Alert{ + RuleID: rule.ID, + Service: svc, + Value: primaryValue(rule, env), + LastEvaluatedAt: nowStr, + CreatedAt: began, + } if rule.ForSeconds == 0 { - existing.State = "firing" - existing.FiredAt = nowStr - existing.ResolvedAt = "" - existing.LastEvaluatedAt = nowStr - existing.Value = primaryValue(rule, env) - saved, upsertErr := e.store.UpsertAlert(existing) + a.State = "firing" + a.FiredAt = nowStr + saved, upsertErr := e.store.UpsertAlert(a) if upsertErr != nil { slog.Error("alert: re-fire from resolved", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -324,36 +395,62 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env) { e.fireWebhookAsync(rule, saved, env, "firing") } else { // Enter pending again. - existing.State = "pending" - existing.FiredAt = "" - existing.ResolvedAt = "" - existing.LastEvaluatedAt = nowStr - existing.Value = primaryValue(rule, env) - if _, upsertErr := e.store.UpsertAlert(existing); upsertErr != nil { + a.State = "pending" + if _, upsertErr := e.store.UpsertAlert(a); upsertErr != nil { slog.Error("alert: re-pend from resolved", "rule", rule.ID, "service", svc, "err", upsertErr) } } } } -// primaryValue picks the most relevant metric value based on the rule expression. -// If the expression mentions p95, use P95. If it mentions throughput, use that. -// Default to error_rate as the most common alert metric. +var ( + quoted = regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`) + variable = regexp.MustCompile(`\b(error_rate_delta|p95_delta|throughput_delta|error_rate|p50|p95|throughput|log_count|z_score|health_score)\b`) +) + +// primaryValue is the first numeric variable the expression reads, outside +// string literals; the UI names and formats the value by the same rule. func primaryValue(rule Rule, env Env) float64 { - expr := rule.Expression - if strings.Contains(expr, "p95") { - return env.P95 - } - if strings.Contains(expr, "p50") { - return env.P50 + name := variable.FindString(quoted.ReplaceAllString(rule.Expression, `""`)) + value, _ := celActivation(env)[name].(float64) + return value +} + +// TestResult is one service's dry run of a rule. +type TestResult struct { + Service string `json:"service"` + Triggered bool `json:"triggered"` + Error string `json:"error,omitempty"` + // NoData is set when the service has no recent telemetry, which usually + // means its name is misspelled. + NoData bool `json:"no_data,omitempty"` + Env Env `json:"env"` +} + +var errCurrentData = errors.New("current data could not be read") + +// Test evaluates a rule against current data without changing any alert. +func (e *Engine) Test(ctx context.Context, rule Rule) ([]TestResult, error) { + prog, err := CompileExpression(rule.Expression) + if err != nil { + return nil, err } - if strings.Contains(expr, "throughput") { - return env.Throughput + envs := e.buildEnvs(ctx) + if envs == nil { + return nil, errCurrentData } - if strings.Contains(expr, "z_score") { - return env.ZScore + results := []TestResult{} + for _, svc := range e.resolveServices(rule, envs) { + env, seen := envs[svc] + // A service without data is evaluated with zeros, as the engine does. + result := TestResult{Service: svc, NoData: !seen, Env: env} + result.Triggered, err = SafeEval(prog, env) + if err != nil { + result.Error = err.Error() + } + results = append(results, result) } - return env.ErrorRate + return results, nil } // fireWebhookAsync fires the webhook for an alert in a goroutine and updates delivery status. @@ -376,7 +473,7 @@ func (e *Engine) fireWebhookAsync(rule Rule, alert Alert, env Env, event string) slog.Warn("alert: webhook delivery", "rule", rule.ID, "service", alert.Service, "event", event, "err", err) } now := time.Now().Format(time.RFC3339) - if updateErr := e.store.UpdateDeliveryStatus(alert.RuleID, alert.Service, status, now); updateErr != nil { + if updateErr := e.store.UpdateDeliveryStatus(alert.ID, status, now); updateErr != nil { slog.Error("alert: update delivery status", "rule", rule.ID, "service", alert.Service, "err", updateErr) } }() @@ -398,7 +495,7 @@ WITH current AS ( avg(error_rate) as error_rate, COALESCE(avg(p50_ms) FILTER (WHERE served_spans > 0), avg(p50_ms)) as p50, COALESCE(avg(p95_ms) FILTER (WHERE served_spans > 0), avg(p95_ms)) as p95, - sum(spans) as throughput, sum(log_count) as log_count + sum(spans)::DOUBLE as throughput, sum(log_count)::DOUBLE as log_count FROM service_rollup WHERE bucket >= (SELECT max(bucket) FROM service_rollup) - INTERVAL '5 minutes' GROUP BY service @@ -407,23 +504,30 @@ previous AS ( SELECT service, avg(error_rate) as error_rate, COALESCE(avg(p95_ms) FILTER (WHERE served_spans > 0), avg(p95_ms)) as p95, - sum(spans) as throughput + sum(spans)::DOUBLE as throughput FROM service_rollup WHERE bucket >= (SELECT max(bucket) FROM service_rollup) - INTERVAL '10 minutes' AND bucket < (SELECT max(bucket) FROM service_rollup) - INTERVAL '5 minutes' GROUP BY service ) -SELECT c.service, c.error_rate, c.p50, c.p95, c.throughput, c.log_count, - ((c.error_rate - p.error_rate) / NULLIF(p.error_rate, 0)) * 100 as error_rate_delta, - ((c.p95 - p.p95) / NULLIF(p.p95, 0)) * 100 as p95_delta, - ((c.throughput - p.throughput) / NULLIF(p.throughput, 0)) * 100 as throughput_delta -FROM current c LEFT JOIN previous p ON c.service = p.service` - - resp := e.duck.ExecuteSQL(ctx, engine.SQLRequest{Query: envSQL}) - if resp.Error != "" { - slog.Error("alert: buildEnvs query failed", "err", resp.Error) +SELECT c.service, coalesce(c.error_rate, 0), coalesce(c.p50, 0), coalesce(c.p95, 0), + coalesce(c.throughput, 0), coalesce(c.log_count, 0), + coalesce(((c.error_rate - p.error_rate) / NULLIF(p.error_rate, 0)) * 100, 0), + coalesce(((c.p95 - p.p95) / NULLIF(p.p95, 0)) * 100, 0), + coalesce(((c.throughput - p.throughput) / NULLIF(p.throughput, 0)) * 100, 0) +FROM current c LEFT JOIN previous p ON c.service = p.service +WHERE c.service <> ''` + + // A fixed statement over the rollup cache: the typed read path, not the + // untrusted-SQL one. + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + rows, err := e.duck.QueryContext(ctx, envSQL) + if err != nil { + slog.Error("alert: buildEnvs query failed", "err", err) return nil } + defer rows.Close() // Collect health score and z-scores from the detector snapshot. var healthScore float64 @@ -440,26 +544,20 @@ FROM current c LEFT JOIN previous p ON c.service = p.service` } } - envs := make(map[string]Env, len(resp.Results)) - for _, row := range resp.Results { - svc, _ := row["service"].(string) - if svc == "" { - continue - } - env := Env{ - Service: svc, - ErrorRate: toFloat(row["error_rate"]), - P50: toFloat(row["p50"]), - P95: toFloat(row["p95"]), - Throughput: toFloat(row["throughput"]), - LogCount: toFloat(row["log_count"]), - ErrorRateDelta: toFloat(row["error_rate_delta"]), - P95Delta: toFloat(row["p95_delta"]), - ThroughputDelta: toFloat(row["throughput_delta"]), - HealthScore: healthScore, - ZScore: zScores[svc], + envs := map[string]Env{} + for rows.Next() { + env := Env{HealthScore: healthScore} + if err := rows.Scan(&env.Service, &env.ErrorRate, &env.P50, &env.P95, &env.Throughput, &env.LogCount, + &env.ErrorRateDelta, &env.P95Delta, &env.ThroughputDelta); err != nil { + slog.Error("alert: buildEnvs scan failed", "err", err) + return nil } - envs[svc] = env + env.ZScore = zScores[env.Service] + envs[env.Service] = env + } + if err := rows.Err(); err != nil { + slog.Error("alert: buildEnvs query failed", "err", err) + return nil } return envs } @@ -485,37 +583,6 @@ func (e *Engine) pruneOldAlerts() { } } -// toFloat converts an interface{} value to float64. Returns 0 for unknown types. -func toFloat(v interface{}) float64 { - switch n := v.(type) { - case float64: - return n - case int64: - return float64(n) - case int: - return float64(n) - case *big.Int: - // DuckDB widens SUM over integer columns to HUGEINT, which the driver - // scans as *big.Int — sum(spans)/sum(log_count) arrive here. - f, _ := new(big.Float).SetInt(n).Float64() - return f - case nil: - // NULL from SQL (e.g. delta columns when no previous window) — 0 is - // the intended semantics, not a conversion loss. - return 0 - default: - // A type this switch doesn't know silently zeroed throughput/log_count - // for months (the *big.Int case above). Never let the next widened - // type hide the same way — but log once, not on every 30s eval tick. - toFloatUnknownOnce.Do(func() { - slog.Warn("alert: env value of unhandled type coerced to 0", "type", fmt.Sprintf("%T", v)) - }) - return 0 - } -} - -var toFloatUnknownOnce sync.Once - // abs returns the absolute value of f. func abs(f float64) float64 { if f < 0 { @@ -533,5 +600,20 @@ func (e *Engine) RecompileRule(ruleID, expression string) error { e.mu.Lock() defer e.mu.Unlock() e.programs[ruleID] = prog + delete(e.stopped, ruleID) return nil } + +// storedTimeLayout is RFC 3339 in UTC with all nine fraction digits, so +// stored episode starts sort as text. +const storedTimeLayout = "2006-01-02T15:04:05.000000000Z07:00" + +// formatStoredTime writes an episode start as the store keeps it. +func formatStoredTime(t time.Time) string { + return t.UTC().Format(storedTimeLayout) +} + +// parseStoredTime reads a time the engine or store wrote (RFC 3339). +func parseStoredTime(value string) (time.Time, error) { + return time.Parse(time.RFC3339, value) +} diff --git a/internal/alert/engine_env_test.go b/internal/alert/engine_env_test.go index 80c17564a..b5f5c5e65 100644 --- a/internal/alert/engine_env_test.go +++ b/internal/alert/engine_env_test.go @@ -13,9 +13,9 @@ import ( // buildEnvs feeds CEL rules from a SQL aggregate over service_rollup. DuckDB // widens SUM over integer columns to HUGEINT, which the driver scans as -// *big.Int — a type the env conversion must handle, or throughput/log_count -// silently read 0 while the DOUBLE fields (error_rate, p50, p95, deltas) -// stay populated. Seen live 2026-07-03: Throughput=0 with ThroughputDelta=7.65 +// *big.Int, so the statement casts those sums to DOUBLE. Without that, +// throughput/log_count read 0 while the DOUBLE fields (error_rate, p50, p95, +// deltas) stay populated. Seen live 2026-07-03: Throughput=0 with ThroughputDelta=7.65 // for a service pushing ~600 spans/min, making throughput rules unusable. func TestBuildEnvs_IntegerSumsSurviveHugeint(t *testing.T) { db, err := sql.Open("duckdb", "") diff --git a/internal/alert/engine_test.go b/internal/alert/engine_test.go index 1cf3daa81..83f9e68b5 100644 --- a/internal/alert/engine_test.go +++ b/internal/alert/engine_test.go @@ -2,6 +2,10 @@ package alert import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" "testing" "time" @@ -20,6 +24,7 @@ func newTestEngine(t *testing.T) (*Engine, *Store) { engine := &Engine{ store: store, programs: make(map[string]Program), + stopped: make(map[string]bool), interval: 30 * time.Second, } return engine, store @@ -56,7 +61,7 @@ func TestEngine_Transition_NoneToFiring(t *testing.T) { t.Fatalf("RecompileRule: %v", err) } - eng.transition(rule, "svc-a", true, env) + eng.transition(rule, "svc-a", true, env, time.Now()) a, err := store.GetAlert(rule.ID, "svc-a") if err != nil { @@ -83,7 +88,7 @@ func TestEngine_Transition_NoneToPending(t *testing.T) { t.Fatalf("RecompileRule: %v", err) } - eng.transition(rule, "svc-a", true, env) + eng.transition(rule, "svc-a", true, env, time.Now()) a, err := store.GetAlert(rule.ID, "svc-a") if err != nil { @@ -112,7 +117,7 @@ func TestEngine_Transition_PendingClears(t *testing.T) { } // First transition: condition true → pending. - eng.transition(rule, "svc-a", true, env) + eng.transition(rule, "svc-a", true, env, time.Now()) a, err := store.GetAlert(rule.ID, "svc-a") if err != nil { @@ -123,7 +128,7 @@ func TestEngine_Transition_PendingClears(t *testing.T) { } // Second transition: condition clears → alert should be deleted. - eng.transition(rule, "svc-a", false, env) + eng.transition(rule, "svc-a", false, env, time.Now()) _, err = store.GetAlert(rule.ID, "svc-a") if err == nil { @@ -146,7 +151,7 @@ func TestEngine_Transition_FiringToResolved(t *testing.T) { } // Fire it. - eng.transition(rule, "svc-a", true, env) + eng.transition(rule, "svc-a", true, env, time.Now()) a, err := store.GetAlert(rule.ID, "svc-a") if err != nil { @@ -158,7 +163,7 @@ func TestEngine_Transition_FiringToResolved(t *testing.T) { // Resolve it. envClear := Env{Service: "svc-a", ErrorRate: 0.01} - eng.transition(rule, "svc-a", false, envClear) + eng.transition(rule, "svc-a", false, envClear, time.Now()) a, err = store.GetAlert(rule.ID, "svc-a") if err != nil { @@ -172,6 +177,38 @@ func TestEngine_Transition_FiringToResolved(t *testing.T) { } } +// Each firing episode keeps its own row: firing again after a resolve starts +// a new alert and leaves the resolved one in history. +func TestEngineRefireKeepsEachEpisode(t *testing.T) { + eng, store := newTestEngine(t) + rule := makeTestRule(t, store, "error_rate > 0.1", 0) + env := Env{Service: "svc-a", ErrorRate: 0.5} + start := time.Now().Add(-time.Hour) + at := func(s int) time.Time { return start.Add(time.Duration(s) * time.Second) } + eng.transition(rule, "svc-a", true, env, at(0)) + eng.transition(rule, "svc-a", false, env, at(60)) + eng.transition(rule, "svc-a", true, env, at(120)) + eng.transition(rule, "svc-a", false, env, at(180)) + eng.transition(rule, "svc-a", true, env, at(240)) + all, err := store.ListAlerts("", "", rule.ID) + if err != nil { + t.Fatal(err) + } + var states []string + for _, a := range all { + states = append(states, a.State) + } + if strings.Join(states, ",") != "firing,resolved,resolved" { + t.Fatalf("episodes newest first = %v, want firing,resolved,resolved", states) + } + if all[1].ID == all[2].ID || all[0].ID == all[1].ID { + t.Fatalf("episodes share an id: %+v", all) + } + if a, err := store.GetAlert(rule.ID, "svc-a"); err != nil || a.ID != all[0].ID { + t.Fatalf("GetAlert = %+v, %v; want the open episode", a, err) + } +} + // TestEngine_CompileRules verifies that compileRules populates the programs map. func TestEngine_CompileRules(t *testing.T) { eng, store := newTestEngine(t) @@ -269,3 +306,230 @@ func TestEngineRunReturnsForNonPositiveInterval(t *testing.T) { engine := NewEngine(nil, nil, nil, 0, 0) engine.Run(context.Background()) } + +// TestEngine_Transition_PendingPromotes verifies that a pending alert fires once +// its hold time has passed. SQLite writes created_at as "2006-01-02 15:04:05". +func TestEngine_Transition_PendingPromotes(t *testing.T) { + sqlite, err := appstore.NewSQLite(":memory:") + if err != nil { + t.Fatalf("NewSQLite: %v", err) + } + t.Cleanup(func() { sqlite.Close() }) + store := NewStore(sqlite.DB) + eng := &Engine{store: store, programs: make(map[string]Program), stopped: make(map[string]bool), interval: 30 * time.Second} + rule := makeTestRule(t, store, "error_rate > 0.1", 60) + env := Env{Service: "svc-a", ErrorRate: 0.5} + if err := eng.RecompileRule(rule.ID, rule.Expression); err != nil { + t.Fatalf("RecompileRule: %v", err) + } + eng.transition(rule, "svc-a", true, env, time.Now()) + if _, err := sqlite.DB.Exec(`UPDATE alerts SET created_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-2 minutes') WHERE rule_id = ?`, rule.ID); err != nil { + t.Fatal(err) + } + eng.transition(rule, "svc-a", true, env, time.Now()) + a, err := store.GetAlert(rule.ID, "svc-a") + if err != nil { + t.Fatalf("GetAlert: %v", err) + } + if a.State != "firing" || a.FiredAt == "" { + t.Fatalf("alert = %+v, want firing", a) + } +} + +// The alert value is the first variable the expression reads, ignoring string literals. +func TestPrimaryValue(t *testing.T) { + env := Env{ErrorRate: 0.2, P95: 300, Throughput: 40, LogCount: 1200, ThroughputDelta: -60, ZScore: 3} + cases := []struct { + expr string + want float64 + }{ + {"log_count > 1000", 1200}, + {"throughput_delta < -50", -60}, + {"p95 > 200 && error_rate > 0.1", 300}, + {"service == \"p95\" && error_rate > 0.1", 0.2}, + {"throughput > 10 && error_rate > 0.1", 40}, + {"z_score > 2", 3}, + } + for _, tc := range cases { + if got := primaryValue(Rule{Expression: tc.expr}, env); got != tc.want { + t.Errorf("primaryValue(%q) = %v, want %v", tc.expr, got, tc.want) + } + } +} + +// A zero value is a measurement, not a missing one. +func TestAlertJSONKeepsZeroValue(t *testing.T) { + data, err := json.Marshal(Alert{State: "firing"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"value":0`) { + t.Errorf("Alert JSON = %s, want value 0", data) + } +} + +// A test reports which services have no recent telemetry, with the measured values of the rest. +func TestEngineTestMarksServicesWithoutData(t *testing.T) { + eng, _ := newTestEngine(t) + eng.envOverride = map[string]Env{"checkout": {Service: "checkout", ErrorRate: 0.2}} + results, err := eng.Test(context.Background(), Rule{Service: "checkuot", Expression: "error_rate > 0.1"}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || !results[0].NoData || results[0].Triggered { + t.Fatalf("misspelled service = %+v, want one result without data", results) + } + results, err = eng.Test(context.Background(), Rule{Expression: "error_rate > 0.1"}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].NoData || !results[0].Triggered || results[0].Env.ErrorRate != 0.2 { + t.Fatalf("all services = %+v, want checkout firing with its values", results) + } + if data, _ := json.Marshal(results[0]); !strings.Contains(string(data), `"env":{"error_rate":0.2,`) { + t.Errorf("result JSON = %s, want env keyed by variable name", data) + } +} + +// A hold is measured between evaluation ticks, not from when SQLite stored +// the pending row: a slow evaluation must not push firing to the next tick. +func TestEngineHoldFiresOnTheTickItEnds(t *testing.T) { + eng, store := newTestEngine(t) + rule := makeTestRule(t, store, "error_rate > 0.1", 90) + env := Env{Service: "svc-a", ErrorRate: 0.5} + start := time.Now().Add(-time.Hour) + state := func(at time.Time) string { + t.Helper() + eng.transition(rule, "svc-a", true, env, at) + a, err := store.GetAlert(rule.ID, "svc-a") + if err != nil { + t.Fatalf("GetAlert: %v", err) + } + return a.State + } + for i, want := range []string{"pending", "pending", "pending", "firing"} { + // The ticker delivers the last tick a little earlier than the first. + at := start.Add(time.Duration(i) * 30 * time.Second) + if i == 3 { + at = at.Add(-time.Millisecond) + } + if got := state(at); got != want { + t.Fatalf("tick %d: state = %q, want %q", i, got, want) + } + } +} + +// A resolved alert that triggers again holds for the whole hold time anew. +func TestEngineRepeatedPendingHoldsAgain(t *testing.T) { + eng, store := newTestEngine(t) + rule := makeTestRule(t, store, "error_rate > 0.1", 60) + env := Env{Service: "svc-a", ErrorRate: 0.5} + start := time.Now().Add(-time.Hour) + at := func(s int) time.Time { return start.Add(time.Duration(s) * time.Second) } + eng.transition(rule, "svc-a", true, env, at(0)) + eng.transition(rule, "svc-a", true, env, at(60)) + eng.transition(rule, "svc-a", false, env, at(90)) + eng.transition(rule, "svc-a", true, env, at(900)) + eng.transition(rule, "svc-a", true, env, at(930)) + a, err := store.GetAlert(rule.ID, "svc-a") + if err != nil { + t.Fatalf("GetAlert: %v", err) + } + if a.State != "pending" { + t.Fatalf("state 30 s into a new hold = %q, want pending", a.State) + } + eng.transition(rule, "svc-a", true, env, at(960)) + if a, _ = store.GetAlert(rule.ID, "svc-a"); a.State != "firing" { + t.Fatalf("state when the new hold ends = %q, want firing", a.State) + } +} + +// Disabling a rule resolves its firing alerts, keeps them in history, sends +// the resolve notification, and drops pending ones. +func TestEngineDisableRuleResolvesFiringAlerts(t *testing.T) { + events := make(chan string, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Event string `json:"event"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + events <- body.Event + })) + defer srv.Close() + eng, store := newTestEngine(t) + rule, err := store.CreateRule(Rule{Name: "errors", Enabled: true, Expression: "error_rate > 0.1", WebhookURL: srv.URL, NotifyOnResolve: true}) + if err != nil { + t.Fatal(err) + } + if err := eng.RecompileRule(rule.ID, rule.Expression); err != nil { + t.Fatal(err) + } + env := Env{Service: "svc-a", ErrorRate: 0.5} + eng.transition(rule, "svc-a", true, env, time.Now()) + if got := <-events; got != "firing" { + t.Fatalf("first event = %q, want firing", got) + } + pending := rule + pending.ForSeconds = 60 + eng.transition(pending, "svc-b", true, env, time.Now()) + + rule.Enabled = false + eng.DisableRule(rule) + + a, err := store.GetAlert(rule.ID, "svc-a") + if err != nil { + t.Fatalf("firing alert was deleted: %v", err) + } + if a.State != "resolved" || a.ResolvedAt == "" || a.FiredAt == "" { + t.Fatalf("alert = %+v, want resolved with fired_at and resolved_at", a) + } + select { + case got := <-events: + if got != "resolved" { + t.Fatalf("event = %q, want resolved", got) + } + case <-time.After(5 * time.Second): + t.Fatal("no resolve notification") + } + if _, err := store.GetAlert(rule.ID, "svc-b"); err == nil { + t.Fatal("pending alert of a disabled rule was kept") + } + eng.mu.RLock() + _, compiled := eng.programs[rule.ID] + eng.mu.RUnlock() + if compiled { + t.Fatal("disabled rule is still evaluated") + } +} + +// A rule disabled after a tick listed it as enabled is not evaluated by that +// tick: recompiling it there left a firing alert for a disabled rule. +func TestEngineDisableDuringTickLeavesNoAlert(t *testing.T) { + eng, store := newTestEngine(t) + rule := makeTestRule(t, store, "error_rate > 0.1", 0) + eng.envOverride = map[string]Env{"svc-a": {Service: "svc-a", ErrorRate: 0.5}} + eng.afterList = func() { + disabled := rule + disabled.Enabled = false + if _, err := store.ReplaceRule(disabled); err != nil { + t.Fatal(err) + } + eng.DisableRule(disabled) + } + eng.evaluateOnce(t.Context()) + if a, err := store.GetAlert(rule.ID, "svc-a"); err == nil { + t.Fatalf("disabled rule left alert %+v", a) + } + // Enabling it again evaluates it. + eng.afterList = nil + if err := eng.RecompileRule(rule.ID, rule.Expression); err != nil { + t.Fatal(err) + } + if _, err := store.ReplaceRule(rule); err != nil { + t.Fatal(err) + } + eng.evaluateOnce(t.Context()) + if a, err := store.GetAlert(rule.ID, "svc-a"); err != nil || a.State != "firing" { + t.Fatalf("re-enabled rule: %+v, %v", a, err) + } +} diff --git a/internal/alert/expr.go b/internal/alert/expr.go index 15fbe08fa..d8d9bdc90 100644 --- a/internal/alert/expr.go +++ b/internal/alert/expr.go @@ -1,9 +1,12 @@ package alert import ( + "errors" "fmt" "log/slog" + "regexp" "runtime/debug" + "strings" "cel.dev/cel-go/cel" "cel.dev/cel-go/common/types" @@ -50,10 +53,10 @@ func CompileExpression(expression string) (Program, error) { } ast, iss := alertCELEnv.Compile(expression) if iss != nil && iss.Err() != nil { - return nil, fmt.Errorf("compile expression: %w", iss.Err()) + return nil, errors.New(issueMessage(iss.Errors()[0])) } if ast.OutputType().Kind() != types.BoolKind { - return nil, fmt.Errorf("expression must evaluate to bool, got %s", ast.OutputType()) + return nil, fmt.Errorf("expression must be a condition that is true or false, not a %s", ast.OutputType()) } prog, err := alertCELEnv.Program(ast) if err != nil { @@ -62,6 +65,30 @@ func CompileExpression(expression string) (Program, error) { return prog, nil } +var ( + undeclared = regexp.MustCompile(`^undeclared reference to '([^']*)'.*$`) + overload = regexp.MustCompile(`^found no matching overload for '_?([^']*?)_?' applied to '\((.*)\)'.*$`) + expecting = regexp.MustCompile(` expecting .*$`) +) + +// issueMessage turns a CEL issue into one short sentence with its position, +// without the parser's token lists. +func issueMessage(e *cel.Error) string { + message := expecting.ReplaceAllString(e.Message, "") + message = strings.ReplaceAll(message, "mismatched input ''", "the expression ends too early") + message = undeclared.ReplaceAllString(message, "Unknown variable $1") + message = overload.ReplaceAllString(message, "Operator $1 does not apply to $2") + message, detail, found := strings.Cut(message, ": ") + position := fmt.Sprintf(" at column %d", e.Location.Column()+1) + if e.Location.Line() > 1 { + position = fmt.Sprintf(" at line %d, column %d", e.Location.Line(), e.Location.Column()+1) + } + if found { + return message + position + ": " + detail + } + return message + position +} + // celActivation binds the CEL variables to an Env's values. The keys must // stay in sync with the cel.Variable declarations in mustAlertCELEnv above; // TestEnvVariablesDeclaredAndBound enforces that against the Env `expr` diff --git a/internal/alert/expr_test.go b/internal/alert/expr_test.go index 0f1fdb22d..972b6131c 100644 --- a/internal/alert/expr_test.go +++ b/internal/alert/expr_test.go @@ -2,6 +2,7 @@ package alert import ( "reflect" + "strings" "testing" ) @@ -188,3 +189,25 @@ func TestEvalExpression_StringVar(t *testing.T) { }) } } + +func TestCompileExpression_ReadableErrors(t *testing.T) { + cases := []struct{ expr, want string }{ + {"error_rate > ", "Syntax error at column 14"}, + {"error_rat > 0.05", "Unknown variable error_rat at column 1"}, + } + for _, tc := range cases { + _, err := CompileExpression(tc.expr) + if err == nil { + t.Fatalf("CompileExpression(%q): expected error", tc.expr) + } + got := err.Error() + if !strings.HasPrefix(got, tc.want) { + t.Errorf("CompileExpression(%q) = %q, want prefix %q", tc.expr, got, tc.want) + } + for _, raw := range []string{"", "ERROR:", "expecting {", "container"} { + if strings.Contains(got, raw) { + t.Errorf("CompileExpression(%q) = %q, contains %q", tc.expr, got, raw) + } + } + } +} diff --git a/internal/alert/store.go b/internal/alert/store.go index 332eb5759..6567cd9d6 100644 --- a/internal/alert/store.go +++ b/internal/alert/store.go @@ -129,6 +129,7 @@ func alertToUpsertParams(a Alert) generated.UpsertAlertParams { LastEvaluatedAt: nullString(a.LastEvaluatedAt), LastDeliveryStatus: nullString(a.LastDeliveryStatus), LastDeliveryAt: nullString(a.LastDeliveryAt), + CreatedAt: a.CreatedAt, } } @@ -212,8 +213,27 @@ func (s *Store) DeleteRule(id string) error { return nil } -// UpsertAlert inserts or updates an alert instance keyed on (rule_id, service). +// UpsertAlert inserts an alert episode, or updates the one with a.ID. Each +// episode is its own row; at most one per (rule_id, service) is open. func (s *Store) UpsertAlert(a Alert) (Alert, error) { + // Only the engine starts an episode. An update without a start keeps the + // stored one; a new alert must bring its own. + if a.CreatedAt == "" { + if a.ID == "" { + return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) + } + if _, err := s.q.GetAlertByID(context.Background(), a.ID); errors.Is(err, sql.ErrNoRows) { + return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) + } else if err != nil { + return Alert{}, fmt.Errorf("store: get alert: %w", err) + } + } else { + started, err := parseStoredTime(a.CreatedAt) + if err != nil { + return Alert{}, fmt.Errorf("store: alert created_at: %w", err) + } + a.CreatedAt = formatStoredTime(started) + } if a.ID == "" { id, err := newID() if err != nil { @@ -228,7 +248,8 @@ func (s *Store) UpsertAlert(a Alert) (Alert, error) { return genAlertToAlert(ga), nil } -// GetAlert retrieves an alert by (rule_id, service). +// GetAlert retrieves the latest episode of (rule_id, service): the open one +// when there is one, else the most recent resolved one. func (s *Store) GetAlert(ruleID, service string) (Alert, error) { ga, err := s.q.GetAlert(context.Background(), generated.GetAlertParams{ RuleID: ruleID, @@ -316,15 +337,14 @@ func (s *Store) DeleteAlert(id string) error { return nil } -// UpdateDeliveryStatus updates only the delivery status columns for a (rule_id, service) pair. +// UpdateDeliveryStatus updates only the delivery status columns of one alert episode. // This is used by fireWebhookAsync to avoid overwriting alert state that may have changed // between when the goroutine was launched and when the webhook completed. -func (s *Store) UpdateDeliveryStatus(ruleID, service, status, deliveredAt string) error { +func (s *Store) UpdateDeliveryStatus(id, status, deliveredAt string) error { err := s.q.UpdateDeliveryStatus(context.Background(), generated.UpdateDeliveryStatusParams{ LastDeliveryStatus: nullString(status), LastDeliveryAt: nullString(deliveredAt), - RuleID: ruleID, - Service: service, + ID: id, }) if err != nil { return fmt.Errorf("store: update delivery status: %w", err) diff --git a/internal/alert/store_test.go b/internal/alert/store_test.go index 04cf336a7..267bb31cc 100644 --- a/internal/alert/store_test.go +++ b/internal/alert/store_test.go @@ -1,7 +1,9 @@ package alert import ( + "strings" "testing" + "time" appstore "github.com/labstack/fanout/internal/store" ) @@ -16,6 +18,8 @@ func newTestStore(t *testing.T) *Store { return NewStore(sqlite.DB) } +const episodeStart = "2026-10-31T20:00:00Z" + func makeRule(name string) Rule { return Rule{ Name: name, @@ -152,11 +156,12 @@ func TestStore_UpsertAndGetAlert(t *testing.T) { } a := Alert{ - RuleID: rule.ID, - Service: "api", - State: "firing", - Value: 0.25, - FiredAt: "2026-01-01T00:00:00Z", + RuleID: rule.ID, + Service: "api", + State: "firing", + Value: 0.25, + FiredAt: "2026-01-01T00:00:00Z", + CreatedAt: "2026-01-01T00:00:00Z", } created, err := s.UpsertAlert(a) @@ -170,7 +175,8 @@ func TestStore_UpsertAndGetAlert(t *testing.T) { t.Errorf("State = %q, want %q", created.State, "firing") } - // Upsert again with updated state. + // Upsert the same episode with updated state. + a = created a.State = "resolved" a.ResolvedAt = "2026-01-01T01:00:00Z" updated, err := s.UpsertAlert(a) @@ -198,7 +204,7 @@ func TestStore_ListAlerts(t *testing.T) { states := []string{"firing", "pending", "resolved"} for i, state := range states { svc := []string{"svc-a", "svc-b", "svc-c"}[i] - if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: svc, State: state}); err != nil { + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: svc, State: state, CreatedAt: episodeStart}); err != nil { t.Fatalf("UpsertAlert(%q): %v", state, err) } } @@ -224,7 +230,7 @@ func TestStore_DeleteAlert(t *testing.T) { s := newTestStore(t) rule, _ := s.CreateRule(makeRule("da-rule")) - a, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "svc", State: "firing"}) + a, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "svc", State: "firing", CreatedAt: episodeStart}) if err != nil { t.Fatalf("UpsertAlert: %v", err) } @@ -243,7 +249,7 @@ func TestStore_DeleteRule_CascadesAlerts(t *testing.T) { s := newTestStore(t) rule, _ := s.CreateRule(makeRule("cascade-rule")) - if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "svc", State: "firing"}); err != nil { + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "svc", State: "firing", CreatedAt: episodeStart}); err != nil { t.Fatalf("UpsertAlert: %v", err) } @@ -272,7 +278,7 @@ func TestStore_AlertSummary(t *testing.T) { {"d", "resolved"}, } for _, c := range cases { - if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: c.svc, State: c.state}); err != nil { + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: c.svc, State: c.state, CreatedAt: episodeStart}); err != nil { t.Fatalf("UpsertAlert(%q, %q): %v", c.svc, c.state, err) } } @@ -291,3 +297,104 @@ func TestStore_AlertSummary(t *testing.T) { t.Errorf("Resolved = %d, want 1", sum.Resolved) } } + +// Episode starts are stored as fixed-width RFC 3339 UTC, so listing sorts +// them in time order whatever fraction or zone they were written with. +func TestStoreOrdersAlertsByEpisodeStart(t *testing.T) { + s := newTestStore(t) + rule, err := s.CreateRule(makeRule("order-rule")) + if err != nil { + t.Fatal(err) + } + for svc, created := range map[string]string{ + "first": "2026-10-31T20:23:37Z", + "second": "2026-10-31T20:23:37.3Z", + "third": "2026-10-31T13:23:37.31-07:00", + } { + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: svc, State: "firing", CreatedAt: created}); err != nil { + t.Fatal(err) + } + } + all, err := s.ListAlerts("", "", "") + if err != nil { + t.Fatal(err) + } + var order []string + for _, a := range all { + order = append(order, a.Service) + } + if strings.Join(order, ",") != "third,second,first" { + t.Fatalf("order %v, want newest first", order) + } + if all[0].CreatedAt != "2026-10-31T20:23:37.310000000Z" { + t.Fatalf("stored created_at %q", all[0].CreatedAt) + } +} + +// Only the engine starts an episode: an update without created_at keeps the +// stored start, and a new alert without one is refused. +func TestStoreUpsertKeepsEpisodeStart(t *testing.T) { + s := newTestStore(t) + rule, err := s.CreateRule(makeRule("keep-rule")) + if err != nil { + t.Fatal(err) + } + a, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "api", State: "firing", CreatedAt: "2026-10-31T20:00:00Z"}) + if err != nil { + t.Fatal(err) + } + a.CreatedAt, a.State = "", "resolved" + updated, err := s.UpsertAlert(a) + if err != nil { + t.Fatal(err) + } + if updated.CreatedAt != "2026-10-31T20:00:00.000000000Z" || updated.State != "resolved" { + t.Fatalf("updated %+v", updated) + } + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "new", State: "firing"}); err == nil { + t.Fatal("a new alert without an episode start was stored") + } +} + +// A rule and service may have many resolved episodes but only one open alert. +func TestStoreAllowsOneOpenAlertPerService(t *testing.T) { + s := newTestStore(t) + rule, err := s.CreateRule(makeRule("open-rule")) + if err != nil { + t.Fatal(err) + } + for _, state := range []string{"resolved", "resolved", "firing"} { + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "api", State: state, CreatedAt: episodeStart}); err != nil { + t.Fatalf("store %s episode: %v", state, err) + } + } + if _, err := s.UpsertAlert(Alert{RuleID: rule.ID, Service: "api", State: "pending", CreatedAt: episodeStart}); err == nil { + t.Fatal("stored a second open alert for one rule and service") + } +} + +// Resolved times carry the engine's local offset; pruning compares instants, +// not text, so an alert resolved 30 hours ago in any zone is removed after one day. +func TestStorePruneResolvedComparesInstants(t *testing.T) { + s := newTestStore(t) + r, err := s.CreateRule(makeRule("prune")) + if err != nil { + t.Fatal(err) + } + // 30 hours ago written at +14:00 reads as a later calendar day than the + // UTC cutoff, so a text comparison keeps it. + old := time.Now().Add(-30 * time.Hour).In(time.FixedZone("LINT", 14*3600)).Format(time.RFC3339) + recent := time.Now().Add(-time.Hour).In(time.FixedZone("IST", 5*3600+1800)).Format(time.RFC3339) + for svc, resolved := range map[string]string{"old": old, "recent": recent} { + if _, err := s.UpsertAlert(Alert{RuleID: r.ID, Service: svc, State: "resolved", ResolvedAt: resolved, CreatedAt: episodeStart}); err != nil { + t.Fatal(err) + } + } + n, err := s.PruneResolved(1) + if err != nil || n != 1 { + t.Fatalf("pruned %d, %v; want 1", n, err) + } + if _, err := s.GetAlert(r.ID, "recent"); err != nil { + t.Fatalf("recent alert pruned: %v", err) + } +} diff --git a/internal/alert/types.go b/internal/alert/types.go index 02e3ddf71..146f81719 100644 --- a/internal/alert/types.go +++ b/internal/alert/types.go @@ -28,7 +28,7 @@ type Alert struct { RuleID string `json:"rule_id"` Service string `json:"service"` State string `json:"state"` - Value float64 `json:"value,omitempty"` // Primary metric snapshot (error_rate) at time of evaluation + Value float64 `json:"value"` // The first variable the rule reads, at the last evaluation FiredAt string `json:"fired_at,omitempty"` ResolvedAt string `json:"resolved_at,omitempty"` RepeatedAt string `json:"repeated_at,omitempty"` @@ -47,38 +47,38 @@ type Alert struct { type Env struct { // Mean of minute-rollup span error proportions over the evaluation window, // from 0 to 1; not a request-weighted whole-window error rate. - ErrorRate float64 `expr:"error_rate"` + ErrorRate float64 `expr:"error_rate" json:"error_rate"` // Mean of approximate minute-rollup median span latencies in milliseconds. - P50 float64 `expr:"p50"` + P50 float64 `expr:"p50" json:"p50"` // Mean of approximate minute-rollup 95th-percentile span latencies in // milliseconds; not the whole-window p95. Latency prefers served work, // falling back to outbound spans when no served work is available. - P95 float64 `expr:"p95"` + P95 float64 `expr:"p95" json:"p95"` // Total span count over the current rollup window, not requests per second. // Pair it with an error threshold to avoid firing on very little traffic. - Throughput float64 `expr:"throughput"` + Throughput float64 `expr:"throughput" json:"throughput"` // Log records recorded for the service over the window. - LogCount float64 `expr:"log_count"` + LogCount float64 `expr:"log_count" json:"log_count"` // How far the service's current behaviour sits from its own recent baseline, // in standard deviations. Use it to catch a service that has changed without // committing to an absolute threshold that fits every service. - ZScore float64 `expr:"z_score"` + ZScore float64 `expr:"z_score" json:"z_score"` // The detector's instance-wide composite 0-100 health score. Every service // receives the same score; it is not a per-service value. - HealthScore float64 `expr:"health_score"` + HealthScore float64 `expr:"health_score" json:"health_score"` // Relative change in error rate against the preceding rollup window, as a // percentage; zero when the baseline is missing or zero. - ErrorRateDelta float64 `expr:"error_rate_delta"` + ErrorRateDelta float64 `expr:"error_rate_delta" json:"error_rate_delta"` // Relative change in p95 latency against the preceding rollup window, as a // percentage; zero when the baseline is missing or zero. - P95Delta float64 `expr:"p95_delta"` + P95Delta float64 `expr:"p95_delta" json:"p95_delta"` // Relative change in span count against the preceding rollup window, as a // percentage. Negative when traffic drops; zero when the baseline is missing // or zero. - ThroughputDelta float64 `expr:"throughput_delta"` + ThroughputDelta float64 `expr:"throughput_delta" json:"throughput_delta"` // The service name being evaluated, as a string. Compare it when one rule // needs to behave differently for one service. - Service string `expr:"service"` + Service string `expr:"service" json:"service"` } // Summary aggregates alert state counts. diff --git a/internal/alert/webhook.go b/internal/alert/webhook.go index 1c2b3c37f..77be89cf7 100644 --- a/internal/alert/webhook.go +++ b/internal/alert/webhook.go @@ -3,13 +3,17 @@ package alert import ( "bytes" "encoding/json" + "errors" "fmt" "io" "log/slog" "net/http" + "sort" "strings" "text/template" "time" + + "golang.org/x/net/http/httpguts" ) // webhookClient is shared across all webhook deliveries. @@ -39,6 +43,37 @@ func RenderTemplate(tmplStr string, ctx ActionContext) (string, error) { return buf.String(), nil } +// CheckHeaders reports whether headers is empty or a JSON object of valid +// HTTP header names and string values, naming the first header that is not. +// A value is never echoed, since it may hold a credential. +func CheckHeaders(headers string) error { + if headers == "" { + return nil + } + var values map[string]any + if err := json.Unmarshal([]byte(headers), &values); err != nil || values == nil { + return errors.New("webhook headers must be a JSON object") + } + names := make([]string, 0, len(values)) + for name := range values { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if !httpguts.ValidHeaderFieldName(name) { + return fmt.Errorf("webhook header name %q is not a valid HTTP header name", name) + } + value, ok := values[name].(string) + if !ok { + return fmt.Errorf("webhook header %s must be a string", name) + } + if !httpguts.ValidHeaderFieldValue(value) { + return fmt.Errorf("webhook header %s has a line break or control character in its value", name) + } + } + return nil +} + // FireWebhook sends the webhook for the given rule. Returns "skipped" if no // URL is configured, "success" on 2xx, or "failed" after exhausting retries. // diff --git a/internal/alert/webhook_test.go b/internal/alert/webhook_test.go index 5438ff572..afb6782b8 100644 --- a/internal/alert/webhook_test.go +++ b/internal/alert/webhook_test.go @@ -203,3 +203,16 @@ func TestFireWebhook_TemplateRendered(t *testing.T) { t.Errorf("body missing service: %q", receivedBody) } } + +func TestCheckHeaders(t *testing.T) { + for _, ok := range []string{"", `{}`, `{"Authorization":"Bearer x"}`} { + if err := CheckHeaders(ok); err != nil { + t.Errorf("CheckHeaders(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{`{"X-Test": 5}`, `[]`, `"x"`, `{`, `{"":"x"}`, `{"Bad Name":"x"}`, `{"X-Test":"a\r\nInjected: y"}`, `{"X-Test":"a\u0000b"}`} { + if err := CheckHeaders(bad); err == nil { + t.Errorf("CheckHeaders(%q) = nil, want error", bad) + } + } +} diff --git a/internal/api/alerts.go b/internal/api/alerts.go index 518409cd7..cd142fd93 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -33,7 +33,7 @@ type alertHandler struct { // ListAlerts returns alerts with optional state/service/ruleID filters. func (h *alertHandler) ListAlerts(c *echo.Context) error { if h.store == nil { - return c.JSON(200, []alert.Alert{}) + return c.JSON(200, map[string][]alert.Alert{"alerts": {}}) } state := c.QueryParam("state") service := c.QueryParam("service") @@ -47,7 +47,7 @@ func (h *alertHandler) ListAlerts(c *echo.Context) error { if alerts == nil { alerts = []alert.Alert{} } - return c.JSON(200, alerts) + return c.JSON(200, map[string][]alert.Alert{"alerts": alerts}) } // AlertSummary returns firing/pending/resolved counts. @@ -66,7 +66,7 @@ func (h *alertHandler) AlertSummary(c *echo.Context) error { // ListRules returns all alert rules. func (h *alertHandler) ListRules(c *echo.Context) error { if h.store == nil { - return c.JSON(200, []alert.Rule{}) + return c.JSON(200, map[string][]alert.Rule{"rules": {}}) } rules, err := h.store.ListRules() if err != nil { @@ -76,7 +76,7 @@ func (h *alertHandler) ListRules(c *echo.Context) error { if rules == nil { rules = []alert.Rule{} } - return c.JSON(200, rules) + return c.JSON(200, map[string][]alert.Rule{"rules": rules}) } // CreateRule creates a new alert rule. @@ -93,9 +93,8 @@ func (h *alertHandler) CreateRule(c *echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "name and expression are required") } - // Validate expression compiles - if _, err := alert.CompileExpression(req.Expression); err != nil { - return echo.NewHTTPError(http.StatusBadRequest, "invalid expression: "+err.Error()) + if err := checkRule(req, true); err != nil { + return err } created, err := h.store.CreateRule(req) @@ -127,10 +126,12 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { } req.ID = id - if strings.TrimSpace(req.Expression) != "" { - if _, err := alert.CompileExpression(req.Expression); err != nil { - return echo.NewHTTPError(http.StatusBadRequest, "invalid expression: "+err.Error()) - } + // Headers are checked only when the replace changes them, so a rule saved + // before the check tightened can still be toggled or edited elsewhere. + stored, err := h.store.GetRule(id) + changed := err != nil || stored.WebhookHeaders != req.WebhookHeaders + if err := checkRule(req, changed); err != nil { + return err } updated, err := h.store.ReplaceRule(req) @@ -148,15 +149,7 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { slog.Error("alert rule updated but failed to compile in engine", "rule", updated.ID, "err", err) } } else { - h.engine.RemoveRule(updated.ID) - // Clear active alerts so they don't linger indefinitely - if activeAlerts, err := h.store.ListAlerts("", "", updated.ID); err == nil { - for _, a := range activeAlerts { - if a.State == "firing" || a.State == "pending" { - _ = h.store.DeleteAlert(a.ID) - } - } - } + h.engine.DisableRule(updated) } } @@ -201,49 +194,32 @@ func (h *alertHandler) TestRule(c *echo.Context) error { return echo.NewHTTPError(http.StatusInternalServerError, "failed to get rule") } - prog, compileErr := alert.CompileExpression(rule.Expression) - if compileErr != nil { - return echo.NewHTTPError(http.StatusBadRequest, "expression compile error: "+compileErr.Error()) - } - - envs := h.engine.BuildAllEnvs(c.Request().Context()) - - type testResult struct { - Service string `json:"service"` - Triggered bool `json:"triggered"` - Error string `json:"error,omitempty"` - Env alert.Env `json:"env"` + if _, err := alert.CompileExpression(rule.Expression); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid expression: "+err.Error()) } - - var results []testResult - services := resolveTestServices(rule, envs) - for _, svc := range services { - env := envs[svc] - triggered, evalErr := alert.SafeEval(prog, env) - tr := testResult{ - Service: svc, - Triggered: triggered, - Env: env, - } - if evalErr != nil { - tr.Error = evalErr.Error() - } - results = append(results, tr) + results, err := h.engine.Test(c.Request().Context(), rule) + if err != nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "The rule could not be tested: "+err.Error()).Wrap(err) } - return c.JSON(200, map[string]any{ "rule": rule, "results": results, }) } -func resolveTestServices(rule alert.Rule, envs map[string]alert.Env) []string { - if rule.Service == "" || rule.Service == "*" { - svcs := make([]string, 0, len(envs)) - for svc := range envs { - svcs = append(svcs, svc) +// checkRule rejects an expression that does not compile and, when headers is +// set, webhook headers that are not a JSON object of valid HTTP headers. A +// replace may omit the expression. +func checkRule(rule alert.Rule, headers bool) error { + if strings.TrimSpace(rule.Expression) != "" { + if _, err := alert.CompileExpression(rule.Expression); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid expression: "+err.Error()) + } + } + if headers { + if err := alert.CheckHeaders(rule.WebhookHeaders); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) } - return svcs } - return []string{rule.Service} + return nil } diff --git a/internal/api/alerts_test.go b/internal/api/alerts_test.go index e613dbff8..3dee1d6b1 100644 --- a/internal/api/alerts_test.go +++ b/internal/api/alerts_test.go @@ -1,13 +1,18 @@ package api import ( + "encoding/json" "net/http" "net/http/httptest" + "reflect" + "strconv" "strings" "testing" + "time" "github.com/labstack/echo/v5" + "github.com/labstack/fanout/internal/alert" "github.com/labstack/fanout/internal/auth" ) @@ -20,9 +25,9 @@ func TestAlertingReadRoutesRequireAuthenticationAndPreserveBodies(t *testing.T) } cookie := s.login(t, viewer) for _, tc := range []struct{ path, body string }{ - {"/api/alerting/alerts", "[]"}, + {"/api/alerting/alerts", `{"alerts":[]}`}, {"/api/alerting/summary", `{"firing":0,"pending":0,"resolved":0}`}, - {"/api/alerting/rules", "[]"}, + {"/api/alerting/rules", `{"rules":[]}`}, } { t.Run(tc.path, func(t *testing.T) { anonymous := httptest.NewRecorder() @@ -67,3 +72,134 @@ func TestRetiredAlertRoutesReturn404WithSPACatchAll(t *testing.T) { } } } + +func TestAlertRulesRejectNonStringWebhookHeaders(t *testing.T) { + s := newTestAuthServer(t) + RegisterAlertRoutes(s.e, alert.NewStore(s.db.DB), nil) + admin, err := s.users.CreateWithAudit("alert-admin@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + cookie := s.login(t, admin) + create := func(headers string) *httptest.ResponseRecorder { + body := `{"name":"Errors","enabled":true,"expression":"error_rate > 0.05","webhook_headers":` + strconv.Quote(headers) + `}` + req := sessionRequest(http.MethodPost, "/api/alerting/rules", strings.NewReader(body), cookie) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + return rec + } + if rec := create(`{"X-Test": 5}`); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "X-Test") { + t.Fatalf("numeric header = %d %s, want 400 naming X-Test", rec.Code, rec.Body) + } + if rec := create(`{"X-Test": "5"}`); rec.Code != http.StatusCreated { + t.Fatalf("string header = %d %s, want 201", rec.Code, rec.Body) + } +} + +// A rule saved before the header check tightened can still be toggled: a +// replace checks headers only when it changes them. +func TestAlertRuleReplaceChecksOnlyChangedHeaders(t *testing.T) { + s := newTestAuthServer(t) + store := alert.NewStore(s.db.DB) + RegisterAlertRoutes(s.e, store, nil) + admin, err := s.users.CreateWithAudit("alert-admin@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + cookie := s.login(t, admin) + stored, err := store.CreateRule(alert.Rule{Name: "Errors", Enabled: true, Expression: "error_rate > 0.05", WebhookHeaders: `{"Bad Name":"x"}`}) + if err != nil { + t.Fatal(err) + } + replace := func(headers string) *httptest.ResponseRecorder { + body := `{"name":"Errors","enabled":false,"expression":"error_rate > 0.05","webhook_headers":` + strconv.Quote(headers) + `}` + req := sessionRequest(http.MethodPut, "/api/alerting/rules/"+stored.ID, strings.NewReader(body), cookie) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + return rec + } + if rec := replace(stored.WebhookHeaders); rec.Code != http.StatusOK { + t.Fatalf("toggle with stored headers = %d %s, want 200", rec.Code, rec.Body) + } + if rec := replace(`{"Other Bad":"x"}`); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "Other Bad") { + t.Fatalf("changed invalid headers = %d %s, want 400", rec.Code, rec.Body) + } +} + +// Disabling a rule resolves its firing alerts instead of deleting them. +func TestAlertRuleDisableResolvesFiringAlerts(t *testing.T) { + s := newTestAuthServer(t) + store := alert.NewStore(s.db.DB) + RegisterAlertRoutes(s.e, store, alert.NewEngine(store, nil, nil, 30*time.Second, 7)) + admin, err := s.users.CreateWithAudit("alert-admin@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + cookie := s.login(t, admin) + rule, err := store.CreateRule(alert.Rule{Name: "Errors", Enabled: true, Expression: "error_rate > 0.05"}) + if err != nil { + t.Fatal(err) + } + if _, err := store.UpsertAlert(alert.Alert{RuleID: rule.ID, Service: "checkout", State: "firing", FiredAt: time.Now().Format(time.RFC3339), CreatedAt: time.Now().Format(time.RFC3339)}); err != nil { + t.Fatal(err) + } + req := sessionRequest(http.MethodPut, "/api/alerting/rules/"+rule.ID, strings.NewReader(`{"name":"Errors","enabled":false,"expression":"error_rate > 0.05"}`), cookie) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("disable = %d %s", rec.Code, rec.Body) + } + a, err := store.GetAlert(rule.ID, "checkout") + if err != nil { + t.Fatalf("firing alert was deleted: %v", err) + } + if a.State != "resolved" || a.ResolvedAt == "" { + t.Fatalf("alert = %+v, want resolved", a) + } +} + +func TestAlertCollectionsPreserveItemsAndFilters(t *testing.T) { + s := newTestAuthServer(t) + store := alert.NewStore(s.db.DB) + RegisterAlertRoutes(s.e, store, nil) + viewer, err := s.users.CreateWithAudit("collections@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + cookie := s.login(t, viewer) + read := func(path, key string, want any) { + t.Helper() + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, sessionRequest(http.MethodGet, path, nil, cookie)) + var got map[string]json.RawMessage + if rec.Code != http.StatusOK { + t.Fatalf("%s = %d %s", path, rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || !reflect.DeepEqual(got[key], json.RawMessage(encoded)) { + t.Fatalf("%s = %s, want %s", path, rec.Body, encoded) + } + } + read("/api/alerting/alerts", "alerts", []alert.Alert{}) + read("/api/alerting/rules", "rules", []alert.Rule{}) + rule, err := store.CreateRule(alert.Rule{Name: "Errors", Enabled: true, Expression: "error_rate > 0.05"}) + if err != nil { + t.Fatal(err) + } + item, err := store.UpsertAlert(alert.Alert{RuleID: rule.ID, Service: "checkout", State: "firing", CreatedAt: "2026-10-01T12:00:00Z"}) + if err != nil { + t.Fatal(err) + } + read("/api/alerting/rules", "rules", []alert.Rule{rule}) + read("/api/alerting/alerts?state=firing&service=checkout&rule_id="+rule.ID, "alerts", []alert.Alert{item}) + read("/api/alerting/alerts?service=missing", "alerts", []alert.Alert{}) +} diff --git a/internal/api/annotations.go b/internal/api/annotations.go index 606b25ac1..c0637d0fa 100644 --- a/internal/api/annotations.go +++ b/internal/api/annotations.go @@ -7,7 +7,6 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/annotations" - "github.com/labstack/fanout/internal/engine" ) type AnnotationReader interface { @@ -15,7 +14,7 @@ type AnnotationReader interface { } func RegisterAnnotationRoutes(e *echo.Echo, service AnnotationReader) { - e.POST("/api/annotations", func(c *echo.Context) error { + e.POST("/api/annotations/query", func(c *echo.Context) error { var req annotations.Request if err := decodeStrict(c, &req, 32<<10); err != nil { return err @@ -34,8 +33,6 @@ func annotationError(err error) error { return echo.NewHTTPError(http.StatusBadRequest, err.Error()).Wrap(err) case errors.Is(err, context.DeadlineExceeded): return echo.NewHTTPError(http.StatusGatewayTimeout, "Annotations took longer than 10 seconds").Wrap(err) - case errors.Is(err, engine.ErrParquetReadWait): - return echo.NewHTTPError(http.StatusServiceUnavailable, "annotations unavailable").Wrap(err) case errors.Is(err, context.Canceled): return nil default: diff --git a/internal/api/annotations_test.go b/internal/api/annotations_test.go index e06f4d8a9..83c7b5d1e 100644 --- a/internal/api/annotations_test.go +++ b/internal/api/annotations_test.go @@ -12,7 +12,6 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/annotations" - "github.com/labstack/fanout/internal/engine" ) type annotationFake struct{} @@ -23,7 +22,7 @@ func (annotationFake) Read(context.Context, annotations.Request) (annotations.Re func TestAnnotationErrorsWrapped(t *testing.T) { original := errors.New("storage failed") - for _, cause := range []error{annotations.ErrRequest, context.DeadlineExceeded, engine.ErrParquetReadWait, original} { + for _, cause := range []error{annotations.ErrRequest, context.DeadlineExceeded, original} { wrapped := fmt.Errorf("annotation query: %w", cause) got := annotationError(wrapped) var httpErr *echo.HTTPError @@ -50,7 +49,6 @@ func TestAnnotationErrorMappings(t *testing.T) { }{ {"request", annotations.ErrRequest, 400, annotations.ErrRequest.Error()}, {"deadline", context.DeadlineExceeded, 504, "Annotations took longer than 10 seconds"}, - {"parquet", engine.ErrParquetReadWait, 503, "annotations unavailable"}, {"canceled", context.Canceled, 200, ""}, {"unknown", errors.New("storage failed"), 500, "annotations unavailable"}, } { @@ -64,7 +62,7 @@ func TestAnnotationErrorMappings(t *testing.T) { t.Fatal(err) } RegisterAnnotationRoutes(s.e, annotationErrorFake{tc.err}) - req := sessionRequest(http.MethodPost, "/api/annotations", strings.NewReader(`{"from":"2026-10-01T12:00:00Z","to":"2026-10-01T13:00:00Z"}`), s.login(t, viewer)) + req := sessionRequest(http.MethodPost, "/api/annotations/query", strings.NewReader(`{"from":"2026-10-01T12:00:00Z","to":"2026-10-01T13:00:00Z"}`), s.login(t, viewer)) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() s.e.ServeHTTP(rec, req) @@ -88,18 +86,18 @@ func TestAnnotationRoute(t *testing.T) { t.Fatal(err) } RegisterAnnotationRoutes(s.e, annotationFake{}) - req := sessionRequest(http.MethodPost, "/api/annotations", strings.NewReader(`{"from":"2026-10-01T12:00:00Z","to":"2026-10-01T13:00:00Z","services":["checkout"]}`), s.login(t, viewer)) + req := sessionRequest(http.MethodPost, "/api/annotations/query", strings.NewReader(`{"from":"2026-10-01T12:00:00Z","to":"2026-10-01T13:00:00Z","services":["checkout"]}`), s.login(t, viewer)) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() s.e.ServeHTTP(rec, req) if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"deploys":[]`) { t.Fatalf("%d %s", rec.Code, rec.Body) } - policy, ok := classifyRoute(http.MethodPost, "/api/annotations") + policy, ok := classifyRoute(http.MethodPost, "/api/annotations/query") if !ok || policy.capability != ReadTelemetry { t.Fatalf("policy: %+v %v", policy, ok) } - if _, ok := classifyRoute(http.MethodGet, "/api/annotations"); ok { + if _, ok := classifyRoute(http.MethodGet, "/api/annotations/query"); ok { t.Fatal("GET alias allowed") } } @@ -120,7 +118,7 @@ func TestAnnotationErrRequest400(t *testing.T) { t.Fatal(err) } RegisterAnnotationRoutes(s.e, invalidAnnotationFake{}) - req := sessionRequest(http.MethodPost, "/api/annotations", strings.NewReader(`{"from":"2026-10-01T13:00:00Z","to":"2026-10-01T12:00:00Z"}`), s.login(t, viewer)) + req := sessionRequest(http.MethodPost, "/api/annotations/query", strings.NewReader(`{"from":"2026-10-01T13:00:00Z","to":"2026-10-01T12:00:00Z"}`), s.login(t, viewer)) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() s.e.ServeHTTP(rec, req) diff --git a/internal/api/auth_middleware.go b/internal/api/auth_middleware.go index d931d9a11..ed1ac6c8e 100644 --- a/internal/api/auth_middleware.go +++ b/internal/api/auth_middleware.go @@ -297,7 +297,7 @@ func classifyRoute(method, path string) (routePolicy, bool) { return routePolicy{kind: routePolicyCapability, capability: ReadOperations}, read || method == http.MethodPost case strings.HasPrefix(path, "/api/traces/") && len(strings.TrimPrefix(path, "/api/traces/")) > 0 && !strings.Contains(strings.TrimPrefix(path, "/api/traces/"), "/"): return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read - case path == "/api/intelligence": + case path == "/api/insights": return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read case path == "/api/alerting/alerts" || path == "/api/alerting/summary" || path == "/api/alerting/rules": if read { @@ -308,7 +308,7 @@ func classifyRoute(method, path string) (routePolicy, bool) { } case strings.HasPrefix(path, "/api/alerting/rules/"): return routePolicy{kind: routePolicyCapability, capability: ManageAlerts}, unsafe - case path == "/api/queries/run" || path == "/api/queries/exemplars" || path == "/api/queries/variables/resolve" || path == "/api/annotations": + case path == "/api/queries/run" || path == "/api/queries/exemplars" || path == "/api/queries/variables/resolve" || path == "/api/annotations/query": return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, method == http.MethodPost case path == "/api/fanql/parse" || path == "/api/fanql/format" || path == "/api/fanql/complete": return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, method == http.MethodPost diff --git a/internal/api/auth_test.go b/internal/api/auth_test.go index 2271cb936..9bf98fb29 100644 --- a/internal/api/auth_test.go +++ b/internal/api/auth_test.go @@ -118,6 +118,8 @@ func firstCookie(t *testing.T, rec *httptest.ResponseRecorder, name string) *htt func TestRetiredRoutesDoNotReachSPA(t *testing.T) { s := newTestAuthServer(t) registerTestUserRoutes(s) + RegisterIntelligenceRoutes(s.e, fakeIntelligenceSnapshots{}) + RegisterAnnotationRoutes(s.e, annotationFake{}) admin, err := s.users.CreateWithAudit("retired-routes@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) if err != nil { t.Fatal(err) @@ -130,6 +132,8 @@ func TestRetiredRoutesDoNotReachSPA(t *testing.T) { {http.MethodPost, "/api/auth/login-link"}, {http.MethodGet, "/api/dashboard"}, {http.MethodGet, "/api/health"}, + {http.MethodGet, "/api/intelligence"}, + {http.MethodPost, "/api/annotations"}, {http.MethodGet, "/api/rules"}, {http.MethodPost, "/api/agent"}, {http.MethodPost, "/api/users/" + admin.ID + "/logout-all"}, @@ -167,7 +171,7 @@ func TestRoutePolicyClassification(t *testing.T) { {http.MethodPost, "/api/auth/oauth/connections/client-1/revoke", routePolicyAuthenticated, ""}, {http.MethodGet, "/api/traces/abc", routePolicyCapability, ReadTelemetry}, {http.MethodHead, "/api/traces/abc", routePolicyCapability, ReadTelemetry}, - {http.MethodGet, "/api/intelligence", routePolicyCapability, ReadTelemetry}, + {http.MethodGet, "/api/insights", routePolicyCapability, ReadTelemetry}, {http.MethodGet, "/api/alerting/alerts", routePolicyCapability, ReadTelemetry}, {http.MethodHead, "/api/alerting/alerts", routePolicyCapability, ReadTelemetry}, {http.MethodGet, "/api/alerting/summary", routePolicyCapability, ReadTelemetry}, diff --git a/internal/api/dashboard.go b/internal/api/dashboard.go index 63ef4364b..7ed1b825e 100644 --- a/internal/api/dashboard.go +++ b/internal/api/dashboard.go @@ -125,7 +125,7 @@ func (h *DashboardHandler) delete(c *echo.Context) error { return err } if c.Request().Header.Get("Fanout-Confirm-Delete") != c.Param("id") { - return echo.NewHTTPError(http.StatusPreconditionRequired, "dashboard deletion requires confirmation") + return newAPIError(http.StatusPreconditionRequired, "confirmation_required", "Deleting a dashboard requires the Fanout-Confirm-Delete header set to its ID.") } if err := h.dashboards.Delete(c.Request().Context(), owner, c.Param("id")); err != nil { return dashboardError(c, err) diff --git a/internal/api/dashboard_test.go b/internal/api/dashboard_test.go index 20d5463df..269338b23 100644 --- a/internal/api/dashboard_test.go +++ b/internal/api/dashboard_test.go @@ -94,8 +94,8 @@ func TestDashboardLifecycleOverHTTP(t *testing.T) { if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "version 9 of dashboard "+created.ID+" does not exist") || !strings.Contains(rec.Body.String(), `"code":"dashboard_version_not_found"`) { t.Fatalf("missing version %d %s", rec.Code, rec.Body) } - if rec := call(http.MethodDelete, "/api/dashboards/"+created.ID, "", ownerCookie); rec.Code != http.StatusPreconditionRequired { - t.Fatalf("unconfirmed delete = %d", rec.Code) + if rec := call(http.MethodDelete, "/api/dashboards/"+created.ID, "", ownerCookie); rec.Code != http.StatusPreconditionRequired || !strings.Contains(rec.Body.String(), `"code":"confirmation_required"`) { + t.Fatalf("unconfirmed delete = %d %s", rec.Code, rec.Body) } if rec := call(http.MethodDelete, "/api/dashboards/"+created.ID, "", ownerCookie, "Fanout-Confirm-Delete", created.ID); rec.Code != http.StatusNoContent { t.Fatalf("delete = %d", rec.Code) diff --git a/internal/api/fanql.go b/internal/api/fanql.go index 8ccf160ac..4f5911b6d 100644 --- a/internal/api/fanql.go +++ b/internal/api/fanql.go @@ -2,6 +2,7 @@ package api import ( "context" + "errors" "net/http" "time" @@ -47,16 +48,19 @@ func fanqlNotices(notices []fanql.Notice) []fanql.Notice { func (h *FanQLHandler) parse(c *echo.Context) error { var req struct { fanql.Options - Text string `json:"text"` - Preview bool `json:"preview,omitempty"` + Text string `json:"text"` + Preview bool `json:"preview,omitempty"` + Window string `json:"window,omitempty"` + Namespace string `json:"namespace,omitempty"` } if err := decodeStrict(c, &req, 512<<10); err != nil { return err } + req.Schema, req.SchemaPartial = h.cachedSchema(c.Request().Context(), req.Window, req.Namespace) compiled, problems := fanql.Compile(req.Text, req.Options) if len(problems) != 0 { failure := &fanqlFailure{apiError: newAPIError(http.StatusBadRequest, "invalid_fanql", "The query has problems.")} - failure.Problems = query.PresentProblems(problems) + failure.Problems = query.Diagnoses(problems) if req.Preview { failure.Preview = &fanqlPreview{Text: req.Text, Note: "not executable"} } @@ -82,6 +86,37 @@ func (h *FanQLHandler) parse(c *echo.Context) error { }) } +// cachedSchema is a schema already cached, so parse can compare keys with +// those seen without waiting for a discovery. One for the query's window, or +// the widest, and its namespace covers the query; otherwise the recent +// schema is partial. +func (h *FanQLHandler) cachedSchema(ctx context.Context, window, namespace string) (*query.Schema, bool) { + if h.engine == nil { + return nil, false + } + cached := func(window, namespace string) (*query.Schema, error) { + return h.engine.Schema(ctx, query.SchemaRequest{Window: window, Namespace: namespace, Cached: true}) + } + if window != "" { + // An invalid or longer window is an error other than not cached. + schema, err := cached(window, namespace) + if err == nil && schema != nil { + return schema, false + } + if errors.Is(err, query.ErrSchemaNotCached) { + if schema, err := cached("24h", namespace); err == nil && schema != nil { + return schema, false + } + } + } + for _, window := range []string{"1h", "24h"} { + if schema, err := cached(window, ""); err == nil && schema != nil { + return schema, true + } + } + return nil, false +} + func (h *FanQLHandler) format(c *echo.Context) error { var req struct { fanql.Options @@ -131,12 +166,16 @@ func (h *FanQLHandler) complete(c *echo.Context) error { if req.Window != nil { window = *req.Window } - ctx, cancel := context.WithTimeout(c.Request().Context(), panelDeadline) - defer cancel() - out := h.completer.Complete(ctx, fanql.CompleteRequest{ + creq := fanql.CompleteRequest{ Text: req.Text, Cursor: req.Cursor, Options: req.Options, Window: window, From: req.From, To: req.To, Namespace: req.Namespace, - }) + } + if err := fanql.CheckCompleteRequest(creq); err != nil { + return newAPIError(http.StatusBadRequest, "invalid_input", err.Error()) + } + ctx, cancel := context.WithTimeout(c.Request().Context(), panelDeadline) + defer cancel() + out := h.completer.Complete(ctx, creq) return c.JSON(http.StatusOK, out) } diff --git a/internal/api/fanql_test.go b/internal/api/fanql_test.go index afdbb187d..fb97b82ca 100644 --- a/internal/api/fanql_test.go +++ b/internal/api/fanql_test.go @@ -44,6 +44,9 @@ func (*fakeFanQLParser) RenderSQL(context.Context, map[string]any) (string, erro type fakeFanQLPanels struct{ fakePanels } func (*fakeFanQLPanels) Schema(_ context.Context, req query.SchemaRequest) (*query.Schema, error) { + if req.Cached { + return nil, query.ErrSchemaNotCached + } return &query.Schema{Window: req.Window, Signals: map[string]query.SignalSchema{ "logs": {Columns: []query.ColumnSchema{{Field: query.Field{Name: "severity", Type: query.TypeString}, Values: []query.Option{{Value: "ERROR"}, {Value: "INFO"}, {Value: "WARN"}}}}}, }}, nil @@ -118,7 +121,7 @@ func TestFanQLParseReturnsQueryCanonicalAndVizzes(t *testing.T) { func TestFanQLParseProblemsHaveNoQuery(t *testing.T) { for _, preview := range []bool{false, true} { - rec := serveFanQL(t, nil, &fakePanels{}, http.MethodPost, "/api/fanql/parse", fmt.Sprintf(`{"text":"spans where a:1 b:2 or c:3 | p99(duration) by http_route every 1m","preview":%t}`, preview)) + rec := serveFanQL(t, nil, &fakePanels{}, http.MethodPost, "/api/fanql/parse", fmt.Sprintf(`{"text":"spans where a:1 b:2 or c:3 | p99(duration) by http.route every 1m","preview":%t}`, preview)) var body struct { Code string `json:"code"` Message string `json:"message"` @@ -345,11 +348,12 @@ func TestFanQLResponsesHaveNoLanguageVersion(t *testing.T) { } func TestFanQLCompletionAbsoluteAndLongWindowsDiscloseRecentSchema(t *testing.T) { - for _, window := range []string{ - `"from":"2026-10-20T10:00:00Z","to":"2026-10-20T11:00:00Z"`, - `"window":"7d"`, + for _, tt := range []struct{ window, want string }{ + // A past absolute range keeps its length; schema windows end now. + {`"from":"2026-10-20T10:00:00Z","to":"2026-10-20T11:00:00Z"`, "1h"}, + {`"window":"7d"`, "24h"}, } { - rec := serveFanQL(t, nil, &fakeFanQLPanels{}, http.MethodPost, "/api/fanql/complete", `{"text":"logs where sev","cursor":14,`+window+`}`) + rec := serveFanQL(t, nil, &fakeFanQLPanels{}, http.MethodPost, "/api/fanql/complete", `{"text":"logs where sev","cursor":14,`+tt.window+`}`) var body struct { Schema struct { Window string `json:"window"` @@ -358,12 +362,25 @@ func TestFanQLCompletionAbsoluteAndLongWindowsDiscloseRecentSchema(t *testing.T) } `json:"schema"` } fanqlResponse(t, rec, http.StatusOK, &body) - if body.Schema.Window != "24h" || !body.Schema.Recent || body.Schema.Notice != "keys seen in the last 24h (recent), not in the selected interval" { + if body.Schema.Window != tt.want || !body.Schema.Recent || body.Schema.Notice != "keys seen in the last "+tt.want+" (recent), not in the selected interval" { t.Fatalf("disclosure: %s", rec.Body) } } } +func TestFanQLCompletionRejectsBadCursorAndWindow(t *testing.T) { + for _, body := range []string{ + `{"text":"spans","cursor":999}`, + `{"text":"spans","cursor":-1}`, + `{"text":"spans","cursor":5,"window":"bogus"}`, + } { + rec := serveFanQL(t, nil, &fakeFanQLPanels{}, http.MethodPost, "/api/fanql/complete", body) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "invalid_input") { + t.Fatalf("%s: %d %s", body, rec.Code, rec.Body) + } + } +} + func TestFanQLFormatReportsPanelRequirements(t *testing.T) { for _, tt := range []struct { panel, query, name string @@ -388,3 +405,56 @@ func TestFanQLFormatReportsPanelRequirements(t *testing.T) { } } } + +// FanQL problems quote the user's own text, which is often a URL path; the +// engine-error scrubber must not mistake them for filesystem paths. +func TestFanQLParseProblemsKeepURLPaths(t *testing.T) { + for _, tc := range []struct{ text, message, hint string }{ + {"spans where http.target:/a:b", "unexpected : after /a", `quote the value: http.target:"/a:b"`}, + {"logs where http://x:8080/y", "unexpected : after http", `quote the value: "http://x:8080/y"`}, + {"spans where url.path:/a/b:c", "unexpected : after /a/b", `quote the value: url.path:"/a/b:c"`}, + } { + body, _ := json.Marshal(map[string]string{"text": tc.text}) + rec := serveFanQL(t, nil, &fakePanels{}, http.MethodPost, "/api/fanql/parse", string(body)) + var got struct { + Problems query.Problems `json:"problems"` + } + fanqlResponse(t, rec, http.StatusBadRequest, &got) + if len(got.Problems) == 0 || got.Problems[0].Message != tc.message || got.Problems[0].Hint != tc.hint { + t.Errorf("%s: %s", tc.text, rec.Body) + } + } +} + +type fakeCachedSchemaPanels struct{ fakePanels } + +func (*fakeCachedSchemaPanels) Schema(_ context.Context, req query.SchemaRequest) (*query.Schema, error) { + if req.Window == "7d" { + return nil, query.Problems{{Path: "window", Message: "window must be one of 5m, 15m, 1h, 3h, 6h, 12h or 24h"}} + } + if !req.Cached || req.Window != "1h" || req.Namespace != "" { + return nil, query.ErrSchemaNotCached + } + return &query.Schema{Window: req.Window, Signals: map[string]query.SignalSchema{ + "spans": {Attributes: []query.AttributeSchema{{Key: "http.route", Scope: "attributes"}}}, + }}, nil +} + +// Parse compares keys with a schema already cached, never waiting for one. +// It offers the seen key as a fix only when that schema covers the query's +// window and namespace; otherwise it only names the key. +func TestFanQLParseOffersSeenDottedKey(t *testing.T) { + for _, tt := range []struct { + scope string + fix bool + }{{`,"window":"1h"`, true}, {`,"window":"15m"`, false}, {``, false}, {`,"window":"7d"`, false}, {`,"window":"1h","namespace":"shop"`, false}} { + rec := serveFanQL(t, nil, &fakeCachedSchemaPanels{}, http.MethodPost, "/api/fanql/parse", `{"text":"spans where http.rout:/api/cart | count()"`+tt.scope+`}`) + var body struct { + Notices []fanql.Notice `json:"notices"` + } + fanqlResponse(t, rec, http.StatusOK, &body) + if len(body.Notices) != 1 || (len(body.Notices[0].FixIts) == 1) != tt.fix || !strings.Contains(body.Notices[0].Message, "did you mean http.route?") { + t.Errorf("%s: %s", tt.scope, rec.Body) + } + } +} diff --git a/internal/api/health.go b/internal/api/health.go index 391d89473..dfacc8feb 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -19,14 +19,26 @@ import ( // HealthHandler handles health check endpoints type HealthHandler struct { - duck *engine.Duck + duck *engine.Duck + // pool is the engine's read pool, which tells a stalled engine from a busy one. + pool readPool cfg config.Config started time.Time } +type readPool interface { + Stalled() bool + Overruns() int + Live(context.Context) error +} + // NewHealthHandler creates a new health handler func NewHealthHandler(duck *engine.Duck, cfg config.Config) *HealthHandler { - return &HealthHandler{duck: duck, cfg: cfg, started: time.Now()} + h := &HealthHandler{duck: duck, cfg: cfg, started: time.Now()} + if duck != nil { + h.pool = duck + } + return h } // CheckResult represents a single health check result @@ -45,8 +57,24 @@ type HealthResponse struct { RuntimeSizing config.RuntimeSizing `json:"runtime_sizing"` } -// Liveness returns 200 if process is running (Kubernetes liveness probe) +// livenessPingTimeout keeps the engine's confirming ping inside the +// container health check's 3 s timeout. +const livenessPingTimeout = time.Second + +// Liveness returns 200 while the process can serve (Kubernetes liveness +// probe). It fails only for a stall in Fanout's own work that a restart would +// free: a Parquet publication holding reads out past its bound, or an idle +// engine that cannot answer a ping. An overloaded engine, however saturated, +// is alive, and so is one running a costly query past its deadline: a +// restart would only let the query run again. func (h *HealthHandler) Liveness(c *echo.Context) error { + if h.pool != nil { + ctx, cancel := context.WithTimeout(c.Request().Context(), livenessPingTimeout) + defer cancel() + if err := h.pool.Live(ctx); err != nil { + return c.String(http.StatusServiceUnavailable, "engine stalled") + } + } return c.String(http.StatusOK, "ok") } @@ -58,14 +86,41 @@ func (h *HealthHandler) Readiness(c *echo.Context) error { RuntimeSizing: h.cfg.RuntimeSizing(), } - // Check DuckDB - resp.Checks["duckdb"] = h.checkDuckDB() - resp.Checks["telemetry"] = h.checkTelemetry() + // The engine probes share one deadline and run concurrently, so a busy + // engine answers within it rather than after each probe's own. + ctx, cancel := context.WithTimeout(c.Request().Context(), readinessTimeout) + defer cancel() + probes := map[string]func(context.Context) CheckResult{"duckdb": h.checkDuckDB, "telemetry": h.checkTelemetry, "rollups": h.checkRollups} + type named struct { + name string + result CheckResult + } + results := make(chan named, len(probes)) + for name, probe := range probes { + go func() { results <- named{name, probe(ctx)} }() + } + // A probe stuck in an engine that ignores cancellation never returns, so + // waiting stops shortly after the deadline and names it. + start := time.Now() + late := time.After(readinessTimeout + 100*time.Millisecond) +collect: + for range probes { + select { + case result := <-results: + resp.Checks[result.name] = result.result + case <-late: + break collect + } + } + for name := range probes { + if _, ok := resp.Checks[name]; !ok { + resp.Checks[name] = h.failedCheck(ctx, start, context.DeadlineExceeded) + } + } - // Check data directory resp.Checks["data"] = h.checkDataDir() - resp.Checks["rollups"] = h.checkRollups() resp.Checks["maintenance"] = h.checkMaintenance() + resp.Checks["queries"] = h.checkQueries() // Determine overall status hasUnhealthy := false @@ -90,7 +145,7 @@ func (h *HealthHandler) Readiness(c *echo.Context) error { return c.JSON(http.StatusOK, resp) } -func (h *HealthHandler) checkDuckDB() CheckResult { +func (h *HealthHandler) checkDuckDB(ctx context.Context) CheckResult { if h.duck == nil { return CheckResult{ Status: "unhealthy", @@ -99,16 +154,10 @@ func (h *HealthHandler) checkDuckDB() CheckResult { } start := time.Now() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() var one int if err := h.duck.DB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { - return CheckResult{ - Status: "unhealthy", - LatencyMs: time.Since(start).Milliseconds(), - Error: err.Error(), - } + return h.failedCheck(ctx, start, err) } return CheckResult{ @@ -117,6 +166,19 @@ func (h *HealthHandler) checkDuckDB() CheckResult { } } +// checkQueries reports reads still running long past their deadlines. DuckDB +// checks for interrupts only between chunks, so a costly expression can hold +// a connection well after its caller gave up; the rest of the pool serves. +func (h *HealthHandler) checkQueries() CheckResult { + if h.pool == nil { + return CheckResult{Status: "ok"} + } + if n := h.pool.Overruns(); n > 0 { + return CheckResult{Status: "degraded", Detail: fmt.Sprintf("%d queries running past their deadlines", n)} + } + return CheckResult{Status: "ok"} +} + // diskDegradedPct is the free-space fraction below which the data dir reports // degraded. A full disk fails Parquet flushes and stops ingest, so we surface // pressure before that point. @@ -178,9 +240,30 @@ func maintenanceStaleThreshold(maintEvery time.Duration) time.Duration { return stale } -var telemetryReadinessTimeout = 5 * time.Second +// readinessTimeout is the engine probes' shared deadline, under the 1 s a +// Kubernetes probe allows by default and the container check's 3 s. +var readinessTimeout = 800 * time.Millisecond + +// failedCheck classifies a probe error. A probe that ran out its deadline +// waited behind long reads holding the engine's connections: the instance +// still serves, so it is degraded rather than unready, and /readyz does not +// flap out of rotation under load. A stalled Parquet publication times out +// the same way but does not recover, so it fails readiness, as /healthz +// does. Any other error is a failing engine. The deadline is checked on ctx +// as well: a driver may report the cancelled statement in its own words. +func (h *HealthHandler) failedCheck(ctx context.Context, start time.Time, err error) CheckResult { + res := CheckResult{Status: "unhealthy", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()} + if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) { + if h.pool != nil && h.pool.Stalled() { + res.Detail = "engine stalled" + } else { + res.Status, res.Detail = "degraded", "engine busy" + } + } + return res +} -func (h *HealthHandler) checkTelemetry() CheckResult { +func (h *HealthHandler) checkTelemetry(ctx context.Context) CheckResult { if h.duck == nil { return CheckResult{ Status: "unhealthy", @@ -189,26 +272,13 @@ func (h *HealthHandler) checkTelemetry() CheckResult { } start := time.Now() - ctx, cancel := context.WithTimeout(context.Background(), telemetryReadinessTimeout) - defer cancel() var one int - // Use the same snapshot gate as public reads so a short compaction/retention - // publication cannot race the Parquet scan and report a false outage. + // Bound to the captured catalog like public reads, so a compaction or + // retention swap cannot race the Parquet scan and report a false outage. err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM telemetry.spans LIMIT 1") - if errors.Is(err, engine.ErrParquetReadWait) { - return CheckResult{ - Status: "degraded", - LatencyMs: time.Since(start).Milliseconds(), - Error: err.Error(), - } - } if err != nil && err != sql.ErrNoRows { - return CheckResult{ - Status: "unhealthy", - LatencyMs: time.Since(start).Milliseconds(), - Error: err.Error(), - } + return h.failedCheck(ctx, start, err) } res := CheckResult{ @@ -291,7 +361,7 @@ func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailu return res } -func (h *HealthHandler) checkRollups() CheckResult { +func (h *HealthHandler) checkRollups(ctx context.Context) CheckResult { if h.duck == nil { return CheckResult{ Status: "unhealthy", @@ -300,8 +370,6 @@ func (h *HealthHandler) checkRollups() CheckResult { } start := time.Now() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() var updatedAt sql.NullTime var cacheCount int @@ -313,11 +381,7 @@ SELECT COALESCE(date_diff('second', MAX(updated_at), now()), 0) FROM rollup_state`).Scan(&updatedAt, &cacheCount, &ageSeconds) if err != nil { - return CheckResult{ - Status: "unhealthy", - LatencyMs: time.Since(start).Milliseconds(), - Error: err.Error(), - } + return h.failedCheck(ctx, start, err) } res := CheckResult{ diff --git a/internal/api/health_test.go b/internal/api/health_test.go index e81a87de4..a7aea401a 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -124,7 +124,9 @@ func TestReadiness_HealthyTelemetryAndRollups(t *testing.T) { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - mock.ExpectQuery("SELECT 1"). + // The engine probes run concurrently. + mock.MatchExpectationsInOrder(false) + mock.ExpectQuery("SELECT 1$"). WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1"). WillReturnRows(sqlmock.NewRows([]string{"1"})) @@ -183,31 +185,129 @@ func TestReadiness_HealthyTelemetryAndRollups(t *testing.T) { } } -func TestTelemetryReadinessReportsPublicationContentionAsDegraded(t *testing.T) { - duck := &engine.Duck{} - entered := make(chan struct{}) - release := make(chan struct{}) - done := make(chan error, 1) - go func() { - done <- duck.PublishParquet(context.Background(), func(context.Context) error { - close(entered) - <-release - return nil - }) - }() - <-entered - original := telemetryReadinessTimeout - telemetryReadinessTimeout = 20 * time.Millisecond - t.Cleanup(func() { telemetryReadinessTimeout = original }) - - result := NewHealthHandler(duck, config.Config{}).checkTelemetry() - if result.Status != "degraded" { - t.Fatalf("telemetry status = %q, want degraded: %+v", result.Status, result) - } - close(release) - if err := <-done; err != nil { +// A long query holding the engine's connections made /readyz answer 503 +// once: a busy engine is still ready and must not flap out of rotation. +func TestReadinessReportsABusyEngineAsDegraded(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + original := readinessTimeout + readinessTimeout = 20 * time.Millisecond + t.Cleanup(func() { readinessTimeout = original }) + slow := 200 * time.Millisecond + mock.MatchExpectationsInOrder(false) + mock.ExpectQuery("SELECT 1$").WillDelayFor(slow).WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1").WillDelayFor(slow).WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery("SELECT\\s+MAX\\(updated_at\\)").WillDelayFor(slow).WillReturnRows(sqlmock.NewRows([]string{"max", "count", "age_seconds"}).AddRow(time.Now().UTC(), 2, int64(30))) + + rec := httptest.NewRecorder() + c := echo.New().NewContext(httptest.NewRequest(http.MethodGet, "/readyz", nil), rec) + started := time.Now() + if err := NewHealthHandler(&engine.Duck{DB: db}, config.Config{DataDir: os.TempDir(), RollupInterval: time.Minute}).Readiness(c); err != nil { + t.Fatal(err) + } + // One shared deadline, not one per probe: three sequential 5 s probes + // outlasted the 3 s container health check. + if elapsed := time.Since(started); elapsed >= 3*readinessTimeout { + t.Fatalf("readiness took %s, want one shared %s deadline", elapsed, readinessTimeout) + } + if original >= time.Second { + t.Fatalf("readiness deadline %s must stay under a 1 s probe timeout", original) + } + var resp HealthResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { t.Fatal(err) } + if rec.Code != http.StatusOK || resp.Status != "degraded" { + t.Fatalf("status = %d %q, want 200 degraded: %+v", rec.Code, resp.Status, resp.Checks) + } + for _, key := range []string{"duckdb", "telemetry", "rollups"} { + if check := resp.Checks[key]; check.Status != "degraded" || check.Detail != "engine busy" { + t.Fatalf("%s check = %+v, want degraded and busy", key, check) + } + } +} + +// A failing engine is not merely busy. +func TestReadinessReportsAFailingEngineAsUnhealthy(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT 1").WillReturnError(errors.New("database is closed")) + if check := NewHealthHandler(&engine.Duck{DB: db}, config.Config{}).checkDuckDB(t.Context()); check.Status != "unhealthy" { + t.Fatalf("duckdb check = %+v, want unhealthy", check) + } +} + +type fakePool struct { + stalled bool + overruns int + live error +} + +func (p *fakePool) Stalled() bool { return p.stalled } +func (p *fakePool) Overruns() int { return p.overruns } +func (p *fakePool) Live(context.Context) error { return p.live } + +// Probes time out alike on an overloaded engine and a stalled one. Only a +// stall in Fanout's own work fails readiness and liveness, so the supervisor +// restarts it; an overloaded engine, or one running a query past its +// deadline, stays alive and reads as degraded. +func TestOnlyAStalledEngineFailsReadinessAndLiveness(t *testing.T) { + original := readinessTimeout + readinessTimeout = 20 * time.Millisecond + t.Cleanup(func() { readinessTimeout = original }) + for _, test := range []struct { + name string + pool *fakePool + ready, live int + status, details string + queries string + }{ + {"stalled", &fakePool{stalled: true, live: engine.ErrStalled}, http.StatusServiceUnavailable, http.StatusServiceUnavailable, "unhealthy", "engine stalled", "ok"}, + {"overloaded", &fakePool{}, http.StatusOK, http.StatusOK, "degraded", "engine busy", "ok"}, + {"overrunning", &fakePool{overruns: 1}, http.StatusOK, http.StatusOK, "degraded", "engine busy", "degraded"}, + } { + t.Run(test.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.MatchExpectationsInOrder(false) + mock.ExpectQuery("SELECT 1$").WillDelayFor(time.Second).WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1").WillDelayFor(time.Second).WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery("SELECT\\s+MAX\\(updated_at\\)").WillDelayFor(time.Second).WillReturnRows(sqlmock.NewRows([]string{"max", "count", "age_seconds"}).AddRow(time.Now().UTC(), 2, int64(30))) + h := NewHealthHandler(&engine.Duck{DB: db}, config.Config{DataDir: os.TempDir(), RollupInterval: time.Minute}) + h.pool = test.pool + + rec := httptest.NewRecorder() + if err := h.Readiness(echo.New().NewContext(httptest.NewRequest(http.MethodGet, "/readyz", nil), rec)); err != nil { + t.Fatal(err) + } + var resp HealthResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if check := resp.Checks["duckdb"]; rec.Code != test.ready || check.Status != test.status || check.Detail != test.details { + t.Fatalf("readiness = %d %+v, want %d %s %s", rec.Code, check, test.ready, test.status, test.details) + } + if check := resp.Checks["queries"]; check.Status != test.queries { + t.Fatalf("queries check = %+v, want %s", check, test.queries) + } + live := httptest.NewRecorder() + if err := h.Liveness(echo.New().NewContext(httptest.NewRequest(http.MethodGet, "/healthz", nil), live)); err != nil { + t.Fatal(err) + } + if live.Code != test.live { + t.Fatalf("liveness = %d, want %d", live.Code, test.live) + } + }) + } } // A maintenance pass that never executes must degrade once the startup grace diff --git a/internal/api/intelligence.go b/internal/api/intelligence.go index 45573944d..f9b832cbd 100644 --- a/internal/api/intelligence.go +++ b/internal/api/intelligence.go @@ -17,7 +17,7 @@ type IntelligenceSnapshots interface { // RegisterIntelligenceRoutes exposes the detector's latest completed snapshot. func RegisterIntelligenceRoutes(e *echo.Echo, snapshots IntelligenceSnapshots) { h := &intelligenceHandler{snapshots: snapshots} - e.GET("/api/intelligence", h.latest, RequireCapability(ReadTelemetry)) + e.GET("/api/insights", h.latest, RequireCapability(ReadTelemetry)) } type intelligenceHandler struct { diff --git a/internal/api/intelligence_test.go b/internal/api/intelligence_test.go index 347788d2c..931c8aa99 100644 --- a/internal/api/intelligence_test.go +++ b/internal/api/intelligence_test.go @@ -4,10 +4,12 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "reflect" "testing" "time" "github.com/labstack/echo/v5" + "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/intelligence" ) @@ -25,16 +27,16 @@ func TestIntelligenceLatestReturnsSnapshot(t *testing.T) { Timeframe: "last_15m", Summary: "System healthy (98/100).", HealthScore: 98, - Insights: []intelligence.Insight{}, - Patterns: []intelligence.Pattern{}, - Anomalies: []intelligence.Anomaly{}, + Insights: []intelligence.Insight{{Type: "trend", Message: "Errors increasing"}}, + Patterns: []intelligence.Pattern{{Template: "request failed", Count: 4}}, + Anomalies: []intelligence.Anomaly{{ServiceName: "checkout", Current: 12}}, } e := echo.New() h := &intelligenceHandler{snapshots: fakeIntelligenceSnapshots{snapshot: want}} - e.GET("/api/intelligence", h.latest) + e.GET("/api/insights", h.latest) rec := httptest.NewRecorder() - e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/intelligence", nil)) + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/insights", nil)) if rec.Code != http.StatusOK { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } @@ -42,7 +44,7 @@ func TestIntelligenceLatestReturnsSnapshot(t *testing.T) { if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { t.Fatalf("decode response: %v", err) } - if got.GeneratedAt != want.GeneratedAt || got.Timeframe != want.Timeframe || got.HealthScore != want.HealthScore || got.Summary != want.Summary { + if !reflect.DeepEqual(got, *want) { t.Fatalf("snapshot = %#v, want %#v", got, *want) } } @@ -50,11 +52,38 @@ func TestIntelligenceLatestReturnsSnapshot(t *testing.T) { func TestIntelligenceLatestReportsNotReady(t *testing.T) { e := echo.New() h := &intelligenceHandler{snapshots: fakeIntelligenceSnapshots{}} - e.GET("/api/intelligence", h.latest) + e.GET("/api/insights", h.latest) rec := httptest.NewRecorder() - e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/intelligence", nil)) + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/insights", nil)) if rec.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable) } } + +func TestInsightsRoutePreservesSnapshotAndRequiresTelemetryRead(t *testing.T) { + s := newTestAuthServer(t) + want := &intelligence.Snapshot{GeneratedAt: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC), Timeframe: "last_15m", HealthScore: 75, Summary: "Errors increased", Insights: []intelligence.Insight{{Message: "Check checkout"}}, Patterns: []intelligence.Pattern{{Template: "failed", Count: 3}}, Anomalies: []intelligence.Anomaly{{ServiceName: "checkout"}}} + RegisterIntelligenceRoutes(s.e, fakeIntelligenceSnapshots{snapshot: want}) + anonymous := httptest.NewRecorder() + s.e.ServeHTTP(anonymous, httptest.NewRequest(http.MethodGet, "/api/insights", nil)) + if anonymous.Code != http.StatusUnauthorized { + t.Fatalf("anonymous = %d", anonymous.Code) + } + viewer, err := s.users.CreateWithAudit("insights@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, sessionRequest(http.MethodGet, "/api/insights", nil, s.login(t, viewer))) + var got intelligence.Snapshot + if rec.Code != http.StatusOK { + t.Fatalf("insights = %d %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, *want) { + t.Fatalf("snapshot = %#v, want %#v", got, *want) + } +} diff --git a/internal/api/oauth_test.go b/internal/api/oauth_test.go index a9a42df21..870c92989 100644 --- a/internal/api/oauth_test.go +++ b/internal/api/oauth_test.go @@ -25,7 +25,7 @@ import ( const testMCPResource = "https://fanout.example.com/mcp" -const testReadMCPCall = `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_observability_overview","arguments":{}}}` +const testReadMCPCall = `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_overview","arguments":{}}}` func newOAuthTestServer(t *testing.T) (*echo.Echo, *auth.UserStore, *auth.BrowserSessions) { return newOAuthTestServerWithConfig(t, config.Config{}) diff --git a/internal/api/queries.go b/internal/api/queries.go index 7f3dcec57..9d7ef8bfc 100644 --- a/internal/api/queries.go +++ b/internal/api/queries.go @@ -6,16 +6,17 @@ import ( "errors" "io" "net/http" + "strings" "time" "github.com/labstack/echo/v5" - "github.com/labstack/fanout/internal/engine" "github.com/labstack/fanout/internal/query" ) type QueryEngine interface { Exemplars(context.Context, query.ExemplarRequest) (query.ExemplarResponse, error) Run(context.Context, query.RunRequest) ([]query.Result, error) + Stream(context.Context, query.RunRequest, func(query.Result)) error ResolveVariables(context.Context, query.ResolveRequest) (map[string][]query.Option, error) Schema(context.Context, query.SchemaRequest) (*query.Schema, error) } @@ -35,16 +36,85 @@ func RegisterQueryRoutes(e *echo.Echo, engine QueryEngine) { e.GET("/api/telemetry/schema", h.schema, read) } +// streamType asks /api/queries/run for one JSON result per line, written as +// each panel finishes, instead of one object once the slowest panel is done. +const streamType = "application/x-ndjson" + +// streamWriteTimeout bounds writing one streamed result to the client. +const streamWriteTimeout = 10 * time.Second + +// resultContext is the request's context, marked to skip stored query results +// when the request carries Cache-Control: no-cache (RFC 9111, section +// 5.2.1.4), as the UI's manual refresh does. Such a request may still join a +// run that started after it arrived, and its fresh result is stored. +func resultContext(c *echo.Context) context.Context { + ctx := c.Request().Context() + for _, value := range c.Request().Header.Values(echo.HeaderCacheControl) { + for directive := range strings.SplitSeq(value, ",") { + name, _, _ := strings.Cut(strings.TrimSpace(directive), "=") + if strings.EqualFold(strings.TrimSpace(name), "no-cache") { + return query.WithoutCachedResults(ctx) + } + } + } + return ctx +} + func (h *QueryHandler) query(c *echo.Context) error { var req query.RunRequest if err := decodeStrict(c, &req, 512<<10); err != nil { return err } - results, err := h.engine.Run(c.Request().Context(), req) - if err != nil { + if !strings.Contains(c.Request().Header.Get(echo.HeaderAccept), streamType) { + results, err := h.engine.Run(resultContext(c), req) + if err != nil { + return panelError(c, err, "panels unavailable") + } + return c.JSON(http.StatusOK, map[string]any{"results": results}) + } + response := c.Response() + control := http.NewResponseController(response) + // A client that stops reading must not hold the stream open: each write + // has its own deadline, and a failed write cancels the panels still running. + ctx, cancel := context.WithCancel(resultContext(c)) + defer cancel() + defer func() { _ = control.SetWriteDeadline(time.Time{}) }() + encoder := json.NewEncoder(response) + started, failed := false, false + begin := func() { + started = true + response.Header().Set(echo.HeaderContentType, streamType) + // Not no-store: Chrome cancels a fully read no-store fetch body whose + // end arrives after its last chunk, so the stream shows as aborted. + // A POST response is not cached; no-transform keeps proxies from + // buffering or compressing the stream. + response.Header().Set(echo.HeaderCacheControl, "no-cache, no-transform") + response.WriteHeader(http.StatusOK) + } + err := h.engine.Stream(ctx, req, func(result query.Result) { + if failed { + return + } + _ = control.SetWriteDeadline(time.Now().Add(streamWriteTimeout)) + if !started { + begin() + } + if encoder.Encode(result) != nil || control.Flush() != nil { + failed = true + cancel() + } + }) + switch { + case err != nil && started: + // Only the client's departure ends a stream early; nobody is reading. + return nil + case err != nil: + // Problems and failures before the first panel still get a status. return panelError(c, err, "panels unavailable") + case !started: + begin() } - return c.JSON(http.StatusOK, map[string]any{"results": results}) + return nil } func (h *QueryHandler) resolve(c *echo.Context) error { @@ -61,26 +131,50 @@ func (h *QueryHandler) resolve(c *echo.Context) error { return c.JSON(http.StatusOK, map[string]any{"options": options}) } +// schemaWait bounds how long a request waits for a schema discovery. The +// discovery itself runs on, so a caller told to retry finds it cached. +const schemaWait = 15 * time.Second + func (h *QueryHandler) schema(c *echo.Context) error { - ctx, cancel := context.WithTimeout(c.Request().Context(), panelDeadline) + ctx, cancel := context.WithTimeout(c.Request().Context(), schemaWait) defer cancel() schema, err := h.engine.Schema(ctx, query.SchemaRequest{Window: c.QueryParam("window"), Namespace: c.QueryParam("namespace")}) - if err != nil { + if errors.Is(err, query.ErrSchemaBusy) { + c.Response().Header().Set("Retry-After", "5") + return newAPIError(http.StatusServiceUnavailable, "schema_busy", "The schema is still being read. Try again in a few seconds.") + } + var problems query.Problems + switch { + case errors.As(err, &problems): + message := "The schema request is invalid." + if len(problems) > 0 { + message = problems[0].Message + } + failure := newAPIError(http.StatusBadRequest, "bad_request", message) + failure.Problems = query.PresentProblems(problems) + return failure + case errors.Is(err, context.DeadlineExceeded): + return echo.NewHTTPError(http.StatusGatewayTimeout, "Reading the schema took too long. Try a shorter window.").Wrap(err) + case err != nil: return panelError(c, err, "schema unavailable") } return c.JSON(http.StatusOK, schema) } // panelError maps an engine failure to a response: spec problems are 400, a -// blown deadline is 504, a departed client gets no body, and anything else is -// a 500. +// full panel queue is 503, a blown deadline is 504, a departed client gets no +// body, and anything else is a 500. func panelError(c *echo.Context, err error, unavailable string) error { if handled, writeErr := writeProblems(c, err); handled { return writeErr } switch { + case errors.Is(err, query.ErrBusy): + // Every panel was turned away before any started: tell the client to + // come back rather than answer a page of busy panels. + return busyError(c) case errors.Is(err, context.DeadlineExceeded): - return echo.NewHTTPError(http.StatusGatewayTimeout, "Panels took longer than 20 seconds. Narrow the time range or reduce the number of panels.").Wrap(err) + return echo.NewHTTPError(http.StatusGatewayTimeout, query.BatchTimeoutError).Wrap(err) case errors.Is(err, context.Canceled): // The client is gone; there is nobody to answer. return nil @@ -124,24 +218,30 @@ func (h *QueryHandler) exemplars(c *echo.Context) error { if err := decodeStrict(c, &req, 512<<10); err != nil { return err } - out, err := h.engine.Exemplars(c.Request().Context(), req) + out, err := h.engine.Exemplars(resultContext(c), req) if err != nil { return exemplarHTTPError(c, err) } return c.JSON(http.StatusOK, out) } +// busyError answers a read the server turned away before it started. +func busyError(c *echo.Context) error { + c.Response().Header().Set("Retry-After", "2") + return newAPIError(http.StatusServiceUnavailable, "busy", query.BusyMessage) +} + func exemplarHTTPError(c *echo.Context, err error) error { if handled, writeErr := writeProblems(c, err); handled { return writeErr } switch { + case errors.Is(err, query.ErrBusy): + return busyError(c) case errors.Is(err, context.DeadlineExceeded): return echo.NewHTTPError(http.StatusGatewayTimeout, "Exemplars took longer than 10 seconds").Wrap(err) case errors.Is(err, context.Canceled): return nil - case errors.Is(err, engine.ErrParquetReadWait): - return echo.NewHTTPError(http.StatusServiceUnavailable, "exemplars unavailable").Wrap(err) default: return echo.NewHTTPError(http.StatusInternalServerError, "exemplars unavailable").Wrap(err) } diff --git a/internal/api/queries_test.go b/internal/api/queries_test.go index 78fa890b3..f68a8515e 100644 --- a/internal/api/queries_test.go +++ b/internal/api/queries_test.go @@ -3,11 +3,14 @@ package api import ( "context" "encoding/json" + "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/auth" "net/http" "net/http/httptest" + "os" "strings" "testing" + "time" "github.com/labstack/fanout/internal/query" ) @@ -16,10 +19,12 @@ type fakePanels struct { run query.RunRequest problem bool err error + bypass bool } -func (f *fakePanels) Run(_ context.Context, req query.RunRequest) ([]query.Result, error) { +func (f *fakePanels) Run(ctx context.Context, req query.RunRequest) ([]query.Result, error) { f.run = req + f.bypass = query.BypassesCache(ctx) if f.err != nil { return nil, f.err } @@ -29,6 +34,14 @@ func (f *fakePanels) Run(_ context.Context, req query.RunRequest) ([]query.Resul return []query.Result{{ID: "a", Status: query.StatusOK}}, nil } +func (f *fakePanels) Stream(ctx context.Context, req query.RunRequest, emit func(query.Result)) error { + results, err := f.Run(ctx, req) + for _, r := range results { + emit(r) + } + return err +} + func (f *fakePanels) ResolveVariables(context.Context, query.ResolveRequest) (map[string][]query.Option, error) { return map[string][]query.Option{"service": {{Value: "checkout", Count: 3}}}, nil } @@ -82,9 +95,42 @@ func TestPanelQueryRoute(t *testing.T) { } } +func TestPanelQueryRouteStreamsWhenAsked(t *testing.T) { + s := newTestAuthServer(t) + viewer, err := s.users.CreateWithAudit("viewer@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + engine := &fakePanels{} + RegisterQueryRoutes(s.e, engine) + post := func(body string) *httptest.ResponseRecorder { + req := sessionRequest(http.MethodPost, "/api/queries/run", strings.NewReader(body), s.login(t, viewer)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/x-ndjson") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + return rec + } + rec := post(`{"dashboard":{"name":"x","panels":[]}}`) + if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/x-ndjson" || rec.Body.String() != `{"id":"a","status":"ok","elapsed_ms":0,"from_ms":0,"to_ms":0}`+"\n" || !rec.Flushed { + t.Fatalf("stream %d %q %q", rec.Code, rec.Header().Get("Content-Type"), rec.Body) + } + // Chrome cancels a fully read no-store fetch body when its end arrives + // after the last chunk, reporting the stream as aborted; a POST response is + // not cached anyway, so the stream only keeps proxies from transforming it. + if got := rec.Header().Get("Cache-Control"); got != "no-cache, no-transform" { + t.Fatalf("stream cache control %q", got) + } + // Spec problems come before any result, so they keep their status. + engine.problem = true + if rec := post(`{"dashboard":{"name":"x","panels":[]}}`); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "panels[0].viz") { + t.Fatalf("stream problems %d %s", rec.Code, rec.Body) + } +} + func TestPanelQueryOperationalErrors(t *testing.T) { rec := servePanels(t, &fakePanels{err: context.DeadlineExceeded}, http.MethodPost, "/api/queries/run", `{"dashboard":{"name":"x","panels":[]}}`) - if rec.Code != http.StatusGatewayTimeout || !strings.Contains(rec.Body.String(), `"code":"timeout"`) { + if rec.Code != http.StatusGatewayTimeout || !strings.Contains(rec.Body.String(), `"code":"timeout"`) || !strings.Contains(rec.Body.String(), query.BatchTimeoutError) { t.Fatalf("deadline %d %s", rec.Code, rec.Body) } rec = servePanels(t, &fakePanels{err: context.Canceled}, http.MethodPost, "/api/queries/run", `{"dashboard":{"name":"x","panels":[]}}`) @@ -93,6 +139,27 @@ func TestPanelQueryOperationalErrors(t *testing.T) { } } +// Every panel turned away at the queue bound is one retryable 503, streamed +// or not, like the schema's busy answer. +func TestPanelQueryBusyIs503(t *testing.T) { + for _, accept := range []string{"application/json", "application/x-ndjson"} { + s := newTestAuthServer(t) + viewer, err := s.users.CreateWithAudit("viewer@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + RegisterQueryRoutes(s.e, &fakePanels{err: query.ErrBusy}) + req := sessionRequest(http.MethodPost, "/api/queries/run", strings.NewReader(`{"dashboard":{"name":"x","panels":[]}}`), s.login(t, viewer)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", accept) + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "2" || !strings.Contains(rec.Body.String(), `"code":"busy"`) { + t.Fatalf("%s: %d %q %s", accept, rec.Code, rec.Header().Get("Retry-After"), rec.Body) + } + } +} + func TestPanelQueryRejectsUnknownHistogramTemporality(t *testing.T) { engine := &fakePanels{} rec := servePanels(t, engine, http.MethodPost, "/api/queries/run", `{"dashboard":{"name":"Hist","panels":[{"id":"h","title":"Hist","viz":"histogram","query":{"from":"metrics","measures":["count()"],"histogram":{"field":"value","buckets":"explicit","temporality":"delta"}}}]}}`) @@ -136,23 +203,28 @@ func (f *partialBatchPanels) Run(ctx context.Context, _ query.RunRequest) ([]que if _, ok := ctx.Deadline(); ok { return nil, context.DeadlineExceeded } - return []query.Result{{ID: "fast", Status: query.StatusOK}, {ID: "slow", Status: query.StatusError, Error: "Not run: the dashboard ran out of time. Narrow the time range or split the dashboard."}}, nil + return []query.Result{{ID: "fast", Status: query.StatusOK}, {ID: "slow", Status: query.StatusError, Error: query.BatchDeadlineError, Reason: query.ReasonBusy}}, nil } func TestPanelBatchHTTPKeepsPartialResults(t *testing.T) { rec := servePanels(t, &partialBatchPanels{}, http.MethodPost, "/api/queries/run", `{"dashboard":{"name":"x","panels":[]}}`) - if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "Not run: the dashboard ran out of time. Narrow the time range or split the dashboard.") { + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"reason":"busy"`) { t.Fatalf("partial batch %d %s", rec.Code, rec.Body) } } -func (f *fakePanels) Exemplars(context.Context, query.ExemplarRequest) (query.ExemplarResponse, error) { - return query.ExemplarResponse{Traces: []query.Exemplar{}}, nil +func (f *fakePanels) Exemplars(ctx context.Context, _ query.ExemplarRequest) (query.ExemplarResponse, error) { + f.bypass = query.BypassesCache(ctx) + return query.ExemplarResponse{Traces: []query.Exemplar{}}, f.err } func TestExemplarRoutePolicy(t *testing.T) { rec := servePanels(t, &fakePanels{}, http.MethodPost, "/api/queries/exemplars", `{"dashboard":{"name":"Inline","panels":[]},"panel_id":"latency","from":"2026-10-01T12:00:00Z","to":"2026-10-01T12:05:00Z","dimensions":{}}`) if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"traces":[]`) { t.Fatalf("%d %s", rec.Code, rec.Body) } + rec = servePanels(t, &fakePanels{err: query.ErrBusy}, http.MethodPost, "/api/queries/exemplars", `{"dashboard":{"name":"Inline","panels":[]},"panel_id":"latency","from":"2026-10-01T12:00:00Z","to":"2026-10-01T12:05:00Z"}`) + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "2" { + t.Fatalf("busy exemplars: %d %s", rec.Code, rec.Body) + } policy, ok := classifyRoute(http.MethodPost, "/api/queries/exemplars") if !ok || policy.capability != ReadTelemetry { t.Fatalf("policy: %+v %v", policy, ok) @@ -161,3 +233,114 @@ func TestExemplarRoutePolicy(t *testing.T) { t.Fatal("GET accepted") } } + +type stalledWriter struct { + *httptest.ResponseRecorder + deadlines []time.Time +} + +func (w *stalledWriter) Write([]byte) (int, error) { return 0, os.ErrDeadlineExceeded } + +func (w *stalledWriter) SetWriteDeadline(t time.Time) error { + w.deadlines = append(w.deadlines, t) + return nil +} + +type cancelWatch struct { + fakePanels + cancelled bool +} + +func (f *cancelWatch) Stream(ctx context.Context, _ query.RunRequest, emit func(query.Result)) error { + emit(query.Result{ID: "a", Status: query.StatusOK}) + f.cancelled = ctx.Err() != nil + return ctx.Err() +} + +// A client that stops reading must not hold a stream open: each write has a +// deadline, and a failed write cancels the request so its panels stop. +func TestPanelStreamWriteDeadline(t *testing.T) { + s := newTestAuthServer(t) + viewer, err := s.users.CreateWithAudit("viewer@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + engine := &cancelWatch{} + RegisterQueryRoutes(s.e, engine) + req := sessionRequest(http.MethodPost, "/api/queries/run", strings.NewReader(`{"dashboard":{"name":"x","panels":[]}}`), s.login(t, viewer)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/x-ndjson") + w := &stalledWriter{ResponseRecorder: httptest.NewRecorder()} + s.e.ServeHTTP(w, req) + if len(w.deadlines) == 0 || w.deadlines[0].IsZero() { + t.Fatalf("write deadlines = %v", w.deadlines) + } + if !engine.cancelled { + t.Fatal("a failed write did not cancel the stream") + } +} + +// Cache-Control: no-cache, in any case and among other directives, skips +// stored query results on every cached route; nothing else does. +func TestNoCacheSkipsStoredResults(t *testing.T) { + for _, tc := range []struct { + header []string + want bool + }{ + {nil, false}, + {[]string{"no-cache"}, true}, + {[]string{"max-age=0, No-Cache"}, true}, + {[]string{"max-age=0", "no-cache"}, true}, + {[]string{"no-store"}, false}, + {[]string{"no-cache-please"}, false}, + } { + req := httptest.NewRequest(http.MethodPost, "/api/queries/run", nil) + for _, value := range tc.header { + req.Header.Add("Cache-Control", value) + } + c := echo.New().NewContext(req, httptest.NewRecorder()) + if got := query.BypassesCache(resultContext(c)); got != tc.want { + t.Errorf("Cache-Control %q: bypass=%v", tc.header, got) + } + } + + s := newTestAuthServer(t) + viewer, err := s.users.CreateWithAudit("viewer@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + panels := &fakePanels{} + traces := &traceProbe{} + RegisterQueryRoutes(s.e, panels) + RegisterTraceRoutes(s.e, traces, 7) + cookie := s.login(t, viewer) + for _, stream := range []bool{false, true} { + req := sessionRequest(http.MethodPost, "/api/queries/run", strings.NewReader(`{"dashboard":{"name":"x","panels":[]}}`), cookie) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Cache-Control", "no-cache") + if stream { + req.Header.Set("Accept", streamType) + } + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || !panels.bypass { + t.Fatalf("stream=%v: %d bypass=%v", stream, rec.Code, panels.bypass) + } + } + panels.bypass = false + req := sessionRequest(http.MethodPost, "/api/queries/exemplars", strings.NewReader(`{"dashboard":{"name":"Inline","panels":[]},"panel_id":"latency","from":"2026-10-01T12:00:00Z","to":"2026-10-01T12:05:00Z"}`), cookie) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Cache-Control", "no-cache") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || !panels.bypass { + t.Fatalf("exemplars: %d bypass=%v", rec.Code, panels.bypass) + } + req = sessionRequest(http.MethodGet, "/api/traces/abc?namespace=shop&from=2026-10-01T12:00:00Z&to=2026-10-01T13:00:00Z", nil, cookie) + req.Header.Set("Cache-Control", "no-cache") + rec = httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || !traces.bypass { + t.Fatalf("trace: %d bypass=%v", rec.Code, traces.bypass) + } +} diff --git a/internal/api/query_request_errors_test.go b/internal/api/query_request_errors_test.go index 9c23081ef..f63faf49e 100644 --- a/internal/api/query_request_errors_test.go +++ b/internal/api/query_request_errors_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "github.com/labstack/fanout/internal/engine" "github.com/labstack/fanout/internal/query" ) @@ -31,7 +30,6 @@ func TestExemplarHTTPErrorMapping(t *testing.T) { {"problem", query.Problems{{Path: "panel_id", Message: "missing"}}, 400, "problems"}, {"deadline", fmt.Errorf("wrapped: %w", context.DeadlineExceeded), 504, "Exemplars took longer than 10 seconds"}, {"cancel", context.Canceled, 200, ""}, - {"read_wait", fmt.Errorf("wrapped: %w", engine.ErrParquetReadWait), 503, "exemplars unavailable"}, {"internal", errors.New("failure"), 500, "exemplars unavailable"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/api/schema_busy_test.go b/internal/api/schema_busy_test.go new file mode 100644 index 000000000..c48f4b0da --- /dev/null +++ b/internal/api/schema_busy_test.go @@ -0,0 +1,65 @@ +package api + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/labstack/fanout/internal/query" +) + +type busySchema struct{ fakePanels } + +func (*busySchema) Schema(context.Context, query.SchemaRequest) (*query.Schema, error) { + return nil, query.ErrSchemaBusy +} + +func TestTelemetrySchemaBusyIs503WithRetryAfter(t *testing.T) { + rec := servePanels(t, &busySchema{}, http.MethodGet, "/api/telemetry/schema?window=6h", "") + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "5" || !strings.Contains(rec.Body.String(), `"code":"schema_busy"`) || strings.Contains(rec.Body.String(), "Panels") { + t.Fatalf("busy schema: %d %v %s", rec.Code, rec.Header(), rec.Body) + } +} + +type slowSchema struct{ fakePanels } + +func (*slowSchema) Schema(context.Context, query.SchemaRequest) (*query.Schema, error) { + return nil, context.DeadlineExceeded +} + +// A discovery that overran its own deadline is a schema timeout, not a +// message about panels. +func TestTelemetrySchemaTimeoutHasItsOwnMessage(t *testing.T) { + rec := servePanels(t, &slowSchema{}, http.MethodGet, "/api/telemetry/schema?window=24h", "") + if rec.Code != http.StatusGatewayTimeout || !strings.Contains(rec.Body.String(), "schema took too long") || strings.Contains(rec.Body.String(), "Panels") { + t.Fatalf("slow schema: %d %s", rec.Code, rec.Body) + } +} + +type windowSchema struct{ fakePanels } + +func (*windowSchema) Schema(context.Context, query.SchemaRequest) (*query.Schema, error) { + return nil, query.Problems{{Path: "window", Message: "window must be one of 5m, 15m, 1h, 3h, 6h, 12h or 24h"}} +} + +func TestTelemetrySchemaUnknownWindowNamesTheWindows(t *testing.T) { + rec := servePanels(t, &windowSchema{}, http.MethodGet, "/api/telemetry/schema?window=7d", "") + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), `"code":"bad_request"`) || !strings.Contains(rec.Body.String(), `"message":"window must be one of 5m, 15m, 1h, 3h, 6h, 12h or 24h"`) || strings.Contains(rec.Body.String(), "dashboard") { + t.Fatalf("unknown window: %d %s", rec.Code, rec.Body) + } +} + +type emptyProblemsSchema struct{ fakePanels } + +func (*emptyProblemsSchema) Schema(context.Context, query.SchemaRequest) (*query.Schema, error) { + return nil, query.Problems{} +} + +// An empty problem list is still a bad request, not a panic. +func TestTelemetrySchemaEmptyProblems(t *testing.T) { + rec := servePanels(t, &emptyProblemsSchema{}, http.MethodGet, "/api/telemetry/schema?window=1h", "") + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), `"code":"bad_request"`) { + t.Fatalf("empty problems: %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/api/shared_listener_test.go b/internal/api/shared_listener_test.go index 49095d9e1..a4b1223f7 100644 --- a/internal/api/shared_listener_test.go +++ b/internal/api/shared_listener_test.go @@ -69,3 +69,5 @@ func TestSharedListenerCredentialBoundaries(t *testing.T) { }) } } + +func (discardExport) Admit(context.Context, int) error { return nil } diff --git a/internal/api/traces.go b/internal/api/traces.go index 1f66df61e..afc6324da 100644 --- a/internal/api/traces.go +++ b/internal/api/traces.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/observability" + "github.com/labstack/fanout/internal/query" "net/http" "strconv" "strings" @@ -44,16 +45,18 @@ func (h *TraceHandler) get(c *echo.Context) error { return bad() } } - ctx, cancel := context.WithTimeout(c.Request().Context(), 20*time.Second) + ctx, cancel := context.WithTimeout(resultContext(c), 20*time.Second) defer cancel() out, err := h.queries.Trace(ctx, observability.Scope{Start: from.UTC(), End: to.UTC(), Namespace: namespace}, id, "", limit) if err != nil { - return traceError(err) + return traceError(c, err) } return c.JSON(http.StatusOK, out) } -func traceError(err error) error { +func traceError(c *echo.Context, err error) error { switch { + case errors.Is(err, query.ErrBusy): + return busyError(c) case errors.Is(err, observability.ErrInvalidScope), errors.Is(err, observability.ErrInvalidLimit): return echo.NewHTTPError(http.StatusBadRequest, "invalid trace scope").Wrap(err) case errors.Is(err, context.DeadlineExceeded): diff --git a/internal/api/traces_test.go b/internal/api/traces_test.go index d2b582f47..d1e2147a4 100644 --- a/internal/api/traces_test.go +++ b/internal/api/traces_test.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/observability" + "github.com/labstack/fanout/internal/query" "net/http" "net/http/httptest" "testing" @@ -15,6 +16,7 @@ type traceProbe struct { scope observability.Scope id string bounded bool + bypass bool err error } @@ -22,6 +24,7 @@ func (p *traceProbe) Trace(ctx context.Context, s observability.Scope, id, servi p.scope = s p.id = id _, p.bounded = ctx.Deadline() + p.bypass = query.BypassesCache(ctx) return observability.Result[observability.TraceDetail]{}, p.err } func TestTraceRoutePreservesCapturedNanoseconds(t *testing.T) { @@ -90,11 +93,11 @@ func TestTraceRouteClassifiesQueryFailures(t *testing.T) { for _, tc := range []struct { err error want int - }{{context.Canceled, 499}, {context.DeadlineExceeded, 504}, {observability.ErrInvalidScope, 400}, {observability.ErrInvalidLimit, 400}, {errors.New("engine failure"), 500}} { + }{{context.Canceled, 499}, {context.DeadlineExceeded, 504}, {observability.ErrInvalidScope, 400}, {observability.ErrInvalidLimit, 400}, {errors.New("engine failure"), 500}, {query.ErrBusy, 503}} { p.err = tc.err rec := httptest.NewRecorder() s.e.ServeHTTP(rec, sessionRequest("GET", "/api/traces/abc?from=2026-10-01T12:00:00Z&to=2026-10-01T13:00:00Z", nil, cookie)) - if rec.Code != tc.want { + if rec.Code != tc.want || tc.want == 503 && rec.Header().Get("Retry-After") != "2" { t.Fatalf("err=%v status=%d", tc.err, rec.Code) } } diff --git a/internal/api/users.go b/internal/api/users.go index 5b56921d0..8ff8c119e 100644 --- a/internal/api/users.go +++ b/internal/api/users.go @@ -61,7 +61,10 @@ func (h *UserHandler) ListUsers(c *echo.Context) error { slog.Error("list users failed", "err", err) return echo.NewHTTPError(http.StatusInternalServerError, "failed to list users") } - return c.JSON(200, users) + if users == nil { + users = []auth.User{} + } + return c.JSON(200, map[string][]auth.User{"users": users}) } // CreateUser adds a new user (admin only). diff --git a/internal/api/users_test.go b/internal/api/users_test.go index 164aada32..db8f62361 100644 --- a/internal/api/users_test.go +++ b/internal/api/users_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/labstack/echo/v5" + "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" ) @@ -329,14 +331,16 @@ func TestUserProjectionsAndProvisioning(t *testing.T) { if list.Code != http.StatusOK { t.Fatalf("list = %d", list.Code) } - var users []json.RawMessage - if err := json.Unmarshal(list.Body.Bytes(), &users); err != nil { + var envelope struct { + Users []json.RawMessage `json:"users"` + } + if err := json.Unmarshal(list.Body.Bytes(), &envelope); err != nil { t.Fatal(err) } - if len(users) != 2 { + if len(envelope.Users) != 2 { t.Fatalf("users = %s", list.Body.String()) } - for _, user := range users { + for _, user := range envelope.Users { assertUserProjection(user) } for _, body := range []string{ @@ -403,3 +407,15 @@ func TestStatusRouteSuspensionBlocksSessionsAndSignIn(t *testing.T) { t.Fatalf("suspension audits = %d, %v", audits, err) } } + +func TestListUsersEmptyEnvelope(t *testing.T) { + s := newTestAuthServer(t) + e := echo.New() + h := &UserHandler{users: s.users} + e.GET("/api/users", h.ListUsers) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/users", nil)) + if rec.Code != http.StatusOK || strings.TrimSpace(rec.Body.String()) != `{"users":[]}` { + t.Fatalf("empty users = %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/cmd/e2eseed/main_test.go b/internal/cmd/e2eseed/main_test.go index 82ab02f58..e4050429a 100644 --- a/internal/cmd/e2eseed/main_test.go +++ b/internal/cmd/e2eseed/main_test.go @@ -17,6 +17,7 @@ import ( "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/dashboard" "github.com/labstack/fanout/internal/engine" + "github.com/labstack/fanout/internal/engine/querytest" "github.com/labstack/fanout/internal/ingest" "github.com/labstack/fanout/internal/observability" "github.com/labstack/fanout/internal/query" @@ -80,7 +81,10 @@ func TestAllPanelsFixtureValidates(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - if _, problems, err := query.Check(t.Context(), engine, &spec); err != nil || len(problems) != 0 { + if err := querytest.RefreshSpanRollup(engine.DB); err != nil { + t.Fatal(err) + } + if _, problems, err := query.NewExecutor(engine, 30).Check(t.Context(), &spec); err != nil || len(problems) != 0 { t.Fatalf("fixture check: %v; %v", err, problems) } types := map[string]bool{} @@ -447,7 +451,10 @@ func TestPerformanceFixtureValidatesAndExecutes(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - if _, problems, err := query.Check(t.Context(), engine, &spec); err != nil || len(problems) != 0 { + if err := querytest.RefreshSpanRollup(engine.DB); err != nil { + t.Fatal(err) + } + if _, problems, err := query.NewExecutor(engine, 30).Check(t.Context(), &spec); err != nil || len(problems) != 0 { t.Fatalf("fixture check: %v; %v", err, problems) } from := base.Add(-24 * time.Hour) @@ -480,7 +487,28 @@ func TestPerformanceFixtureValidatesAndExecutes(t *testing.T) { } t.Run("system scenarios through OTLP", func(t *testing.T) { system := dashboard.DefaultSpec() - // Execute the system SQL panels and the existing external-peer reader. + // Topology reads completed edge summaries, including external peers. + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { defer close(done); engine.RunRollups(ctx) }() + defer func() { cancel(); <-done }() + horizon, _, _ := repo.PublicationBounds() + // Bound only a hang: CI runs packages side by side, and the wait ends as + // soon as the edge rollup reaches the horizon. + deadline := time.Now().Add(2 * time.Minute) + for { + var n int64 + if err := engine.DB.QueryRow(`SELECT coalesce(max(last_ingested_unix_nano),0) FROM rollup_state WHERE cache_key='edge_rollup_v2'`).Scan(&n); err != nil { + t.Fatal(err) + } + if n >= horizon { + break + } + if time.Now().After(deadline) { + t.Fatal("peer rollup did not complete") + } + time.Sleep(10 * time.Millisecond) + } system.Panels = system.Panels[8:] system.Panels = append(system.Panels[:2], system.Panels[3:]...) executor := query.NewExecutor(engine, 30) @@ -531,3 +559,5 @@ func TestPerformanceFixtureValidatesAndExecutes(t *testing.T) { }) }) } + +func (*fixtureSubmitter) Admit(context.Context, int) error { return nil } diff --git a/internal/cmd/semconvgen/main.go b/internal/cmd/semconvgen/main.go new file mode 100644 index 000000000..d46306950 --- /dev/null +++ b/internal/cmd/semconvgen/main.go @@ -0,0 +1,284 @@ +// semconvgen applies the dedicated-column key rule of proposal 0005 to a +// weaver-resolved semantic conventions registry and writes +// internal/telemetry/semconv/keys_gen.go. scripts/semconv-generate.sh pins +// and verifies its inputs; run it through `just semconv-generate`. +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "go/format" + "io" + "os" + "slices" + "strings" + + "github.com/labstack/fanout/internal/telemetry/semconv" +) + +// Rule A: the Stable duration metrics whose non-Opt-In attributes are span keys. +var metrics = []string{ + "http.server.request.duration", + "http.client.request.duration", + "db.client.operation.duration", + "rpc.server.call.duration", + "rpc.client.call.duration", +} + +// Rule C: the namespaces whose Stable registry attributes are resource keys. +var namespaces = []string{"service", "deployment", "k8s", "container", "host", "cloud"} + +// The service and namespace identity columns already store these. +var identity = []string{"service.name", "service.namespace"} + +type registry struct { + Groups []group `json:"groups"` +} + +type group struct { + ID string `json:"id"` + Type string `json:"type"` + Stability string `json:"stability"` + MetricName string `json:"metric_name"` + Attributes []attribute `json:"attributes"` +} + +type attribute struct { + Name string `json:"name"` + Type json.RawMessage `json:"type"` + Level json.RawMessage `json:"requirement_level"` + Stability string `json:"stability"` + Deprecated json.RawMessage `json:"deprecated"` +} + +func main() { + in := flag.String("in", "", "weaver-resolved registry JSON") + out := flag.String("out", "internal/telemetry/semconv/keys_gen.go", "generated Go file") + tag := flag.String("tag", "", "semantic conventions tag") + weaver := flag.String("weaver", "", "weaver version that resolved the registry") + sum := flag.String("sha256", "", "sha256 of the semantic conventions tag archive") + flag.Parse() + if err := run(*in, *out, *tag, *weaver, *sum, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "semconvgen:", err) + os.Exit(1) + } +} + +func run(in, out, tag, weaver, sum string, log io.Writer) error { + if in == "" || tag == "" || weaver == "" || sum == "" { + return errors.New("-in, -tag, -weaver and -sha256 are required") + } + data, err := os.ReadFile(in) + if err != nil { + return err + } + var reg registry + if err := json.Unmarshal(data, ®); err != nil { + return fmt.Errorf("parse %s: %w", in, err) + } + keys, err := selectKeys(reg) + if err != nil { + return err + } + src, err := render(keys, fmt.Sprintf("semantic conventions %s (archive sha256 %s), resolved by weaver %s", tag, sum, weaver)) + if err != nil { + return err + } + if changes := diff(keys); len(changes) > 0 { + for _, c := range changes { + fmt.Fprintln(log, c) + } + fmt.Fprintln(log, "The key list changed: this is a schema change and needs a new batch format version.") + } + return os.WriteFile(out, src, 0o644) +} + +// selectKeys applies rules A and C. Span keys come first, then resource keys, +// each sorted by name; a key's bit is its index. +func selectKeys(reg registry) ([]semconv.Key, error) { + var span, resource []semconv.Key + seen := map[string]bool{} + for _, name := range metrics { + i := slices.IndexFunc(reg.Groups, func(g group) bool { return g.Type == "metric" && g.MetricName == name }) + if i < 0 { + return nil, fmt.Errorf("metric %s is missing from the registry", name) + } + g := reg.Groups[i] + if g.Stability != "stable" { + continue + } + for _, a := range g.Attributes { + if seen[a.Name] || !stable(a) { + continue + } + level, err := requirement(a.Level) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", g.ID, a.Name, err) + } + if level == "opt_in" { + continue + } + typ, scalar, err := valueType(a.Type) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", g.ID, a.Name, err) + } + if !scalar { + continue + } + seen[a.Name] = true + span = append(span, semconv.Key{Name: a.Name, Scope: semconv.Span, Type: typ, Source: g.ID}) + } + } + seen = map[string]bool{} + for _, g := range reg.Groups { + if g.Type != "attribute_group" || !strings.HasPrefix(g.ID, "registry.") { + continue + } + for _, a := range g.Attributes { + ns, _, _ := strings.Cut(a.Name, ".") + if seen[a.Name] || !slices.Contains(namespaces, ns) || slices.Contains(identity, a.Name) || !stable(a) { + continue + } + typ, scalar, err := valueType(a.Type) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", g.ID, a.Name, err) + } + if !scalar { + continue + } + seen[a.Name] = true + resource = append(resource, semconv.Key{Name: a.Name, Scope: semconv.Resource, Type: typ, Source: g.ID}) + } + } + byName := func(a, b semconv.Key) int { return strings.Compare(a.Name, b.Name) } + slices.SortFunc(span, byName) + slices.SortFunc(resource, byName) + keys := append(span, resource...) + if len(keys) > 64 { + return nil, fmt.Errorf("%d keys exceed the 64-key limit", len(keys)) + } + for i := range keys { + prefix := "_a_" + if keys[i].Scope == semconv.Resource { + prefix = "_r_" + } + keys[i].Column = prefix + strings.ReplaceAll(keys[i].Name, ".", "_") + keys[i].Bit = uint8(i) + } + return keys, nil +} + +func stable(a attribute) bool { + deprecated := len(a.Deprecated) > 0 && string(a.Deprecated) != "null" + return a.Stability == "stable" && !deprecated +} + +// requirement returns the level of "required" or {"conditionally_required": note}. +func requirement(raw json.RawMessage) (string, error) { + var level string + if json.Unmarshal(raw, &level) == nil { + return level, nil + } + var noted map[string]json.RawMessage + if err := json.Unmarshal(raw, ¬ed); err != nil || len(noted) != 1 { + return "", fmt.Errorf("unknown requirement level %s", raw) + } + for level := range noted { + return level, nil + } + return "", nil +} + +// valueType maps a semconv type to a column type. Arrays, templates and any +// are not scalar; other scalar types have no column type yet and fail. +func valueType(raw json.RawMessage) (semconv.Type, bool, error) { + var name string + if json.Unmarshal(raw, &name) == nil { + switch name { + case "string": + return semconv.String, true, nil + case "int": + return semconv.Int, true, nil + case "boolean", "double": + return 0, false, fmt.Errorf("type %s has no column type", name) + } + return 0, false, nil + } + var enum struct { + Members []struct { + Value json.RawMessage `json:"value"` + } `json:"members"` + } + if err := json.Unmarshal(raw, &enum); err != nil || len(enum.Members) == 0 { + return 0, false, fmt.Errorf("unknown type %s", raw) + } + var strs, ints int + for _, m := range enum.Members { + var s string + var n int64 + switch { + case json.Unmarshal(m.Value, &s) == nil: + strs++ + case json.Unmarshal(m.Value, &n) == nil: + ints++ + } + } + switch len(enum.Members) { + case strs: + return semconv.String, true, nil + case ints: + return semconv.Int, true, nil + } + return 0, false, fmt.Errorf("enum members mix types in %s", raw) +} + +func render(keys []semconv.Key, source string) ([]byte, error) { + var b bytes.Buffer + fmt.Fprintf(&b, "// Code generated by internal/cmd/semconvgen; DO NOT EDIT.\n// Source: %s.\n\npackage semconv\n\n", source) + b.WriteString("// Keys lists the dedicated columns in bit order.\nvar Keys = [...]Key{\n") + for _, k := range keys { + fmt.Fprintf(&b, "\t{Name: %q, Scope: %s, Type: %s, Column: %q, Bit: %d, Source: %q},\n", + k.Name, scopeName(k.Scope), typeName(k.Type), k.Column, k.Bit, k.Source) + } + b.WriteString("}\n") + return format.Source(b.Bytes()) +} + +func scopeName(s semconv.Scope) string { + if s == semconv.Resource { + return "Resource" + } + return "Span" +} + +func typeName(t semconv.Type) string { + if t == semconv.Int { + return "Int" + } + return "String" +} + +// diff reports how keys differ from the committed list. +func diff(keys []semconv.Key) []string { + var changes []string + next := map[string]bool{} + for _, k := range keys { + next[scopeName(k.Scope)+" "+k.Name] = true + old, ok := semconv.Lookup(k.Scope, k.Name) + switch { + case !ok: + changes = append(changes, "added "+scopeName(k.Scope)+" "+k.Name) + case old != k: + changes = append(changes, "changed "+scopeName(k.Scope)+" "+k.Name) + } + } + for _, k := range semconv.Keys { + if !next[scopeName(k.Scope)+" "+k.Name] { + changes = append(changes, "removed "+scopeName(k.Scope)+" "+k.Name) + } + } + return changes +} diff --git a/internal/cmd/semconvgen/main_test.go b/internal/cmd/semconvgen/main_test.go new file mode 100644 index 000000000..1e621efe3 --- /dev/null +++ b/internal/cmd/semconvgen/main_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/labstack/fanout/internal/telemetry/semconv" +) + +// A miniature resolved registry with one case per rule clause. +const fixture = `{"groups": [ + {"id": "metric.http.server.request.duration", "type": "metric", "stability": "stable", + "metric_name": "http.server.request.duration", "attributes": [ + {"name": "http.route", "type": "string", "stability": "stable", "requirement_level": {"conditionally_required": "If available."}}, + {"name": "http.response.status_code", "type": "int", "stability": "stable", "requirement_level": "required"}, + {"name": "server.address", "type": "string", "stability": "stable", "requirement_level": "opt_in"}, + {"name": "url.template", "type": "string", "stability": "development", "requirement_level": {"recommended": "If available."}}, + {"name": "db.query.text", "type": "string", "stability": "stable", "requirement_level": "opt_in"}, + {"name": "http.request.method", "type": {"members": [{"id": "get", "value": "GET"}]}, "stability": "stable", "requirement_level": "required"}]}, + {"id": "metric.http.client.request.duration", "type": "metric", "stability": "stable", + "metric_name": "http.client.request.duration", "attributes": [ + {"name": "server.address", "type": "string", "stability": "stable", "requirement_level": "required"}, + {"name": "http.old", "type": "string", "stability": "stable", "deprecated": {"reason": "renamed"}, "requirement_level": "recommended"}]}, + {"id": "metric.db.client.operation.duration", "type": "metric", "stability": "stable", + "metric_name": "db.client.operation.duration", "attributes": [ + {"name": "db.code", "type": {"members": [{"id": "a", "value": 1}, {"id": "b", "value": 2}]}, "stability": "stable", "requirement_level": "recommended"}]}, + {"id": "metric.rpc.server.call.duration", "type": "metric", "stability": "release_candidate", + "metric_name": "rpc.server.call.duration", "attributes": [ + {"name": "rpc.method", "type": "string", "stability": "release_candidate", "requirement_level": "required"}]}, + {"id": "metric.rpc.client.call.duration", "type": "metric", "stability": "release_candidate", + "metric_name": "rpc.client.call.duration", "attributes": []}, + {"id": "registry.service", "type": "attribute_group", "attributes": [ + {"name": "service.name", "type": "string", "stability": "stable"}, + {"name": "service.namespace", "type": "string", "stability": "stable"}, + {"name": "service.version", "type": "string", "stability": "stable"}]}, + {"id": "registry.k8s", "type": "attribute_group", "attributes": [ + {"name": "k8s.pod.name", "type": "string", "stability": "stable"}, + {"name": "k8s.pod.label", "type": "template[string]", "stability": "stable"}, + {"name": "k8s.container.restart_count", "type": "int", "stability": "stable"}, + {"name": "k8s.pod.status.phase", "type": "string", "stability": "development"}]}, + {"id": "registry.container", "type": "attribute_group", "attributes": [ + {"name": "container.image.tags", "type": "string[]", "stability": "stable"}]}, + {"id": "registry.cloudevents", "type": "attribute_group", "attributes": [ + {"name": "cloudevents.event_id", "type": "string", "stability": "stable"}]}, + {"id": "registry.url", "type": "attribute_group", "attributes": [ + {"name": "url.path", "type": "string", "stability": "stable"}]}, + {"id": "span.k8s", "type": "span", "attributes": [ + {"name": "k8s.extra", "type": "string", "stability": "stable"}]} +]}` + +func load(t *testing.T, text string) registry { + t.Helper() + var reg registry + if err := json.Unmarshal([]byte(text), ®); err != nil { + t.Fatal(err) + } + return reg +} + +func TestSelectKeysAppliesRules(t *testing.T) { + keys, err := selectKeys(load(t, fixture)) + if err != nil { + t.Fatal(err) + } + want := []semconv.Key{ + {Name: "db.code", Scope: semconv.Span, Type: semconv.Int, Column: "_a_db_code", Bit: 0, Source: "metric.db.client.operation.duration"}, + {Name: "http.request.method", Scope: semconv.Span, Type: semconv.String, Column: "_a_http_request_method", Bit: 1, Source: "metric.http.server.request.duration"}, + {Name: "http.response.status_code", Scope: semconv.Span, Type: semconv.Int, Column: "_a_http_response_status_code", Bit: 2, Source: "metric.http.server.request.duration"}, + {Name: "http.route", Scope: semconv.Span, Type: semconv.String, Column: "_a_http_route", Bit: 3, Source: "metric.http.server.request.duration"}, + {Name: "server.address", Scope: semconv.Span, Type: semconv.String, Column: "_a_server_address", Bit: 4, Source: "metric.http.client.request.duration"}, + {Name: "k8s.container.restart_count", Scope: semconv.Resource, Type: semconv.Int, Column: "_r_k8s_container_restart_count", Bit: 5, Source: "registry.k8s"}, + {Name: "k8s.pod.name", Scope: semconv.Resource, Type: semconv.String, Column: "_r_k8s_pod_name", Bit: 6, Source: "registry.k8s"}, + {Name: "service.version", Scope: semconv.Resource, Type: semconv.String, Column: "_r_service_version", Bit: 7, Source: "registry.service"}, + } + if len(keys) != len(want) { + t.Fatalf("got %d keys %+v, want %d", len(keys), keys, len(want)) + } + for i := range want { + if keys[i] != want[i] { + t.Errorf("key %d = %+v, want %+v", i, keys[i], want[i]) + } + } +} + +func TestSelectKeysRejectsUnknownInputs(t *testing.T) { + cases := map[string]string{ + "missing metric": strings.Replace(fixture, `"metric_name": "rpc.client.call.duration"`, `"metric_name": "rpc.other"`, 1), + "boolean key": strings.Replace(fixture, `"name": "url.path", "type": "string"`, `"name": "cloud.flag", "type": "boolean"`, 1), + "mixed enum": strings.Replace(fixture, `"value": 2`, `"value": "two"`, 1), + } + for name, text := range cases { + t.Run(name, func(t *testing.T) { + if _, err := selectKeys(load(t, text)); err == nil { + t.Fatal("expected an error") + } + }) + } +} + +func TestRunWritesHeaderAndReportsChanges(t *testing.T) { + dir := t.TempDir() + in := filepath.Join(dir, "resolved.json") + out := filepath.Join(dir, "keys_gen.go") + if err := os.WriteFile(in, []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + var log strings.Builder + if err := run(in, out, "v9.9.9", "0.0.1", "abc123", &log); err != nil { + t.Fatal(err) + } + src, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "// Code generated by internal/cmd/semconvgen; DO NOT EDIT.\n", + "semantic conventions v9.9.9 (archive sha256 abc123), resolved by weaver 0.0.1", + `{Name: "k8s.pod.name", Scope: Resource, Type: String, Column: "_r_k8s_pod_name", Bit: 6, Source: "registry.k8s"},`, + } { + if !strings.Contains(string(src), want) { + t.Errorf("output lacks %q:\n%s", want, src) + } + } + // The fixture drops most committed keys and adds db.code. + for _, want := range []string{"added Span db.code", "removed Resource k8s.pod.uid", "needs a new batch format version"} { + if !strings.Contains(log.String(), want) { + t.Errorf("log lacks %q:\n%s", want, log.String()) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index e62a3a840..8bf176645 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -82,6 +82,15 @@ type Config struct { // CPU/heap/mutex/goroutine profiling under load. Off by default; the routes // require an admin browser session with operations:read. PprofEnabled bool `group:"server" env:"FANOUT_PPROF_ENABLED" default:"false"` + // QueryCacheTTL is the oldest a dashboard panel, exemplar or trace + // detail result served from memory may be, measured from the end of the + // time window it covers (for traces and exemplars, from when their run + // started). A result younger than half the TTL is served as is; one + // younger than the TTL is served while one background run refreshes it; + // an older one is rerun. A request with Cache-Control: no-cache, such as + // a dashboard's manual refresh, always reruns. Zero disables the cache; + // concurrent identical requests still share one execution. + QueryCacheTTL time.Duration `group:"server" env:"FANOUT_QUERY_CACHE_TTL" default:"10s"` // The DuckDB knobs below self-size where possible and otherwise default to // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing @@ -102,9 +111,11 @@ type Config struct { // report the resolved budget and detection status. Set an explicit cap when // memory limit detection is incomplete. DuckDBMemory string `group:"storage" env:"FANOUT_DUCKDB_MEMORY"` - // DuckDBThreads caps DuckDB's global query worker pool. Zero leaves - // DuckDB's own default in place (one worker per core). Set it to leave - // cores free for ingest on a query-heavy co-tenant host. + // DuckDBThreads caps DuckDB's global query worker pool. Zero selects one + // worker per CPU the process may run on (the affinity mask and cgroup + // quota, as GOMAXPROCS resolves them) rather than DuckDB's count of the + // machine's hardware threads. Set it to leave cores free for ingest on a + // query-heavy co-tenant host. DuckDBThreads int `group:"storage" env:"FANOUT_DUCKDB_THREADS" default:"0"` // DuckDBMaxConns caps the DuckDB read connection pool. Reads scan immutable // Parquet concurrently; analytical and completed-batch writes use separate @@ -268,6 +279,9 @@ func (c Config) Validate() error { if c.MaintenanceInterval < time.Second { return fmt.Errorf("FANOUT_MAINTENANCE_INTERVAL must be at least 1s, got %s", c.MaintenanceInterval) } + if c.QueryCacheTTL < 0 { + return fmt.Errorf("FANOUT_QUERY_CACHE_TTL must be >= 0, got %s", c.QueryCacheTTL) + } if c.DuckDBThreads < 0 { return fmt.Errorf("FANOUT_DUCKDB_THREADS must be >= 0, got %d", c.DuckDBThreads) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 42a14c842..5523f32bc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strings" "testing" "time" @@ -28,8 +29,8 @@ func TestLoadReturnsDefaults(t *testing.T) { if cfg.Addr != "127.0.0.1:7520" || cfg.DataDir != "./data" || cfg.IngestBatchSize != 50000 || cfg.RollupInterval != time.Minute || cfg.RetentionDays != 30 || cfg.DefaultNamespace != "default" || !cfg.MCPEnabled || !cfg.AlertEnabled || cfg.AIProvider != "" { t.Fatal("built-in defaults were not resolved") } - if cfg.DuckDBThreads != 0 || cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { - t.Fatal("machine sizing did not preserve the connection and thread defaults") + if cfg.DuckDBThreads != runtime.GOMAXPROCS(0) || cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { + t.Fatal("machine sizing did not resolve the connection and thread defaults") } } diff --git a/internal/config/sizing.go b/internal/config/sizing.go index b8fa03127..30def46ac 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -33,14 +33,45 @@ const ( // leaving its own default in place. minDetectableMemory = 512 << 20 - // maxAutoDuckDBConns bounds the pool. Past the point where connections - // exceed available parallelism they add contention rather than concurrency. + // maxAutoDuckDBConns bounds the pool, including the two connections outside + // read admission for annotations and readiness checks. maxAutoDuckDBConns = 16 // minDuckDBConns keeps one long scan from serializing all other reads. minDuckDBConns = 2 ) +// The query result cache's bounds. Entries are panel, exemplar and trace +// detail results; bytes are estimated from their contents. +const ( + QueryCacheMaxEntries = 1024 + QueryCacheMaxBytes = 64 << 20 +) + +// QueryAdmission derives the executor's read admission from the pool: +// slots reads hold connections at once. reserved slots serve interactive +// reads only; queue bounds dashboard batch tickets (interactive waits are 2*slots). +// A one-slot pool has no reservation so panels can still execute. A pool of four or more keeps two connections +// for annotations and readiness checks, which do not queue behind them. +func QueryAdmission(pool int) (slots, reserved, queue int) { + slots = max(1, pool) + if pool >= 4 { + slots = pool - 2 + } + reserved = min(slots-1, max(1, slots/6)) + // Measured refresh work: 15 panels at 150 ms is 2.25 slot-seconds. + // Five refreshes per guaranteed panel slot fit the 12-second horizon. + // Interactive reads may borrow the rest, so count only the two slots + // retained for pending panels under contention. + // Subtract running refreshes from the lifetime population. Individual + // panel slots let an odd remaining slot serve another refresh. + shared := slots - reserved + batchSlots := min(2, shared) + running := max(1, shared/batchSlots) + queue = max(1, 5*batchSlots-running) + return slots, reserved, queue +} + // sizingSource records which values the machine chose, so the startup log can // distinguish a resolved value from one the operator pinned. Without that // distinction an adaptive default is unfalsifiable in support: nobody can say @@ -48,6 +79,7 @@ const ( type sizingSource struct { MemoryAuto bool MaxConnsAuto bool + ThreadsAuto bool DetectedRAM uint64 DetectedHostRAM uint64 DetectedCgroupLimit uint64 @@ -64,6 +96,7 @@ type RuntimeSizing struct { DuckDBMaxConns int `json:"duckdb_max_conns"` DuckDBMaxConnsAuto bool `json:"duckdb_max_conns_auto"` DuckDBThreads int `json:"duckdb_threads"` + DuckDBThreadsAuto bool `json:"duckdb_threads_auto"` DetectedMemoryBytes uint64 `json:"detected_memory_bytes"` DetectedHostMemoryBytes uint64 `json:"detected_host_memory_bytes"` DetectedCgroupLimitBytes uint64 `json:"detected_cgroup_limit_bytes"` @@ -90,6 +123,14 @@ func (c *Config) resolveSizing() sizingSource { src.MaxConnsAuto = true c.DuckDBMaxConns = resolveDuckDBMaxConns(runtime.GOMAXPROCS(0)) } + if c.DuckDBThreads == 0 { + // DuckDB sizes its worker pool from the machine's hardware threads, not + // from the CPUs this process may run on. Pinned by taskset or a cpuset, + // that ran twice as many workers as CPUs; GOMAXPROCS already honors the + // affinity mask and the cgroup quota. + src.ThreadsAuto = true + c.DuckDBThreads = runtime.GOMAXPROCS(0) + } c.resolvedSizing = src return src } @@ -103,6 +144,7 @@ func (c Config) RuntimeSizing() RuntimeSizing { DuckDBMaxConns: c.DuckDBMaxConns, DuckDBMaxConnsAuto: c.resolvedSizing.MaxConnsAuto, DuckDBThreads: c.DuckDBThreads, + DuckDBThreadsAuto: c.resolvedSizing.ThreadsAuto, DetectedMemoryBytes: c.resolvedSizing.DetectedRAM, DetectedHostMemoryBytes: c.resolvedSizing.DetectedHostRAM, DetectedCgroupLimitBytes: c.resolvedSizing.DetectedCgroupLimit, @@ -121,17 +163,27 @@ func (c Config) logResolvedSizing(src sizingSource) { // logging an empty string that reads like "nothing to see here". memory = "duckdb-default(80% of detected RAM)" } + slots, reserved, queue := QueryAdmission(c.DuckDBMaxConns) slog.Info("runtime sizing resolved", "duckdb_memory", memory, "duckdb_memory_auto", src.MemoryAuto, "duckdb_max_conns", c.DuckDBMaxConns, "duckdb_max_conns_auto", src.MaxConnsAuto, + "duckdb_threads", c.DuckDBThreads, + "duckdb_threads_auto", src.ThreadsAuto, "detected_memory_bytes", src.DetectedRAM, "detected_host_memory_bytes", src.DetectedHostRAM, "detected_cgroup_limit_bytes", src.DetectedCgroupLimit, "memory_source", src.MemorySource, "memory_detection_incomplete", src.MemoryDetectionIncomplete, "gomaxprocs", runtime.GOMAXPROCS(0), + "query_slots", slots, + "query_interactive_slots", reserved, + "query_reserved_connections", c.DuckDBMaxConns-slots, + "query_queue_bound", queue, + "query_cache_ttl", c.QueryCacheTTL.String(), + "query_cache_max_entries", QueryCacheMaxEntries, + "query_cache_max_bytes", QueryCacheMaxBytes, ) if src.MemoryAuto && src.MemoryDetectionIncomplete { slog.Warn("memory limit detection was incomplete; automatic DuckDB sizing may exceed a container limit — configure FANOUT_DUCKDB_MEMORY explicitly", @@ -237,9 +289,12 @@ func parseByteSize(value string) (uint64, bool) { return uint64(amount * scale), true } -// resolveDuckDBMaxConns sizes the connection pool from available parallelism. +// resolveDuckDBMaxConns leaves one interactive slot per CPU, two panel progress +// slots, and two connections for reads outside admission. DuckDB's worker count +// stays at the CPU budget; a connection need not consume a worker while waiting +// on I/O. The short panel work must not reduce interactive concurrency. func resolveDuckDBMaxConns(cores int) int { - conns := cores + conns := cores + 4 if conns < minDuckDBConns { conns = minDuckDBConns } diff --git a/internal/config/sizing_test.go b/internal/config/sizing_test.go index 403e2837d..b201e773c 100644 --- a/internal/config/sizing_test.go +++ b/internal/config/sizing_test.go @@ -72,6 +72,14 @@ func TestResolveDuckDBMaxConnsScalesButIsBounded(t *testing.T) { } } +func TestResolveDuckDBMaxConnsKeepsInteractiveAndPanelSlots(t *testing.T) { + pool := resolveDuckDBMaxConns(8) + slots, _, _ := QueryAdmission(pool) + if pool != 12 || slots != 10 { + t.Fatalf("eight CPUs: pool=%d slots=%d, want twelve connections and ten admitted reads", pool, slots) + } +} + func TestParseCgroupMemoryLimitTreatsMaxAsUnlimited(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "memory.max") @@ -217,7 +225,7 @@ func TestResolveLeavesExplicitValuesAlone(t *testing.T) { t.Fatalf("DuckDBMaxConns = %d, want the operator's 9 untouched", cfg.DuckDBMaxConns) } sizing := cfg.RuntimeSizing() - if sizing.DuckDBMemoryAuto || sizing.DuckDBMaxConnsAuto { + if sizing.DuckDBMemoryAuto || sizing.DuckDBMaxConnsAuto || sizing.DuckDBThreadsAuto { t.Fatalf("explicit values reported as automatic: %+v", sizing) } if sizing.DuckDBMemory != "12GB" || sizing.DuckDBMaxConns != 9 || sizing.DuckDBThreads != 4 { @@ -233,8 +241,13 @@ func TestResolveFillsUnsetValues(t *testing.T) { if cfg.DuckDBMaxConns < 2 { t.Fatalf("DuckDBMaxConns = %d, want an auto-sized pool of at least 2", cfg.DuckDBMaxConns) } + // DuckDB's own default counts the machine's hardware threads; the pool is + // sized to the CPUs this process may run on instead. + if cfg.DuckDBThreads != runtime.GOMAXPROCS(0) { + t.Fatalf("DuckDBThreads = %d, want GOMAXPROCS %d", cfg.DuckDBThreads, runtime.GOMAXPROCS(0)) + } sizing := cfg.RuntimeSizing() - if !sizing.DuckDBMemoryAuto || !sizing.DuckDBMaxConnsAuto { + if !sizing.DuckDBMemoryAuto || !sizing.DuckDBMaxConnsAuto || !sizing.DuckDBThreadsAuto { t.Fatalf("resolved values not reported as automatic: %+v", sizing) } if sizing.DetectedMemoryBytes != detectAvailableMemory() { diff --git a/internal/dashboard/defaults.go b/internal/dashboard/defaults.go index 46318a62f..1b29f0c31 100644 --- a/internal/dashboard/defaults.go +++ b/internal/dashboard/defaults.go @@ -24,14 +24,14 @@ func DefaultSpec() query.Dashboard { {ID: "traffic", Title: "Requests by service", Viz: "timeseries", Options: &query.Options{Style: "stacked"}, Query: &query.Spec{From: "spans", Where: where, Measures: []string{"rate()"}, By: []string{"service"}, Bucket: "auto"}}, {ID: "latency", Better: "lower", Title: "p95 latency by service", Viz: "timeseries", Query: &query.Spec{From: "spans", Where: where, Measures: []string{"p95(duration_ms)"}, By: []string{"service"}, Bucket: "auto"}, Thresholds: []query.Threshold{{Value: 2000, Status: "bad"}}}, {ID: "error_services", Title: "Error rate by service", Viz: "bar", Width: 4, Click: &query.Click{SetVariable: "service"}, Query: &query.Spec{From: "spans", Where: []string{server}, Measures: []string{"error_rate()"}, By: []string{"service"}, Limit: 10}}, - {ID: "endpoints", Title: "Slowest endpoints", Viz: "table", Width: 8, Query: &query.Spec{From: "spans", Where: []string{server, service, "http_route <> ''"}, Measures: []string{"p95(duration_ms)", "error_rate()", "rate()"}, By: []string{"service", "http_route"}, Sort: "p95", Limit: 15}}, - {ID: "check_states", Title: "Is it reachable?", Description: receiverDescription("httpcheck", "Endpoint and host.name; silence is bad after two intervals. Labels warn if all checks run on service hosts."), Viz: "state_timeline", Width: 12, SQL: reachabilitySQL, Better: "lower", Thresholds: []query.Threshold{{Value: 1, Status: "bad"}}}, - {ID: "check_latency", Title: "Check latency", Description: receiverDescription("httpcheck", "Latest check duration by endpoint and reporting host.name."), Viz: "timeseries", Width: 12, SQL: checkLatencySQL, Unit: "ms"}, + {ID: "endpoints", Title: "Slowest endpoints", Description: "Minute summaries retain 128 routes per service; excess routes are combined as Other routes (overflow). Named routes may cover only some minutes. Latency is approximate.", Viz: "table", Width: 8, Query: &query.Spec{From: "spans", Where: []string{server, service, "attributes['http.route'] <> ''"}, Measures: []string{"p95(duration_ms)", "error_rate()", "rate()"}, By: []string{"service", "attributes['http.route']"}, Sort: "p95", Limit: 15}}, + {ID: "check_states", Title: "Is it reachable?", Description: receiverDescription("httpcheck", "Minute states by endpoint and host; silence is bad after two intervals. Labels flag service-host-only checks."), Viz: "state_timeline", Width: 12, SQL: reachabilitySQL, Better: "lower", Thresholds: []query.Threshold{{Value: 1, Status: "bad"}}}, + {ID: "check_latency", Title: "Check latency", Description: receiverDescription("httpcheck", "Latest minute check duration by endpoint and reporting host.name."), Viz: "timeseries", Width: 12, SQL: checkLatencySQL, Unit: "ms"}, {ID: "service_health", Title: "Is anything failing or slow?", Description: receiverDescription("otlp", "Service health and error trend with deploy markers."), Viz: "health", Width: 12, Query: &query.Spec{From: "spans", Where: []string{service}}}, {ID: "service_map", Title: "Services and external dependencies", Description: receiverDescription("otlp", "Failing or slow services and external dependencies observed in client spans."), Viz: "service_map", Width: 12, Height: "l", Query: &query.Spec{From: "spans", Where: []string{service}}}, - capacityPanel("hosts", "Is anything running out? Hosts", "hostmetrics", "CPU busy, memory used and the fullest filesystem; worst first. Not reporting after two intervals is bad.", hostCapacitySQL()), - capacityPanel("containers", "Containers", "docker_stats", "CPU: 100% is one core. Status uses limits; memory only if no CPU limit. Silence after two intervals is bad.", containerCapacitySQL()), - {ID: "versions", Title: "What changed?", Description: receiverDescription("otlp", "Service versions, first seen in the selected range, runtime and SDK."), Viz: "table", Width: 12, SQL: `WITH observations AS (` + versionObservationsSQL + `) SELECT namespace,service,version,min(seen) AS first_seen,runtime_name,runtime_version,sdk_language,sdk_version FROM observations WHERE version IS NOT NULL AND version<>'' GROUP BY namespace,service,version,runtime_name,runtime_version,sdk_language,sdk_version ORDER BY first_seen DESC,namespace,service,version`}, + capacityPanel("hosts", "Is anything running out? Hosts", "hostmetrics", "Minute summaries: CPU busy, memory used and the fullest filesystem; worst first. Not reporting after two intervals is bad.", hostCapacitySQL()), + capacityPanel("containers", "Containers", "docker_stats", "Minute summaries. CPU: 100% is one core. Status uses limits; silence after two intervals is bad.", containerCapacitySQL()), + {ID: "versions", Title: "What changed?", Description: receiverDescription("otlp", "Service versions, first seen in selected minute buckets, runtime and SDK."), Viz: "table", Width: 12, SQL: `WITH observations AS (` + versionObservationsSQL + `) SELECT namespace,service,version,min(seen) AS first_seen,runtime_name,runtime_version,sdk_language,sdk_version FROM observations WHERE version IS NOT NULL AND version<>'' GROUP BY namespace,service,version,runtime_name,runtime_version,sdk_language,sdk_version ORDER BY first_seen DESC,namespace,service,version`}, }, } } diff --git a/internal/dashboard/defaults_sql.go b/internal/dashboard/defaults_sql.go index 28c19c057..74b2fbd74 100644 --- a/internal/dashboard/defaults_sql.go +++ b/internal/dashboard/defaults_sql.go @@ -3,86 +3,122 @@ package dashboard import "strings" // SQL for the saved default dashboard, executed through the ordinary panel guard. +// Merge grid cells with observations instead of ASOF: low cardinality estimates +// can choose a quadratic nested-loop join. Cells sort before observations at the +// same instant, preserving the strict observation.time < cell.end contract. const reachabilitySQL = `WITH raw AS ( - SELECT series_id,ingested_unix_nano,namespace,time,value,type,attributes['http.url']::VARCHAR AS url, - attributes['http.method']::VARCHAR AS method,resource['host.name']::VARCHAR AS host, - attributes['http.status_class']::VARCHAR AS class,attributes['http.status_code']::VARCHAR AS code - FROM metrics WHERE time >= make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS - AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS AND name='httpcheck.status' QUALIFY row_number() OVER (PARTITION BY series_id,time ORDER BY ingested_unix_nano DESC,value DESC)=1), - keyed AS (SELECT *,to_json([namespace,url,method,host])::VARCHAR AS entity, - coalesce(namespace,'')||' / '||coalesce(method,'method unknown')||' '||coalesce(url,'endpoint unknown')||' from '||coalesce(host,'vantage unknown') AS endpoint_from FROM raw), - service_hosts AS (SELECT DISTINCT namespace,resource['host.name']::VARCHAR AS host FROM resource_inventory WHERE signal='spans' AND $__window(first_seen) AND service IS NOT NULL AND service<>''), + SELECT namespace,host,entity,endpoint_from,p.time,p.flag FROM receiver_entity_rollup, + unnest([first_observation,CASE WHEN previous_observation.time<>first_observation.time THEN previous_observation END,CASE WHEN last_observation.time<>first_observation.time THEN last_observation END]) t(p) + WHERE receiver='check' AND family='httpcheck.status' + AND bucket>=time_bucket(INTERVAL '1 minute',make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)) AND bucket<$__to::TIMESTAMP_NS + AND p.time>=make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS AND p.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), + keyed AS (SELECT * FROM raw), check_hosts AS (SELECT DISTINCT namespace,host FROM keyed WHERE $__window(time)), + service_hosts AS (SELECT DISTINCT s.namespace,s.host FROM (SELECT namespace,resource['host.name']::VARCHAR AS host FROM resource_rollup WHERE $__window(bucket) AND signal='spans' AND service IS NOT NULL AND service<>'') s + SEMI JOIN (SELECT namespace,host FROM check_hosts WHERE host<>'') c ON s.namespace=c.namespace AND s.host=c.host), vantage AS (SELECT count(*)>0 AND bool_and(coalesce(host<>'',false) AND EXISTS (SELECT 1 FROM service_hosts s WHERE s.namespace=check_hosts.namespace AND s.host=check_hosts.host)) AS all_local FROM check_hosts), inventory AS (SELECT DISTINCT entity,endpoint_from||CASE WHEN all_local THEN ' (all checks run on service hosts; outside reachability unverified)' ELSE '' END AS endpoint_from FROM keyed CROSS JOIN vantage WHERE $__window(time)), - attempts AS (SELECT entity,time, - CASE WHEN bool_or(url IS NULL OR url='' OR method IS NULL OR method='' OR host IS NULL OR host='' OR type NOT IN ('gauge','updown') OR NOT isfinite(value) OR (class IS NULL AND code IS NULL)) THEN 2 - WHEN bool_or(value>0 AND (class IN ('2xx','3xx') OR try_cast(code AS INTEGER) BETWEEN 200 AND 399)) THEN 0 ELSE 1 END AS failure - FROM keyed GROUP BY entity,time), + attempts AS (SELECT entity,time,CASE max(flag) WHEN 2 THEN 2 WHEN 1 THEN 0 ELSE 1 END AS failure FROM keyed GROUP BY entity,time), gaps AS (SELECT *,epoch_ns(time::TIMESTAMP_NS)::HUGEINT-epoch_ns(lag(time) OVER (PARTITION BY entity ORDER BY time)::TIMESTAMP_NS)::HUGEINT AS gap_ns FROM attempts), cadence AS (SELECT *,last_value(CASE WHEN gap_ns>0 THEN gap_ns END IGNORE NULLS) OVER (PARTITION BY entity ORDER BY time) AS interval_ns FROM gaps), buckets AS (SELECT b::TIMESTAMPTZ_NS AS time,least((b+INTERVAL '1 minute')::TIMESTAMPTZ_NS,$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS) AS at FROM generate_series(time_bucket(INTERVAL '1 minute',$__from::TIMESTAMP_NS),$__to::TIMESTAMP_NS,INTERVAL '1 minute') t(b) WHERE b::TIMESTAMPTZ_NS < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), grid AS (SELECT * FROM inventory CROSS JOIN buckets), + events AS (SELECT entity,time AS event_at,1 AS row_kind,NULL::TIMESTAMPTZ_NS AS time,NULL::VARCHAR AS endpoint_from, + struct_pack(time:=time,failure:=failure,interval_ns:=interval_ns) AS observation FROM cadence + UNION ALL SELECT entity,g.at,0,time,endpoint_from,NULL FROM grid g), + carried AS (SELECT *,last_value(observation IGNORE NULLS) OVER (PARTITION BY entity ORDER BY event_at,row_kind ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS latest FROM events), states AS (SELECT g.time,g.endpoint_from,g.entity, - CASE WHEN a.interval_ns IS NULL OR a.failure=2 THEN 2 WHEN epoch_ns(g.at::TIMESTAMP_NS)::HUGEINT-epoch_ns(a.time::TIMESTAMP_NS)::HUGEINT>=2*a.interval_ns THEN 2 ELSE a.failure END::DOUBLE AS failure, - CASE WHEN a.interval_ns IS NULL THEN 'reporting interval unknown' WHEN epoch_ns(g.at::TIMESTAMP_NS)::HUGEINT-epoch_ns(a.time::TIMESTAMP_NS)::HUGEINT>=2*a.interval_ns THEN 'no recent result' WHEN a.failure=2 THEN 'check metadata unsupported or missing' WHEN a.failure=1 THEN 'down' ELSE 'reachable' END AS state - FROM grid g ASOF LEFT JOIN cadence a ON g.entity=a.entity AND g.at>a.time), + CASE WHEN g.latest.interval_ns IS NULL OR g.latest.failure=2 THEN 2 WHEN epoch_ns(g.event_at::TIMESTAMP_NS)::HUGEINT-epoch_ns(g.latest.time::TIMESTAMP_NS)::HUGEINT>=2*g.latest.interval_ns THEN 2 ELSE g.latest.failure END::DOUBLE AS failure, + CASE WHEN g.latest.interval_ns IS NULL THEN 'reporting interval unknown' WHEN epoch_ns(g.event_at::TIMESTAMP_NS)::HUGEINT-epoch_ns(g.latest.time::TIMESTAMP_NS)::HUGEINT>=2*g.latest.interval_ns THEN 'no recent result' WHEN g.latest.failure=2 THEN 'check metadata unsupported or missing' WHEN g.latest.failure=1 THEN 'down' ELSE 'reachable' END AS state + FROM carried g WHERE g.row_kind=0), ranked AS (SELECT *,arg_max(failure,time) OVER (PARTITION BY entity) AS current_failure FROM states) SELECT time,endpoint_from,failure,state FROM ranked ORDER BY current_failure DESC,time DESC,endpoint_from` -const checkLatencySQL = `WITH points AS ( - SELECT *,to_json([namespace,attributes['http.url']::VARCHAR,attributes['http.method']::VARCHAR,resource['host.name']::VARCHAR])::VARCHAR AS entity, - namespace||' / '||coalesce(attributes['http.method']::VARCHAR,'method unknown')||' '||coalesce(attributes['http.url']::VARCHAR,'endpoint unknown')||' from '||coalesce(resource['host.name']::VARCHAR,'vantage unknown') AS endpoint_from, - value*CASE unit WHEN 's' THEN 1000 WHEN 'ms' THEN 1 WHEN 'us' THEN .001 WHEN 'µs' THEN .001 WHEN 'ns' THEN .000001 END AS duration_ms - FROM metrics WHERE time >= make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND name='httpcheck.duration' AND type IN ('gauge','updown') AND isfinite(value) - QUALIFY row_number() OVER (PARTITION BY series_id,time ORDER BY ingested_unix_nano DESC,value DESC,start_time DESC NULLS LAST)=1), +const checkLatencySQL = `WITH snapshots AS ( + SELECT series_id,namespace||' / '||coalesce(method,'method unknown')||' '||coalesce(url,'endpoint unknown')||' from '||coalesce(host,'vantage unknown') AS endpoint_from,unit, + CASE WHEN last_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN last_finite_point + WHEN previous_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN previous_finite_point + WHEN first_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN first_finite_point END point + FROM receiver_rollup WHERE name='httpcheck.duration' AND type IN ('gauge','updown') + AND bucket>=time_bucket(INTERVAL '1 minute',make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-300000000000)) AND bucket<$__to::TIMESTAMP_NS), + points AS (SELECT series_id,endpoint_from,point.time AS time,point.value*CASE unit WHEN 's' THEN 1000 WHEN 'ms' THEN 1 WHEN 'us' THEN .001 WHEN 'µs' THEN .001 WHEN 'ns' THEN .000001 END AS duration_ms + FROM snapshots WHERE point.time>=make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS), inventory AS (SELECT DISTINCT series_id,endpoint_from FROM points WHERE $__window(time)), buckets AS (SELECT b::TIMESTAMPTZ_NS AS time,least((b+INTERVAL '1 minute')::TIMESTAMPTZ_NS,$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS) AS at FROM generate_series(time_bucket(INTERVAL '1 minute',$__from::TIMESTAMP_NS),$__to::TIMESTAMP_NS,INTERVAL '1 minute') t(b) WHERE b::TIMESTAMPTZ_NS < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), - grid AS (SELECT * FROM inventory CROSS JOIN buckets) - SELECT g.time,g.endpoint_from,sum(CASE WHEN p.time>=make_timestamp_ns(epoch_ns(g.at::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS THEN p.duration_ms END)::DOUBLE AS latency - FROM grid g ASOF LEFT JOIN points p ON g.series_id=p.series_id AND g.at>p.time GROUP BY g.time,g.endpoint_from ORDER BY g.time DESC,g.endpoint_from` + grid AS (SELECT * FROM inventory CROSS JOIN buckets), + events AS (SELECT series_id,time AS event_at,1 AS row_kind,NULL::TIMESTAMPTZ_NS AS time,NULL::VARCHAR AS endpoint_from, + struct_pack(time:=time,duration_ms:=duration_ms) AS observation FROM points + UNION ALL SELECT series_id,g.at,0,time,endpoint_from,NULL FROM grid g), + carried AS (SELECT *,last_value(observation IGNORE NULLS) OVER (PARTITION BY series_id ORDER BY event_at,row_kind ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS latest FROM events) + SELECT time,endpoint_from,sum(CASE WHEN latest.time>=make_timestamp_ns(epoch_ns(event_at::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS THEN latest.duration_ms END)::DOUBLE AS latency + FROM carried WHERE row_kind=0 GROUP BY time,endpoint_from ORDER BY time DESC,endpoint_from` -// One finite, retry-deduplicated latest-point stage shared by both tables. +// Bounded, retry-deduplicated witnesses supply recent capacity and cadence. func capacityCTE(containers bool) string { names := "'system.cpu.time','system.memory.usage','system.filesystem.usage'" identity := "host IS NOT NULL AND host<>''" + receiver := "host" entity := "host" priority := "CASE name WHEN 'system.cpu.time' THEN 1 WHEN 'system.memory.usage' THEN 2 ELSE 3 END" if containers { + receiver = "container" identity += " AND container_id IS NOT NULL AND container_id<>''" names = "'container.cpu.utilization','container.cpu.limit','container.memory.usage.total','container.memory.usage.limit'" - entity = "to_json([host,container_id])::VARCHAR" + entity = "sha256(to_json([host,container_id]))" priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 WHEN 'container.cpu.limit' THEN 4 ELSE 3 END" } - return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit,scope_name, - resource['host.name']::VARCHAR AS host,resource['container.id']::VARCHAR AS container_id, - resource['container.name']::VARCHAR AS container_name,attributes['state']::VARCHAR AS state, - attributes['mountpoint']::VARCHAR AS mountpoint,attributes['device']::VARCHAR AS device - FROM metrics WHERE time >= make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS AND name IN (` + names + `)), - keyed AS (SELECT *,` + entity + ` AS entity,` + priority + ` AS priority FROM raw WHERE ` + identity + `), - inventory AS (SELECT namespace,entity,any_value(host) AS host,any_value(container_id) AS container_id,arg_max(container_name,time) AS container FROM keyed WHERE $__window(time) GROUP BY namespace,entity), - witness_family AS (SELECT namespace,entity,min(priority) AS priority FROM keyed GROUP BY namespace,entity), - witness AS (SELECT DISTINCT k.namespace,k.entity,k.time FROM keyed k JOIN witness_family f USING(namespace,entity,priority)), - gaps AS (SELECT *,epoch_ns(time::TIMESTAMP_NS)::HUGEINT-epoch_ns(lag(time) OVER (PARTITION BY namespace,entity ORDER BY time)::TIMESTAMP_NS)::HUGEINT AS gap_ns FROM witness), - cadence AS (SELECT namespace,entity,max(time) AS last_seen,arg_max(gap_ns,time) FILTER (WHERE gap_ns>0) AS interval_ns FROM gaps GROUP BY namespace,entity), - unique_points AS (SELECT * FROM keyed WHERE isfinite(value) QUALIFY row_number() OVER (PARTITION BY series_id,time ORDER BY ingested_unix_nano DESC,value DESC,start_time DESC NULLS LAST)=1), - latest AS (SELECT * FROM unique_points WHERE time >= make_timestamp_ns(epoch_ns($__to::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS AND type IN ('gauge','updown') QUALIFY row_number() OVER (PARTITION BY series_id ORDER BY time DESC)=1)` + return `WITH minutes AS MATERIALIZED (SELECT namespace,bucket,family AS name,host,container_id,container_name,entity, + first_observation,previous_observation,last_observation,` + strings.ReplaceAll(priority, "name", "family") + ` AS priority + FROM receiver_entity_rollup WHERE receiver='` + receiver + `' AND family IN (` + names + `) AND ` + identity + ` + AND bucket>=time_bucket(INTERVAL '1 minute',make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)) AND bucket<$__to::TIMESTAMP_NS + AND last_observation.time>=make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS AND first_observation.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), + inventory AS (SELECT namespace,entity,any_value(host) host,any_value(container_id) container_id,arg_max(container_name,last_observation.time) container,max(last_observation.time) observed + FROM minutes WHERE $__window(bucket) GROUP BY namespace,entity), + witness_family AS (SELECT namespace,entity,min(priority) AS priority FROM minutes GROUP BY namespace,entity), + witness_minutes AS (SELECT k.* FROM minutes k JOIN witness_family f USING(namespace,entity,priority) + QUALIFY dense_rank() OVER (PARTITION BY namespace,entity ORDER BY bucket DESC)<=2), + witness AS (SELECT DISTINCT namespace,entity,t.time FROM witness_minutes, + unnest([first_observation.time,previous_observation.time,last_observation.time]) t(time) + WHERE t.time>=make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS AND t.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), + cadence AS (SELECT namespace,entity,max(time) AS last_seen,epoch_ns(max(time)::TIMESTAMP_NS)::HUGEINT-epoch_ns(arg_max(time,time,2)[2]::TIMESTAMP_NS)::HUGEINT AS interval_ns FROM witness GROUP BY namespace,entity), + gauge_minutes AS (SELECT namespace,` + entity + ` AS entity,series_id,name,unit,scope_name,state,mountpoint,device, + CASE WHEN last_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN last_finite_point + WHEN previous_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN previous_finite_point + WHEN first_finite_point.time<$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS THEN first_finite_point END point + FROM receiver_rollup WHERE bucket>=time_bucket(INTERVAL '1 minute',make_timestamp_ns(epoch_ns($__to::TIMESTAMP_NS)-300000000000)) AND bucket<$__to::TIMESTAMP_NS + AND name IN (` + names + `) AND ` + identity + ` AND type IN ('gauge','updown')), + latest AS (SELECT namespace,entity,series_id,name,point.value AS value,unit,scope_name,point.time AS time,state,mountpoint,device + FROM gauge_minutes WHERE point.time>=make_timestamp_ns(epoch_ns($__to::TIMESTAMP_NS)-300000000000)::TIMESTAMPTZ_NS + QUALIFY row_number() OVER (PARTITION BY series_id ORDER BY point.time DESC)=1)` } const reportingSQL = `CASE WHEN i.host IS NULL OR i.host='' /* identity */ THEN 'stable resource identity missing' + WHEN c.interval_ns IS NULL THEN 'reporting interval unknown' WHEN epoch_ns($__to::TIMESTAMP_NS)::HUGEINT-epoch_ns(c.last_seen::TIMESTAMP_NS)::HUGEINT >= 2*c.interval_ns THEN 'not reporting' ELSE 'reporting' END` func hostCapacitySQL() string { return capacityCTE(false) + `, - cpu_previous AS (SELECT *,lag(value) OVER w AS previous_value,lag(time) OVER w AS previous_time,lag(start_time) OVER w AS previous_start FROM unique_points WHERE name='system.cpu.time' AND type='counter' WINDOW w AS (PARTITION BY series_id ORDER BY time)), - cpu_delta AS (SELECT *,CASE WHEN temporality='delta' OR valueprevious_start) THEN value ELSE value-previous_value END AS increase, - CASE WHEN temporality='delta' AND start_time IS NOT NULL AND start_time