diff --git a/cmd/fanout-docgen/mcptools.go b/cmd/fanout-docgen/mcptools.go index 59c84a170..2a7082df0 100644 --- a/cmd/fanout-docgen/mcptools.go +++ b/cmd/fanout-docgen/mcptools.go @@ -3,9 +3,11 @@ package main import ( "context" "fmt" + "sort" "strings" "github.com/labstack/fanout/internal/mcp" + "github.com/labstack/fanout/internal/panel" ) // toolCount is set by renderMCPTools so the summary line can report it. @@ -150,6 +152,22 @@ func renderMCPTools() ([]byte, error) { b.WriteString("\n") } + b.WriteString("## Panel SQL relations\n\n") + relations := panel.SQLRelations() + names := make([]string, 0, len(relations)) + for name := range relations { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + relation := relations[name] + fmt.Fprintf(&b, "### %s\n\n%s\n\n", name, mdx(relation.Description)) + b.WriteString("| Column | SQL type | Means |\n|---|---|---|\n") + for _, column := range relation.Columns { + fmt.Fprintf(&b, "| `%s` | `%s` | %s |\n", column.Name, column.SQLType, cell(column.Description)) + } + b.WriteString("\n") + } b.WriteString("## Authentication\n\n") b.WriteString("MCP uses OAuth, not a static key, and **not the ingest token** — an agent\n") b.WriteString("presenting the ingest token is rejected. A client discovers the endpoints,\n") diff --git a/internal/agent/dashboard_eval_golden_test.go b/internal/agent/dashboard_eval_golden_test.go index e630e2e4f..2f3916807 100644 --- a/internal/agent/dashboard_eval_golden_test.go +++ b/internal/agent/dashboard_eval_golden_test.go @@ -57,6 +57,15 @@ func TestDashboardEvalGoldensCurrent(t *testing.T) { t.Fatal(err) } defer tools.Close() + foundInventoryGuide := false + for _, definition := range tools.Definitions() { + if definition.Name == "create_dashboard" { + foundInventoryGuide = strings.Contains(definition.Description, "resource_inventory") && strings.Contains(definition.Description, "actual in-window event times") && strings.Contains(definition.Description, "not events or instances") + } + } + if !foundInventoryGuide { + t.Fatal("embedded AI tools lost the shared MCP inventory guidance") + } ctx := dashboard.WithOwner(t.Context(), "owner") authored := panel.Dashboard{Name: "Eval fixture", Time: panel.Time{From: &from, To: &to}, Panels: []panel.Panel{ {ID: "actual_latency", Title: "Latency", Viz: "stat", Query: &panel.Query{From: "spans", Measures: []string{"p95(duration_ms)"}, Where: []string{"service = 'checkout'", "namespace = 'shop'"}}, Unit: "ms", Thresholds: []panel.Threshold{{Value: 100, Status: "warn"}}}, diff --git a/internal/api/panels_test.go b/internal/api/panels_test.go index 1de3bd4c2..820c91e74 100644 --- a/internal/api/panels_test.go +++ b/internal/api/panels_test.go @@ -34,7 +34,7 @@ func (f *fakePanels) ResolveVariables(context.Context, panel.ResolveRequest) (ma } func (f *fakePanels) Schema(_ context.Context, req panel.SchemaRequest) (*panel.Schema, error) { - return &panel.Schema{Window: req.Window}, nil + return &panel.Schema{Window: req.Window, SQLRelations: panel.SQLRelations()}, nil } func servePanels(t *testing.T, engine PanelEngine, method, path, body string) *httptest.ResponseRecorder { @@ -107,7 +107,7 @@ func TestVariableAndSchemaRoutes(t *testing.T) { t.Fatalf("resolve %d %s", rec.Code, rec.Body) } rec = servePanels(t, &fakePanels{}, http.MethodGet, "/api/telemetry/schema?window=6h", "") - if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"window":"6h"`) { + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"window":"6h"`) || !strings.Contains(rec.Body.String(), `"sql_relations":{"resource_inventory":`) || !strings.Contains(rec.Body.String(), `"sql_type":"TIMESTAMPTZ_NS"`) { t.Fatalf("schema %d %s", rec.Code, rec.Body) } } diff --git a/internal/dashboard/defaults_sql.go b/internal/dashboard/defaults_sql.go index e787f9c94..28c19c057 100644 --- a/internal/dashboard/defaults_sql.go +++ b/internal/dashboard/defaults_sql.go @@ -11,7 +11,7 @@ const reachabilitySQL = `WITH raw AS ( AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS AND name='httpcheck.status' QUALIFY row_number() OVER (PARTITION BY series_id,time ORDER BY ingested_unix_nano DESC,value DESC)=1), keyed AS (SELECT *,to_json([namespace,url,method,host])::VARCHAR AS entity, coalesce(namespace,'')||' / '||coalesce(method,'method unknown')||' '||coalesce(url,'endpoint unknown')||' from '||coalesce(host,'vantage unknown') AS endpoint_from FROM raw), - service_hosts AS (SELECT DISTINCT namespace,resource['host.name']::VARCHAR AS host FROM spans WHERE $__window(start_time) AND service IS NOT NULL AND service<>''), + service_hosts AS (SELECT DISTINCT namespace,resource['host.name']::VARCHAR AS host FROM resource_inventory WHERE signal='spans' AND $__window(first_seen) AND service IS NOT NULL AND service<>''), check_hosts AS (SELECT DISTINCT namespace,host FROM keyed WHERE $__window(time)), vantage AS (SELECT count(*)>0 AND bool_and(coalesce(host<>'',false) AND EXISTS (SELECT 1 FROM service_hosts s WHERE s.namespace=check_hosts.namespace AND s.host=check_hosts.host)) AS all_local FROM check_hosts), inventory AS (SELECT DISTINCT entity,endpoint_from||CASE WHEN all_local THEN ' (all checks run on service hosts; outside reachability unverified)' ELSE '' END AS endpoint_from FROM keyed CROSS JOIN vantage WHERE $__window(time)), @@ -111,8 +111,4 @@ func containerCapacitySQL() string { FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host,container` } -const versionObservationsSQL = `WITH resources AS MATERIALIZED ( - SELECT namespace,service,resource,min(start_time) AS seen FROM spans WHERE $__window(start_time) GROUP BY namespace,service,resource - UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM logs WHERE $__window(time) GROUP BY namespace,service,resource - UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM metrics WHERE $__window(time) GROUP BY namespace,service,resource) - SELECT namespace,service,resource['service.version']::VARCHAR AS version,resource['process.runtime.name']::VARCHAR AS runtime_name,resource['process.runtime.version']::VARCHAR AS runtime_version,resource['telemetry.sdk.language']::VARCHAR AS sdk_language,resource['telemetry.sdk.version']::VARCHAR AS sdk_version,seen FROM resources` +const versionObservationsSQL = `SELECT namespace,service,resource['service.version']::VARCHAR AS version,resource['process.runtime.name']::VARCHAR AS runtime_name,resource['process.runtime.version']::VARCHAR AS runtime_version,resource['telemetry.sdk.language']::VARCHAR AS sdk_language,resource['telemetry.sdk.version']::VARCHAR AS sdk_version,first_seen AS seen FROM resource_inventory WHERE $__window(first_seen)` diff --git a/internal/dashboard/resource_inventory_test.go b/internal/dashboard/resource_inventory_test.go new file mode 100644 index 000000000..40f495ac3 --- /dev/null +++ b/internal/dashboard/resource_inventory_test.go @@ -0,0 +1,320 @@ +package dashboard + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "syscall" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/panel" + "github.com/labstack/fanout/internal/query" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" +) + +func baselineResourcePanel(t *testing.T, p panel.Panel) panel.Panel { + t.Helper() + name := "check-states-resource-baseline.sql" + if p.ID == "versions" { + name = "versions-resource-baseline.sql" + } + sql, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + if p.ID == "versions" { + p.SQL = strings.Replace(p.SQL, versionObservationsSQL, strings.TrimSpace(string(sql)), 1) + } else { + p.SQL = strings.TrimSpace(string(sql)) + } + return p +} + +func resourcePanels() []panel.Panel { + var panels []panel.Panel + for _, p := range DefaultSpec().Panels { + if p.ID == "versions" || p.ID == "check_states" { + panels = append(panels, p) + } + } + return panels +} + +func executeResourcePanel(t *testing.T, e *panel.Executor, p panel.Panel, start, end time.Time) panel.Result { + t.Helper() + results, err := e.Run(t.Context(), panel.RunRequest{Dashboard: panel.Dashboard{Name: "Resource inventory comparison", Time: panel.Time{From: &start, To: &end}, Panels: []panel.Panel{p}}}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].Status != panel.StatusOK || results[0].Frame == nil { + t.Fatalf("%s result: %+v", p.ID, results) + } + return results[0] +} + +func TestDefaultResourcePanelsMatchRawAcrossCacheStates(t *testing.T) { + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 2} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + d, err := query.NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + defer d.Close() + e := panel.NewExecutor(d, 30) + start := time.Date(2026, 10, 1, 12, 0, 0, 1, time.UTC) + end := start.Add(2 * time.Minute) + for i := range 2 { + b := telemetrystore.Batch{ID: []string{"first", "second"}[i]} + for j, offset := range []time.Duration{-time.Minute, 0, 30 * time.Second, time.Minute, 2 * time.Minute} { + n := start.Add(offset).UnixNano() + int64(i) + r := map[string]any{"host.name": "same-host", "service.version": "v1", "process.runtime.name": []string{"go", "node"}[i], "process.runtime.version": "1", "telemetry.sdk.language": "go", "telemetry.sdk.version": "2"} + // The version/host that exists only in lookback must stay absent. + if j == 0 { + r = map[string]any{"host.name": "lookback-only", "service.version": "lookback"} + } + b.Spans = append(b.Spans, telemetry.Span{Namespace: []string{"other", "prod"}[i], ServiceName: "api", StartUnixNanos: n, EndUnixNanos: n + 1, Resource: r}) + b.Logs = append(b.Logs, telemetry.Log{Namespace: "prod", ServiceName: "api", TimeUnixNanos: n, Resource: r}) + b.Metrics = append(b.Metrics, telemetry.Metric{Namespace: "prod", ServiceName: "api", Name: "httpcheck.status", Type: "gauge", TimeUnixNanos: n, Value: 1, Resource: r, Attributes: map[string]any{"http.url": "https://example.test", "http.method": "GET", "http.status_class": "2xx"}}) + } + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + if i == 0 { + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + } + for _, state := range []string{"partial", "cold", "full"} { + t.Run(state, func(t *testing.T) { + if state == "cold" { + if _, err := d.DB.Exec(`DELETE FROM read_cache_version`); err != nil { + t.Fatal(err) + } + if err := query.CreateCacheTables(d.DB); err != nil { + t.Fatal(err) + } + } + if state == "full" { + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + for _, p := range resourcePanels() { + want := executeResourcePanel(t, e, baselineResourcePanel(t, p), start, end) + got := executeResourcePanel(t, e, p, start, end) + if !reflect.DeepEqual(want.Frame, got.Frame) { + t.Fatalf("%s frames differ: raw=%+v inventory=%+v", p.ID, want.Frame, got.Frame) + } + // A shifted SQL execution carries its own logical inventory window. + shifted := p + shifted.Time = &panel.PanelTime{Shift: "1m"} + want = executeResourcePanel(t, e, baselineResourcePanel(t, shifted), start, end) + got = executeResourcePanel(t, e, shifted, start, end) + if !reflect.DeepEqual(want.Frame, got.Frame) { + t.Fatalf("shifted %s frames differ: raw=%+v inventory=%+v", p.ID, want.Frame, got.Frame) + } + } + }) + } +} + +func TestDefaultResourceInventoryKeepsVantageNamespacesSeparate(t *testing.T) { + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 2} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + d, err := query.NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + defer d.Close() + e := panel.NewExecutor(d, 30) + start := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + end := start.Add(time.Minute) + b := telemetrystore.Batch{ID: "namespaces", Spans: []telemetry.Span{{Namespace: "other", ServiceName: "api", StartUnixNanos: start.UnixNano(), Resource: map[string]any{"host.name": "same-host"}}}} + for _, offset := range []time.Duration{-time.Minute, 0} { + m := telemetry.Metric{Namespace: "prod", Name: "httpcheck.status", Type: "gauge", TimeUnixNanos: start.Add(offset).UnixNano(), Value: 1, Resource: map[string]any{"host.name": "same-host"}, Attributes: map[string]any{"http.url": "https://example.test", "http.method": "GET", "http.status_class": "2xx"}} + b.Metrics = append(b.Metrics, m) + } + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + for range 2 { + for _, p := range resourcePanels() { + if p.ID != "check_states" { + continue + } + want := executeResourcePanel(t, e, baselineResourcePanel(t, p), start, end) + got := executeResourcePanel(t, e, p, start, end) + if !reflect.DeepEqual(want.Frame, got.Frame) { + t.Fatal("namespace vantage frames differ") + } + for _, value := range defaultValues(t, got, "endpoint_from") { + if strings.Contains(value.(string), "all checks run on service hosts") { + t.Fatal("another namespace made a check local") + } + } + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } +} + +// Opt-in evidence uses an extracted immutable snapshot and writes only to the +// supplied evidence directory and its separate disposable query cache. +func TestResourceInventoryFrozenSnapshotFrames(t *testing.T) { + root := os.Getenv("FANOUT_TEST_INVENTORY_SNAPSHOT") + if root == "" { + t.Skip("set FANOUT_TEST_INVENTORY_SNAPSHOT to an extracted telemetry repository") + } + evidence := os.Getenv("FANOUT_TEST_INVENTORY_EVIDENCE") + if evidence == "" { + t.Fatal("FANOUT_TEST_INVENTORY_EVIDENCE is required") + } + if err := os.MkdirAll(evidence, 0755); err != nil { + t.Fatal(err) + } + cfg := config.Config{DataDir: filepath.Join(evidence, "runtime"), DuckDBMemory: "8GB", DuckDBThreads: 8, DuckDBMaxConns: 4} + repo, err := telemetrystore.Open(root) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + d, err := query.NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + defer d.Close() + e := panel.NewExecutor(d, 30) + var engine string + if err := d.DB.QueryRow(`SELECT version()`).Scan(&engine); err != nil { + t.Fatal(err) + } + start := time.Date(2026, 10, 16, 15, 47, 42, 261874001, time.UTC) + end := start.Add(time.Hour) + write := func(name string, value any) { + t.Helper() + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(evidence, name), append(data, '\n'), 0600); err != nil { + t.Fatal(err) + } + } + stats := map[string]any{"engine": engine, "start": start, "end": end} + sqlEvidence := map[string]string{} + for _, p := range resourcePanels() { + sqlEvidence[p.ID+"-before"] = baselineResourcePanel(t, p).SQL + sqlEvidence[p.ID+"-after"] = p.SQL + } + write("panel-sql.json", sqlEvidence) + build := time.Now() + passes := 0 + for { + var markers int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_batches`).Scan(&markers); err != nil { + t.Fatal(err) + } + if markers == len(repo.Parquet.BatchMetadata()) { + break + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + passes++ + if passes > 10000 { + t.Fatal("cache did not catch up") + } + } + stats["cache_build_ms"] = float64(time.Since(build).Microseconds()) / 1000 + stats["cache_passes"] = passes + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&count); err != nil { + t.Fatal(err) + } + stats["inventory_rows"] = count + signalRows, err := d.DB.Query(`SELECT signal,count(*) FROM read_resource_inventory GROUP BY signal ORDER BY signal`) + if err != nil { + t.Fatal(err) + } + bySignal := map[string]int{} + for signalRows.Next() { + var signal string + var count int + if err := signalRows.Scan(&signal, &count); err != nil { + t.Fatal(err) + } + bySignal[signal] = count + } + err = signalRows.Err() + signalRows.Close() + if err != nil { + t.Fatal(err) + } + stats["inventory_rows_by_signal"] = bySignal + for _, p := range resourcePanels() { + var before, after []int64 + var want *panel.Frame + for i := range 4 { + r := executeResourcePanel(t, e, baselineResourcePanel(t, p), start, end) + before = append(before, r.ElapsedMS) + if i == 0 { + want = r.Frame + } else if !reflect.DeepEqual(want, r.Frame) { + t.Fatal("unstable baseline frame") + } + } + write(p.ID+"-before.json", want) + for range 4 { + r := executeResourcePanel(t, e, p, start, end) + after = append(after, r.ElapsedMS) + if !reflect.DeepEqual(want, r.Frame) { + t.Fatalf("frozen %s rows, ordering or frame metadata differ", p.ID) + } + write(p.ID+"-after.json", r.Frame) + } + stats[p.ID+"_before_ms"] = before + stats[p.ID+"_after_ms"] = after + stats[p.ID+"_rows"] = want.Rows + } + stats["cache_wal_bytes_before_checkpoint"] = int64(0) + if info, err := os.Stat(cfg.QueryDuckDBPath() + ".wal"); err == nil { + stats["cache_wal_bytes_before_checkpoint"] = info.Size() + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if _, err := d.DB.Exec(`CHECKPOINT`); err != nil { + t.Fatal(err) + } + if info, err := os.Stat(cfg.QueryDuckDBPath()); err == nil { + stats["cache_database_bytes"] = info.Size() + } else { + t.Fatal(err) + } + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + t.Fatal(err) + } + rss := usage.Maxrss + if runtime.GOOS != "darwin" { + rss *= 1024 + } + stats["peak_rss_bytes"] = rss + write("measurements.json", stats) + t.Logf("snapshot evidence: %+v", stats) +} diff --git a/internal/dashboard/testdata/check-states-resource-baseline.sql b/internal/dashboard/testdata/check-states-resource-baseline.sql new file mode 100644 index 000000000..de1ba642f --- /dev/null +++ b/internal/dashboard/testdata/check-states-resource-baseline.sql @@ -0,0 +1,27 @@ +WITH raw AS ( + SELECT series_id,ingested_unix_nano,namespace,time,value,type,attributes['http.url']::VARCHAR AS url, + attributes['http.method']::VARCHAR AS method,resource['host.name']::VARCHAR AS host, + attributes['http.status_class']::VARCHAR AS class,attributes['http.status_code']::VARCHAR AS code + FROM metrics WHERE time >= make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS + AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS AND name='httpcheck.status' QUALIFY row_number() OVER (PARTITION BY series_id,time ORDER BY ingested_unix_nano DESC,value DESC)=1), + keyed AS (SELECT *,to_json([namespace,url,method,host])::VARCHAR AS entity, + coalesce(namespace,'')||' / '||coalesce(method,'method unknown')||' '||coalesce(url,'endpoint unknown')||' from '||coalesce(host,'vantage unknown') AS endpoint_from FROM raw), + service_hosts AS (SELECT DISTINCT namespace,resource['host.name']::VARCHAR AS host FROM spans WHERE $__window(start_time) AND service IS NOT NULL AND service<>''), + check_hosts AS (SELECT DISTINCT namespace,host FROM keyed WHERE $__window(time)), + vantage AS (SELECT count(*)>0 AND bool_and(coalesce(host<>'',false) AND EXISTS (SELECT 1 FROM service_hosts s WHERE s.namespace=check_hosts.namespace AND s.host=check_hosts.host)) AS all_local FROM check_hosts), + inventory AS (SELECT DISTINCT entity,endpoint_from||CASE WHEN all_local THEN ' (all checks run on service hosts; outside reachability unverified)' ELSE '' END AS endpoint_from FROM keyed CROSS JOIN vantage WHERE $__window(time)), + attempts AS (SELECT entity,time, + CASE WHEN bool_or(url IS NULL OR url='' OR method IS NULL OR method='' OR host IS NULL OR host='' OR type NOT IN ('gauge','updown') OR NOT isfinite(value) OR (class IS NULL AND code IS NULL)) THEN 2 + WHEN bool_or(value>0 AND (class IN ('2xx','3xx') OR try_cast(code AS INTEGER) BETWEEN 200 AND 399)) THEN 0 ELSE 1 END AS failure + FROM keyed GROUP BY entity,time), + gaps AS (SELECT *,epoch_ns(time::TIMESTAMP_NS)::HUGEINT-epoch_ns(lag(time) OVER (PARTITION BY entity ORDER BY time)::TIMESTAMP_NS)::HUGEINT AS gap_ns FROM attempts), + cadence AS (SELECT *,last_value(CASE WHEN gap_ns>0 THEN gap_ns END IGNORE NULLS) OVER (PARTITION BY entity ORDER BY time) AS interval_ns FROM gaps), + buckets AS (SELECT b::TIMESTAMPTZ_NS AS time,least((b+INTERVAL '1 minute')::TIMESTAMPTZ_NS,$__to::TIMESTAMP_NS::TIMESTAMPTZ_NS) AS at + FROM generate_series(time_bucket(INTERVAL '1 minute',$__from::TIMESTAMP_NS),$__to::TIMESTAMP_NS,INTERVAL '1 minute') t(b) WHERE b::TIMESTAMPTZ_NS < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS), + grid AS (SELECT * FROM inventory CROSS JOIN buckets), + states AS (SELECT g.time,g.endpoint_from,g.entity, + CASE WHEN a.interval_ns IS NULL OR a.failure=2 THEN 2 WHEN epoch_ns(g.at::TIMESTAMP_NS)::HUGEINT-epoch_ns(a.time::TIMESTAMP_NS)::HUGEINT>=2*a.interval_ns THEN 2 ELSE a.failure END::DOUBLE AS failure, + CASE WHEN a.interval_ns IS NULL THEN 'reporting interval unknown' WHEN epoch_ns(g.at::TIMESTAMP_NS)::HUGEINT-epoch_ns(a.time::TIMESTAMP_NS)::HUGEINT>=2*a.interval_ns THEN 'no recent result' WHEN a.failure=2 THEN 'check metadata unsupported or missing' WHEN a.failure=1 THEN 'down' ELSE 'reachable' END AS state + FROM grid g ASOF LEFT JOIN cadence a ON g.entity=a.entity AND g.at>a.time), + ranked AS (SELECT *,arg_max(failure,time) OVER (PARTITION BY entity) AS current_failure FROM states) + SELECT time,endpoint_from,failure,state FROM ranked ORDER BY current_failure DESC,time DESC,endpoint_from diff --git a/internal/dashboard/testdata/versions-resource-baseline.sql b/internal/dashboard/testdata/versions-resource-baseline.sql new file mode 100644 index 000000000..54a7a9637 --- /dev/null +++ b/internal/dashboard/testdata/versions-resource-baseline.sql @@ -0,0 +1,5 @@ +WITH resources AS MATERIALIZED ( + SELECT namespace,service,resource,min(start_time) AS seen FROM spans WHERE $__window(start_time) GROUP BY namespace,service,resource + UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM logs WHERE $__window(time) GROUP BY namespace,service,resource + UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM metrics WHERE $__window(time) GROUP BY namespace,service,resource) + SELECT namespace,service,resource['service.version']::VARCHAR AS version,resource['process.runtime.name']::VARCHAR AS runtime_name,resource['process.runtime.version']::VARCHAR AS runtime_version,resource['telemetry.sdk.language']::VARCHAR AS sdk_language,resource['telemetry.sdk.version']::VARCHAR AS sdk_version,seen FROM resources diff --git a/internal/mcp/describe_test.go b/internal/mcp/describe_test.go index 905114a38..7b60243e7 100644 --- a/internal/mcp/describe_test.go +++ b/internal/mcp/describe_test.go @@ -2,6 +2,7 @@ package mcp import ( "context" + "encoding/json" "regexp" "strings" "testing" @@ -53,6 +54,12 @@ func TestDescribeToolsMatchesWhatAClientIsServed(t *testing.T) { served := map[string]string{} for _, tool := range listed.Tools { served[tool.Name] = tool.Description + if tool.Name == "get_telemetry_schema" { + encoded, err := json.Marshal(tool.OutputSchema) + if err != nil || !strings.Contains(string(encoded), `"sql_relations"`) || !strings.Contains(string(encoded), `"sql_type"`) { + t.Fatalf("MCP SQL relation output schema: %s %v", encoded, err) + } + } } for _, doc := range docs { description, ok := served[doc.Name] diff --git a/internal/mcp/panel_examples.go b/internal/mcp/panel_examples.go index 12a9ef6d3..a7bc0ee64 100644 --- a/internal/mcp/panel_examples.go +++ b/internal/mcp/panel_examples.go @@ -10,4 +10,8 @@ const metricMeasureGuide = `Metrics: gauge/updown | counter | histogram. count() const metricExample = `{"name":"Metric measures","panels":[{"id":"network","title":"Network bytes per second","viz":"timeseries","query":{"from":"metrics","where":["name = 'system.network.io'"],"measures":["rate()"],"by":["attributes['direction']"],"bucket":"auto"}},{"id":"duration","title":"Request duration p95","viz":"timeseries","query":{"from":"metrics","where":["name = 'http.server.request.duration'"],"measures":["p95(value)"],"bucket":"auto"},"unit":"s"}]}` -const specGuide = baseSpecGuide + " " + m2SpecGuide + " Example 1: Reference shapes; use only the types the question needs. " + investigationExample + " Example 2: Reference shapes; use only the types the question needs. " + paymentExample + " Metric examples; use when the discovered type fits. " + metricExample +const resourceInventoryGuide = `Panel SQL also supports resource_inventory: signal, namespace, service, full typed resource, first_seen and last_seen. One row per distinct full resource per signal/namespace/service observed inside the logical panel window; first_seen/last_seen are actual in-window event times. Select signal='spans', 'logs' or 'metrics' and use resource['key'] with literal dotted keys. count(*) counts resource identities per signal, not events or instances. Declare $__window(first_seen). This relation is SQL-only: query.from and variables remain spans|logs|metrics; unscoped SQL cannot read it.` + +const resourceInventoryExample = `{"name":"Observed resources","panels":[{"id":"resources","title":"Span resources observed in this window","viz":"table","sql":"SELECT namespace,service,resource['service.version']::VARCHAR AS version,first_seen,last_seen FROM resource_inventory WHERE signal='spans' AND $__window(first_seen) ORDER BY namespace,service,version,first_seen,last_seen"}]}` + +const specGuide = baseSpecGuide + " " + m2SpecGuide + " Example 1: Reference shapes; use only the types the question needs. " + investigationExample + " Example 2: Reference shapes; use only the types the question needs. " + paymentExample + " Metric examples; use when the discovered type fits. " + metricExample + " Observed-resource SQL example. " + resourceInventoryExample diff --git a/internal/mcp/panel_examples_test.go b/internal/mcp/panel_examples_test.go index fb3758d12..9df362cdd 100644 --- a/internal/mcp/panel_examples_test.go +++ b/internal/mcp/panel_examples_test.go @@ -302,3 +302,31 @@ func TestMetricExamplesExecuteWithObservations(t *testing.T) { } } } + +func TestResourceInventoryExampleExecutesThroughPanelTools(t *testing.T) { + now := time.Now().UTC() + s := panelServerFixtureAt(t, true, now) + var d panel.Dashboard + if err := json.Unmarshal([]byte(resourceInventoryExample), &d); err != nil { + t.Fatal(err) + } + if err := s.panels.Validate(t.Context(), &d); err != nil { + t.Fatal(err) + } + _, preview, err := s.previewPanels(t.Context(), nil, PreviewInput{Panels: d.Panels, Time: &d.Time}) + if err != nil || len(preview.Panels) != 1 || preview.Panels[0].Status != "ok" || preview.Panels[0].Rows != 1 { + t.Fatalf("inventory preview=%+v %v", preview, err) + } + _, fragment, err := s.queryTelemetry(t.Context(), nil, QueryTelemetryInput{Panel: d.Panels[0]}) + if err != nil || len(fragment.Results) != 1 || fragment.Results[0].Status != "ok" { + t.Fatalf("inventory query=%+v %v", fragment, err) + } + _, created, err := s.dashboardCreate(t.Context(), ownerRequest(), DashboardCreateInput{Dashboard: d}) + if err != nil || created.Dashboard.Version != 1 { + t.Fatalf("inventory create=%+v %v", created, err) + } + _, schema, err := s.telemetrySchema(t.Context(), nil, SchemaInput{}) + if err != nil || len(schema.Signals) != 3 || len(schema.SQLRelations["resource_inventory"].Columns) != 6 { + t.Fatalf("inventory schema=%+v %v", schema, err) + } +} diff --git a/internal/mcp/panels.go b/internal/mcp/panels.go index ceded8dea..bfe52b459 100644 --- a/internal/mcp/panels.go +++ b/internal/mcp/panels.go @@ -44,7 +44,7 @@ type PreviewOutput struct { Problems []panel.Problem `json:"problems,omitempty"` } -const baseSpecGuide = `A dashboard is {name, description, time:{range}, variables, panels}. Each panel: {id (lowercase), title, viz, width 1-12, query or sql}. viz: stat or gauge (one measure; gauge needs min and max), timeseries (bucket auto, at most one by), bar (one or two by), table (up to three by), text (content in Markdown), heatmap (bucket auto, no by), histogram (at most one by), scatter (exactly two measures in x then y order, one item dimension and an optional colour dimension), state_timeline (one measure, one item dimension, bucket auto and thresholds required). Rollup panel types are service_map (dependency graph) and health (overview metrics, error trend and health distribution). They require query.from=spans, no measures, by, bucket or sort, and only namespace/service equality filters with a literal or single-value variable; explicit casts and values with surrounding whitespace are rejected. Reads default to 400 services/edges and are capped at 400 even when query.limit is larger. Service scope applies before the limit; map nodes carry p95_ms and edges carry average_ms; error_rate is percent. Row panel types are logs, log_patterns and traces. Logs and traces take no measures, by or bucket. Patterns use count() by body_template. Patterns group only by body_template. Severity is the most frequent severity, ties choose the higher severity. Service is the most frequent service, ties choose ascending service name. Both describe the scoped, redacted events, not extra grouping dimensions. Each pattern returns dominant severity, top service, count and a dense trend, computed in the same bounded query after redaction and scoped filters. Redaction applies before filtering, grouping, display and drill-down. Row panels cannot use SQL. "erroring" means slowest erroring traces: filter to erroring traces, then rank by root duration. Trace candidate ranking is approximate when filters exclude the root span, as with exemplars. Categorical charts use fixed-order colour slots. Categorical options.top defaults to 6, maximum 6. Structured panels: the server computes Other; series are chosen worst-first by confidence (Wilson lower bound for error rates; at least 20 samples for latency); the rest fold into Other (N). Count, rate, sum and share rank largest first. Error rates require at least 20 requests for eligible Wilson ranking (z=1.96). Percentiles, quantile, avg, max and min require at least 20 non-null numeric samples for eligible value ranking and use better (explicit or inferred). Ineligible series follow eligible ones by sample count descending. Ties use sample count descending, then name ascending. SQL panels: series past six are left out, with a note. state_timeline defaults to 8 rows, maximum 20; other noncategorical panels use the same limits. Scatter supports options.x_scale and options.y_scale (log or linear), with distinct x_unit for x and unit for y, each inferred independently when omitted. Rows returned use the count unit. State timelines preserve null and absent buckets as unknown and keep the top items without merging them into Other. Scatter requires a structured query. SQL state_timeline requires timestamp time, a string item, one numeric measure and optional string state; its interval is one minute and thresholds are required. Items rank by volume (row count) for top N. Previous-period comparison is not drawn for scatter or state_timeline. Distribution panels use a structured query with one count() measure and histogram: {field: duration_ms, buckets: log2} for spans or {field: value, buckets: explicit} for metrics. Negative span durations clamp to [0,1); +Inf goes to overflow; NaN is excluded because it is not a measurement. Metric histograms use stored temporality and shared per-series increases; resets contribute current counts. Delta histograms sum observation weights. query: {from: spans|logs|metrics, where: [filter expressions], measures: [fn(field) as alias], by: [fields], bucket, sort, limit}. Measures: count(), rate() per second, error_rate() percent, share() percent, avg/min/max/sum(field), last(value) for metrics, p50/p75/p90/p95/p99(field), quantile(field, 0.999), count_distinct(field). Fields are columns from get_telemetry_schema or attributes['key'] / resource['key'] with the key written literally. Filters are SQL boolean expressions: service = $service, kind = 'SPAN_KIND_SERVER', status = 'STATUS_CODE_ERROR', http_route IN $routes, attributes['http.response.status_code']::INTEGER >= 500. Variables: {name, kind: query|custom|constant|text, from, field, default, include_all}; $__all selects All and removes filters that use the variable. sql: one SELECT over spans, logs, metrics, service_rollup or edge_rollup that filters time with $__window(column). Use values exactly as get_telemetry_schema lists them. Example panel: {"id":"latency","title":"Checkout latency","viz":"timeseries","width":8,"query":{"from":"spans","where":["service = 'checkout'","kind = 'SPAN_KIND_SERVER'"],"measures":["p50(duration_ms)","p95(duration_ms)","p99(duration_ms)"],"bucket":"auto"},"unit":"ms","thresholds":[{"value":1500,"status":"bad"}]}.` + " " + metricMeasureGuide +const baseSpecGuide = `A dashboard is {name, description, time:{range}, variables, panels}. Each panel: {id (lowercase), title, viz, width 1-12, query or sql}. viz: stat or gauge (one measure; gauge needs min and max), timeseries (bucket auto, at most one by), bar (one or two by), table (up to three by), text (content in Markdown), heatmap (bucket auto, no by), histogram (at most one by), scatter (exactly two measures in x then y order, one item dimension and an optional colour dimension), state_timeline (one measure, one item dimension, bucket auto and thresholds required). Rollup panel types are service_map (dependency graph) and health (overview metrics, error trend and health distribution). They require query.from=spans, no measures, by, bucket or sort, and only namespace/service equality filters with a literal or single-value variable; explicit casts and values with surrounding whitespace are rejected. Reads default to 400 services/edges and are capped at 400 even when query.limit is larger. Service scope applies before the limit; map nodes carry p95_ms and edges carry average_ms; error_rate is percent. Row panel types are logs, log_patterns and traces. Logs and traces take no measures, by or bucket. Patterns use count() by body_template. Patterns group only by body_template. Severity is the most frequent severity, ties choose the higher severity. Service is the most frequent service, ties choose ascending service name. Both describe the scoped, redacted events, not extra grouping dimensions. Each pattern returns dominant severity, top service, count and a dense trend, computed in the same bounded query after redaction and scoped filters. Redaction applies before filtering, grouping, display and drill-down. Row panels cannot use SQL. "erroring" means slowest erroring traces: filter to erroring traces, then rank by root duration. Trace candidate ranking is approximate when filters exclude the root span, as with exemplars. Categorical charts use fixed-order colour slots. Categorical options.top defaults to 6, maximum 6. Structured panels: the server computes Other; series are chosen worst-first by confidence (Wilson lower bound for error rates; at least 20 samples for latency); the rest fold into Other (N). Count, rate, sum and share rank largest first. Error rates require at least 20 requests for eligible Wilson ranking (z=1.96). Percentiles, quantile, avg, max and min require at least 20 non-null numeric samples for eligible value ranking and use better (explicit or inferred). Ineligible series follow eligible ones by sample count descending. Ties use sample count descending, then name ascending. SQL panels: series past six are left out, with a note. state_timeline defaults to 8 rows, maximum 20; other noncategorical panels use the same limits. Scatter supports options.x_scale and options.y_scale (log or linear), with distinct x_unit for x and unit for y, each inferred independently when omitted. Rows returned use the count unit. State timelines preserve null and absent buckets as unknown and keep the top items without merging them into Other. Scatter requires a structured query. SQL state_timeline requires timestamp time, a string item, one numeric measure and optional string state; its interval is one minute and thresholds are required. Items rank by volume (row count) for top N. Previous-period comparison is not drawn for scatter or state_timeline. Distribution panels use a structured query with one count() measure and histogram: {field: duration_ms, buckets: log2} for spans or {field: value, buckets: explicit} for metrics. Negative span durations clamp to [0,1); +Inf goes to overflow; NaN is excluded because it is not a measurement. Metric histograms use stored temporality and shared per-series increases; resets contribute current counts. Delta histograms sum observation weights. query: {from: spans|logs|metrics, where: [filter expressions], measures: [fn(field) as alias], by: [fields], bucket, sort, limit}. Measures: count(), rate() per second, error_rate() percent, share() percent, avg/min/max/sum(field), last(value) for metrics, p50/p75/p90/p95/p99(field), quantile(field, 0.999), count_distinct(field). Fields are columns from get_telemetry_schema or attributes['key'] / resource['key'] with the key written literally. Filters are SQL boolean expressions: service = $service, kind = 'SPAN_KIND_SERVER', status = 'STATUS_CODE_ERROR', http_route IN $routes, attributes['http.response.status_code']::INTEGER >= 500. Variables: {name, kind: query|custom|constant|text, from, field, default, include_all}; $__all selects All and removes filters that use the variable. sql: one SELECT over spans, logs, metrics, service_rollup, edge_rollup or resource_inventory that filters time with $__window(column). Use values exactly as get_telemetry_schema lists them. Example panel: {"id":"latency","title":"Checkout latency","viz":"timeseries","width":8,"query":{"from":"spans","where":["service = 'checkout'","kind = 'SPAN_KIND_SERVER'"],"measures":["p50(duration_ms)","p95(duration_ms)","p99(duration_ms)"],"bucket":"auto"},"unit":"ms","thresholds":[{"value":1500,"status":"bad"}]}.` + " " + metricMeasureGuide + " " + resourceInventoryGuide func (s *Server) registerPanelTools() { readOnly := &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPtr(false)} @@ -57,7 +57,7 @@ func (s *Server) registerPanelTools() { mcp.AddTool(s.mcp, &mcp.Tool{ Name: "get_telemetry_schema", Title: "Telemetry schema", - Description: "List the signals, columns with their common values, attribute keys seen per service, metric names, services, measure functions and units. Read this before drafting panels so filters use values that exist. See create_dashboard for the spec guide.", + Description: "List the signals, columns with their common values, attribute keys seen per service, metric names, services, measure functions, units and SQL-only relations in sql_relations. Read this before drafting panels so filters use values that exist. See create_dashboard for the spec guide.", Annotations: readOnly, }, s.telemetrySchema) mcp.AddTool(s.mcp, &mcp.Tool{ diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index a862cfb5c..cedc1671d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -47,7 +47,7 @@ const ( ) var ( - ReadCachePendingBatches = promauto.NewGauge(prometheus.GaugeOpts{Name: "fanout_read_cache_pending_batches", Help: "Active immutable batches not covered at the start of the last read-cache pass"}) + ReadCachePendingBatches = promauto.NewGauge(prometheus.GaugeOpts{Name: "fanout_read_cache_pending_batches", Help: "Nonempty active immutable batches across spans, logs and metrics not covered at the start of the last read-cache pass"}) IngestBackpressureSeconds = promauto.NewCounter(prometheus.CounterOpts{ Name: "fanout_ingest_backpressure_seconds_total", Help: "Total seconds the ingest writer waited for compaction debt to retire", diff --git a/internal/panel/check_test.go b/internal/panel/check_test.go index 48b2b1716..4618aeb9f 100644 --- a/internal/panel/check_test.go +++ b/internal/panel/check_test.go @@ -130,7 +130,7 @@ func TestCheckReturnsEngineInterruptAsOperationalError(t *testing.T) { func TestSQLCTEsCannotShadowTelemetry(t *testing.T) { engine, _ := newTestEngine(t) - for _, name := range []string{"logs", "spans", "metrics", "service_rollup", "edge_rollup", "LOGS", "safe"} { + for _, name := range []string{"logs", "spans", "metrics", "service_rollup", "edge_rollup", "resource_inventory", "RESOURCE_INVENTORY", "LOGS", "safe", "resources"} { t.Run(name, func(t *testing.T) { d := Dashboard{Name: "CTE", Panels: []Panel{{ID: "p", Title: "P", Viz: "table", SQL: "WITH " + name + " AS (SELECT * FROM logs WHERE $__window(time)) SELECT count(*) FROM " + name}}} Normalize(&d) @@ -138,7 +138,7 @@ func TestSQLCTEsCannotShadowTelemetry(t *testing.T) { if err != nil { t.Fatal(err) } - if name == "safe" { + if name == "safe" || name == "resources" { if len(problems) > 0 { t.Fatalf("normal CTE rejected: %v", problems) } diff --git a/internal/panel/exec.go b/internal/panel/exec.go index 63826655d..1b6259b1b 100644 --- a/internal/panel/exec.go +++ b/internal/panel/exec.go @@ -428,6 +428,7 @@ func (e *Executor) totals(ctx context.Context, p *Panel, checked *Checked, scope } func (e *Executor) runScope(ctx context.Context, p *Panel, checked *Checked, scope Scope) (*Frame, string, error) { + ctx = queryrows.WithInventoryWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End}) read := metricReadScope(p, scope) if p.SQL != "" { // SQL panels may look back up to the metric lookback before their diff --git a/internal/panel/schema.go b/internal/panel/schema.go index 9c5e44394..ef87c833d 100644 --- a/internal/panel/schema.go +++ b/internal/panel/schema.go @@ -16,11 +16,12 @@ type SchemaRequest struct { } type Schema struct { - Window string `json:"window"` - Signals map[string]SignalSchema `json:"signals"` - Services []Option `json:"services"` - Measures []string `json:"measures"` - Units []string `json:"units"` + SQLRelations map[string]SQLRelationSchema `json:"sql_relations"` + Window string `json:"window"` + Signals map[string]SignalSchema `json:"signals"` + Services []Option `json:"services"` + Measures []string `json:"measures"` + Units []string `json:"units"` } type SignalSchema struct { @@ -82,7 +83,7 @@ func (e *Executor) Schema(ctx context.Context, req SchemaRequest) (*Schema, erro end := e.now().UTC() start := end.Add(-span) ctx = queryrows.WithWindow(ctx, queryrows.Window{Start: start, End: end, Namespace: req.Namespace}) - out := &Schema{Window: req.Window, Signals: map[string]SignalSchema{}, Measures: measureNames(), Units: unitNames()} + out := &Schema{Window: req.Window, Signals: map[string]SignalSchema{}, SQLRelations: SQLRelations(), Measures: measureNames(), Units: unitNames()} for _, name := range SignalNames() { sig := signals[name] values, err := e.lowCardinality(ctx, sig, start, end, req.Namespace) diff --git a/internal/panel/schema_test.go b/internal/panel/schema_test.go index 0e991fd3d..472ad96da 100644 --- a/internal/panel/schema_test.go +++ b/internal/panel/schema_test.go @@ -16,6 +16,16 @@ func TestSchemaDiscoversColumnsAttributesAndValues(t *testing.T) { t.Fatal(err) } spans := schema.Signals["spans"] + if len(schema.Signals) != 3 || len(schema.SQLRelations) != 1 || len(schema.SQLRelations["resource_inventory"].Columns) != 6 { + t.Fatalf("SQL relation discovery: %+v", schema) + } + if got := schema.SQLRelations["resource_inventory"]; got.Columns[3].SQLType != "VARIANT" || got.Columns[4].SQLType != "TIMESTAMPTZ_NS" { + t.Fatalf("inventory types: %+v", got) + } + d := Dashboard{Name: "Invalid structured inventory", Panels: []Panel{{ID: "resources", Title: "Resources", Viz: "stat", Query: &Query{From: "resource_inventory", Measures: []string{"count()"}}}}} + if err := e.Validate(t.Context(), &d); err == nil { + t.Fatal("SQL-only relation accepted as structured signal") + } if spans.TimeColumn != "start_time" { t.Fatalf("time column = %q", spans.TimeColumn) } diff --git a/internal/panel/sql_relations.go b/internal/panel/sql_relations.go new file mode 100644 index 000000000..8e2e33494 --- /dev/null +++ b/internal/panel/sql_relations.go @@ -0,0 +1,30 @@ +package panel + +// SQLRelationSchema describes a panel-SQL-only relation. Its rows are not raw +// events, so it deliberately has neither a signal time column nor sample counts. +type SQLRelationSchema struct { + Description string `json:"description"` + Columns []SQLColumnSchema `json:"columns"` +} + +type SQLColumnSchema struct { + Field + SQLType string `json:"sql_type"` +} + +// SQLRelations is the shared catalog for discovery and generated documentation. +func SQLRelations() map[string]SQLRelationSchema { + return map[string]SQLRelationSchema{ + "resource_inventory": { + Description: "Distinct full resource identities per signal, namespace and service observed inside the logical panel window [start,end). Panel SQL only; unavailable to structured query.from, variables and unscoped SQL. count(*) counts resource identities per signal, not events or instances. Resource extrema describe actual observations inside the selected window, not a historical registry.", + Columns: []SQLColumnSchema{ + {Field: str("signal", "spans, logs or metrics; select the required signal"), SQLType: "VARCHAR"}, + {Field: str("namespace", "Original resource namespace, preserving NULL and empty values"), SQLType: "VARCHAR"}, + {Field: str("service", "Original service name, preserving NULL and empty values"), SQLType: "VARCHAR"}, + {Field: dict("resource", "Complete typed resource; use resource['key'], with dotted keys written literally"), SQLType: "VARIANT"}, + {Field: ts("first_seen", "Earliest actual event observation inside the selected panel window"), SQLType: "TIMESTAMPTZ_NS"}, + {Field: ts("last_seen", "Latest actual event observation inside the selected panel window"), SQLType: "TIMESTAMPTZ_NS"}, + }, + }, + } +} diff --git a/internal/panel/sqlpanel.go b/internal/panel/sqlpanel.go index c4bf73de1..61b00854f 100644 --- a/internal/panel/sqlpanel.go +++ b/internal/panel/sqlpanel.go @@ -82,7 +82,7 @@ func redactSQLLogTables(ctx context.Context, parser Parser, value any, ctes map[ entry, _ := entry.(map[string]any) name, _ := entry["key"].(string) switch strings.ToLower(name) { - case "logs", "spans", "metrics", "service_rollup", "edge_rollup": + case "logs", "spans", "metrics", "service_rollup", "edge_rollup", "resource_inventory": return errors.New("CTE names must not shadow telemetry relations") } definition, _ := entry["value"].(map[string]any) diff --git a/internal/query/batch_cache.go b/internal/query/batch_cache.go index a1a085c6f..91f50b6b4 100644 --- a/internal/query/batch_cache.go +++ b/internal/query/batch_cache.go @@ -17,7 +17,7 @@ import ( // readCacheVersion covers both schema and aggregation semantics. These tables // are disposable; a mismatch rebuilds them together from immutable Parquet. -const readCacheVersion = 5 +const readCacheVersion = 6 const ( readCacheBatchLimit = 64 @@ -78,6 +78,7 @@ func createBatchCaches(db *sql.DB) error { for _, stmt := range []string{ `CREATE TABLE IF NOT EXISTS read_edge_buckets (namespace VARCHAR, bucket TIMESTAMPTZ_NS, spans BIGINT)`, `CREATE TABLE IF NOT EXISTS read_batches (batch_id VARCHAR PRIMARY KEY,replacement_id VARCHAR)`, + `CREATE TABLE IF NOT EXISTS read_resource_inventory (batch_id VARCHAR,signal VARCHAR,namespace VARCHAR,service VARCHAR,resource VARIANT,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS)`, `CREATE TABLE IF NOT EXISTS read_trace_candidates (trace_id VARCHAR PRIMARY KEY,namespace VARCHAR,service VARCHAR,min_start BIGINT,max_start BIGINT,max_end BIGINT,has_error INTEGER)`, `CREATE TABLE IF NOT EXISTS read_trace_parts (batch_id VARCHAR, namespace VARCHAR, service VARCHAR, trace_id VARCHAR, min_start BIGINT, max_start BIGINT, max_end BIGINT, has_error INTEGER)`, } { @@ -169,7 +170,7 @@ func (d *Duck) RefreshReadCaches(ctx context.Context) (rows int64, err error) { sort.SliceStable(batches, func(i, j int) bool { return newestBatchEvent(batches[i]) < newestBatchEvent(batches[j]) }) pendingCount := 0 for _, b := range batches { - if !markers[b.ID] && b.Spans > 0 { + if !markers[b.ID] && b.Spans+b.Logs+b.Metrics > 0 { pendingCount++ } } @@ -178,7 +179,7 @@ func (d *Duck) RefreshReadCaches(ctx context.Context) (rows int64, err error) { pendingRows := 0 for i := len(batches) - 1; i >= 0; i-- { if !markers[batches[i].ID] { - rows := batches[i].Spans + rows := batches[i].Spans + batches[i].Logs + batches[i].Metrics // A complete file is indivisible. Admit one oversized file to make // progress, otherwise bound new work by rows as well as file count. if len(pending) > 0 && pendingRows+rows > readCacheRowBudget { @@ -226,7 +227,7 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada } } if len(stale) > 0 { - for _, table := range []string{"read_trace_parts", "read_batches"} { + for _, table := range []string{"read_resource_inventory", "read_trace_parts", "read_batches"} { if err := exec("DELETE FROM " + table + " WHERE batch_id IN (" + idsSQL(stale) + ")"); err != nil { return 0, err } @@ -245,9 +246,8 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada return 0, err } } - // Bind spans once for the whole pass. Materialize new - // batch parts and one indexed candidate per trace. The temporary parts table - // feeds both inserts without rereading Parquet or scanning retained parts. + // Each aggregation scans only the pass's new files. Materialize trace + // parts once for both trace inserts without scanning retained parts. source := func(signal string) string { var selected []telemetry.BatchMetadata for _, b := range batches { @@ -265,6 +265,17 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada physical := strings.Replace(d.physicalSource(signal, selected), "SELECT ", "SELECT filename AS __fanout_filename,", 1) return strings.Replace(cleanSource(signal, physical), "SELECT\n", "SELECT "+d.batchIDColumn(signal, "__fanout_filename")+" AS batch_id,\n", 1) } + for _, signal := range []string{"spans", "logs", "metrics"} { + if rows := source(signal); rows != "" { + clock := "time" + if signal == "spans" { + clock = "start_time" + } + if err := exec(`INSERT INTO read_resource_inventory SELECT batch_id,` + sqlLiteral(signal) + `,namespace,service,resource,min(` + clock + `),max(` + clock + `) FROM (` + rows + `) GROUP BY batch_id,namespace,service,resource`); err != nil { + return 0, err + } + } + } if spans := source("spans"); spans != "" { if err := exec(`CREATE OR REPLACE TEMP TABLE cache_new_trace_parts AS SELECT batch_id,coalesce(namespace,'') AS namespace,coalesce(service,'') AS service,trace_id, min(start_unix_nano) AS min_start,max(start_unix_nano) AS max_start,max(end_unix_nano) AS max_end,max(CASE WHEN upper(status) IN ('ERROR','STATUS_CODE_ERROR') THEN 1 ELSE 0 END) AS has_error @@ -308,11 +319,21 @@ func timeLiteral(t time.Time) string { func windowPredicate(column string, w queryrows.Window) string { return column + ">=" + timeLiteral(w.Start) + " AND " + column + "<" + timeLiteral(w.End) } -func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []telemetry.BatchMetadata, w queryrows.Window, sources map[string]string) error { +func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []telemetry.BatchMetadata, w queryrows.Window, inventory bool, sources map[string]string) error { markers, err := batchMarkers(ctx, db) if err != nil { return err } + if inventory { + window, ok := queryrows.ReadInventoryWindow(ctx) + if !ok { + return fmt.Errorf("resource_inventory requires a logical panel window") + } + sources["resource_inventory"] = d.resourceInventorySource(batches, markers, window) + } + if w.Kind == queryrows.RawRead { + return nil + } var cachedIDs []string var cached, uncached []telemetry.BatchMetadata for _, b := range batches { @@ -436,6 +457,7 @@ func (d *Duck) transferBatchCache(ctx context.Context, output string, inputs []s } where := " WHERE batch_id IN (" + idsSQL(inputs) + ")" for _, stmt := range []string{ + `INSERT INTO read_resource_inventory SELECT ` + sqlLiteral(output) + `,signal,namespace,service,resource,min(first_seen),max(last_seen) FROM read_resource_inventory` + where + ` GROUP BY signal,namespace,service,resource`, `INSERT INTO read_trace_parts SELECT ` + sqlLiteral(output) + `,namespace,service,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts` + where + ` GROUP BY 2,3,4`, `INSERT INTO read_batches(batch_id) VALUES (` + sqlLiteral(output) + `)`, `UPDATE read_batches SET replacement_id=` + sqlLiteral(output) + where, diff --git a/internal/query/duck.go b/internal/query/duck.go index dca88af76..1842c8274 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -1788,6 +1788,20 @@ func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (que d.parquetMu.RUnlock() return nil, err } + } else if strings.Contains(strings.ToLower(query), "resource_inventory") { + // The empty validation descriptor must never become a reader. Inspect + // actual references (a literal or unrelated CTE is harmless). + compiled, compileErr := d.compileSnapshot(ctx, conn, query) + if compileErr == nil { + if _, inventory := compiled.names["resource_inventory"]; inventory { + compileErr = fmt.Errorf("resource_inventory requires a repository and logical panel window") + } + } + if compileErr != nil { + endRead(conn, nil) + d.parquetMu.RUnlock() + return nil, compileErr + } } started := time.Now() rows, err := db.QueryContext(ctx, query, args...) diff --git a/internal/query/file_snapshot.go b/internal/query/file_snapshot.go index f410d9716..45e419bdb 100644 --- a/internal/query/file_snapshot.go +++ b/internal/query/file_snapshot.go @@ -126,8 +126,9 @@ func (d *Duck) bindSnapshot(ctx context.Context, db snapshotSQL, query string, w sources["telemetry."+signal] = physical sources[signal] = cleanSource(signal, physical) } - if w.Kind != queryrows.RawRead { - if err := d.aggregateSources(ctx, db, batches, w, sources); err != nil { + _, inventory := compiled.names["resource_inventory"] + if w.Kind != queryrows.RawRead || inventory { + if err := d.aggregateSources(ctx, db, batches, w, inventory, sources); err != nil { return "", err } } @@ -291,7 +292,7 @@ func (d *Duck) compileSnapshot(ctx context.Context, db snapshotSQL, query string d.snapshotMu.Unlock() if !ok { aliases := map[string]string{} - for _, name := range []string{"telemetry.spans", "telemetry.logs", "telemetry.metrics", "spans", "logs", "metrics", "trace_candidates", "trace_tail"} { + for _, name := range []string{"telemetry.spans", "telemetry.logs", "telemetry.metrics", "spans", "logs", "metrics", "trace_candidates", "trace_tail", "resource_inventory"} { alias := "__fanout_snapshot_" + strings.ReplaceAll(name, ".", "_") aliases[name] = "SELECT * FROM " + alias } diff --git a/internal/query/file_snapshot_test.go b/internal/query/file_snapshot_test.go index defc570f9..bc6b51cbe 100644 --- a/internal/query/file_snapshot_test.go +++ b/internal/query/file_snapshot_test.go @@ -172,6 +172,7 @@ func TestBatchCachesStayExactThroughCompactionAndRetention(t *testing.T) { t.Helper() w := queryrows.Window{Start: at.Add(-time.Minute), End: at.Add(time.Minute)} var count int + assertInventoryOracle(t, d, w, 2) w.Kind = queryrows.TraceCandidateRead w.Namespace = "prod" w.Service = "api" @@ -221,7 +222,7 @@ func TestReadCacheCancellationLeavesNoMarkerOrPartialContributions(t *testing.T) if _, err := d.RefreshReadCaches(ctx); err == nil { t.Fatal("cancelled backfill succeeded") } - for _, table := range []string{"read_batches", "read_trace_parts", "read_trace_candidates"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { t.Fatalf("%s partial rows=%d: %v", table, count, err) @@ -244,7 +245,7 @@ func TestFailedBatchCacheTransactionPublishesNoContributions(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err == nil { t.Fatal("broken cache schema accepted") } - for _, table := range []string{"read_batches", "read_trace_parts"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { t.Fatalf("%s partial rows=%d: %v", table, count, err) @@ -321,7 +322,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { if _, err := d.RefreshReadCaches(t.Context()); err != nil { t.Fatal(err) } - for _, stmt := range []string{`UPDATE read_trace_candidates SET min_start=0`, `UPDATE read_cache_version SET version=4`, `INSERT INTO read_edge_buckets VALUES ('ns',now(),10)`, `INSERT INTO rollup_state VALUES ('edge_rollup_v2_substart',1,now()),('edge_rollup_v2_subwindow_end',2,now())`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, `CREATE TABLE endpoint_rollup (value INTEGER)`, `INSERT INTO rollup_state VALUES ('endpoint_rollup:prod',1,now())`} { + for _, stmt := range []string{`UPDATE read_trace_candidates SET min_start=0`, `UPDATE read_resource_inventory SET first_seen=make_timestamp_ns(0)::TIMESTAMPTZ_NS`, `UPDATE read_cache_version SET version=5`, `INSERT INTO read_edge_buckets VALUES ('ns',now(),10)`, `INSERT INTO rollup_state VALUES ('edge_rollup_v2_substart',1,now()),('edge_rollup_v2_subwindow_end',2,now())`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, `CREATE TABLE endpoint_rollup (value INTEGER)`, `INSERT INTO rollup_state VALUES ('endpoint_rollup:prod',1,now())`} { if _, err := d.DB.Exec(stmt); err != nil { t.Fatal(err) } @@ -329,7 +330,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { if err := CreateCacheTables(d.DB); err != nil { t.Fatal(err) } - for _, table := range []string{"read_batches", "read_trace_parts", "read_trace_candidates", "read_edge_buckets"} { + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates", "read_edge_buckets"} { var count int if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 { t.Fatalf("%s survived semantic change: %d %v", table, count, err) @@ -357,7 +358,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) { t.Fatalf("rebuilt bound=%d: %v", minStart, err) } var version int - if err := d.DB.QueryRow(`SELECT version FROM read_cache_version`).Scan(&version); err != nil || version != 5 { + if err := d.DB.QueryRow(`SELECT version FROM read_cache_version`).Scan(&version); err != nil || version != 6 { t.Fatalf("version=%d: %v", version, err) } @@ -557,6 +558,7 @@ func TestTraceCacheFollowsMultipleCompactionLevels(t *testing.T) { t.Fatalf("replacement=%v", batches) } w := queryrows.Window{Start: at.Add(-time.Minute), End: at.Add(2 * time.Minute), Kind: queryrows.TraceCandidateRead} + assertInventoryOracle(t, d, w, 1) var count int if err := d.QueryRowScan(queryrows.WithWindow(t.Context(), w), []any{&count}, `SELECT count(*) FROM trace_candidates`); err != nil || count != 1 { t.Fatalf("candidate=%d: %v", count, err) @@ -633,7 +635,7 @@ func TestReadCacheRowBudgetAndOversizedFileProgress(t *testing.T) { } } -func TestTraceBackfillBudgetsSpanRowsAndReportsOnlyTraceWork(t *testing.T) { +func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) { d, repo := newBatchCacheTest(t) logs := make([]telemetry.Log, readCacheRowBudget) for i := range logs { @@ -648,18 +650,30 @@ func TestTraceBackfillBudgetsSpanRowsAndReportsOnlyTraceWork(t *testing.T) { t.Fatal(err) } } - if _, err := d.RefreshReadCaches(t.Context()); err != nil { + written, err := d.RefreshReadCaches(t.Context()) + if err != nil { t.Fatal(err) } markers, err := batchMarkers(t.Context(), d.DB) - if err != nil || !markers["older-trace"] || !markers["log-heavy-trace"] { - t.Fatalf("logs consumed trace row budget: %v %v", markers, err) + if err != nil || len(markers) != 1 || !markers["log-only"] || written != 1 { + t.Fatalf("all-signal budget/work: %v written=%d err=%v", markers, written, err) } - var count int - if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_parts`).Scan(&count); err != nil || count != 2 { - t.Fatalf("trace parts=%d %v", count, err) + if got := metricFamily(t, "fanout_read_cache_pending_batches").Metric[0].GetGauge().GetValue(); got != 3 { + t.Fatalf("pending all-signal gauge=%v want 3", got) + } + written, err = d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) } - if got := metricFamily(t, "fanout_read_cache_pending_batches").Metric[0].GetGauge().GetValue(); got != 2 { - t.Fatalf("pending trace gauge=%v want 2", got) + markers, err = batchMarkers(t.Context(), d.DB) + if err != nil || !markers["log-heavy-trace"] || markers["older-trace"] || written != 4 { + t.Fatalf("oversized mixed batch: %v written=%d err=%v", markers, written, err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + markers, err = batchMarkers(t.Context(), d.DB) + if err != nil || len(markers) != 3 { + t.Fatalf("remaining batch: %v %v", markers, err) } } diff --git a/internal/query/panel_sql.go b/internal/query/panel_sql.go index 3a5febf8f..e885d5286 100644 --- a/internal/query/panel_sql.go +++ b/internal/query/panel_sql.go @@ -29,10 +29,10 @@ func (d *Duck) PrepareTelemetrySQL(ctx context.Context, query, describe string, return "", nil, err } defer conn.Close() - if err := validateSQLAST(ctx, conn, query); err != nil { + if err := validateSQLAST(ctx, conn, query, true); err != nil { return "", nil, err } - if err := validateSQLAST(ctx, conn, describe); err != nil { + if err := validateSQLAST(ctx, conn, describe, true); err != nil { return "", nil, err } trim := func(s string) string { return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), ";")) } @@ -40,7 +40,17 @@ func (d *Duck) PrepareTelemetrySQL(ctx context.Context, query, describe string, if err != nil { return "", nil, err } - return fmt.Sprintf("SELECT %s FROM (%s) AS _result(%s) LIMIT %d", strings.Join(projection, ","), trim(query), strings.Join(aliases, ","), maxRows), serialized, nil + execution := trim(query) + if maxRows == 0 { + // Timeline validation inspects driver column types with a zero-row + // execution. Bind its descriptor explicitly; real readers still require + // a repository and a logical inventory window. + execution, err = compileRelations(ctx, conn, execution, map[string]string{"resource_inventory": emptyResourceInventory}, map[string]string{}) + if err != nil { + return "", nil, err + } + } + return fmt.Sprintf("SELECT %s FROM (%s) AS _result(%s) LIMIT %d", strings.Join(projection, ","), execution, strings.Join(aliases, ","), maxRows), serialized, nil } // RenderSQL renders a statement node (as returned by ParseSQL) back to SQL diff --git a/internal/query/resource_inventory.go b/internal/query/resource_inventory.go new file mode 100644 index 000000000..36679d752 --- /dev/null +++ b/internal/query/resource_inventory.go @@ -0,0 +1,51 @@ +package query + +import ( + "strings" + + "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" +) + +// Every signal partitions the captured files independently. Whole cached files +// are exact only inside the logical window; boundary and unacknowledged files +// must supply actual observations, rather than overlapping resource envelopes. +func (d *Duck) resourceInventorySource(batches []telemetry.BatchMetadata, markers map[string]bool, window queryrows.Window) string { + var contributions []string + for _, signal := range []string{"spans", "logs", "metrics"} { + var interior []string + var raw []telemetry.BatchMetadata + for _, batch := range batches { + if !overlapping(batch, signal, window) { + continue + } + bounds, _ := batchTime(batch, signal) + if markers[batch.ID] && bounds.Known && bounds.MinNanos >= window.Start.UnixNano() && bounds.MaxNanos < window.End.UnixNano() { + interior = append(interior, batch.ID) + } else { + raw = append(raw, batch) + } + } + if len(interior) > 0 { + contributions = append(contributions, `SELECT signal,namespace,service,resource,first_seen,last_seen FROM read_resource_inventory WHERE signal=`+sqlLiteral(signal)+` AND batch_id IN (`+idsSQL(interior)+`)`) + } + if len(raw) > 0 { + clock := "time" + if signal == "spans" { + clock = "start_time" + } + contributions = append(contributions, `SELECT `+sqlLiteral(signal)+` AS signal,namespace,service,resource,min(`+clock+`) AS first_seen,max(`+clock+`) AS last_seen FROM (`+cleanSource(signal, d.snapshotSource(signal, raw, batches))+`) WHERE `+windowPredicate(clock, window)+` GROUP BY namespace,service,resource`) + } + } + if len(contributions) == 0 { + return emptyResourceInventory + } + scope := "" + if window.Namespace != "" { + scope += " AND namespace=" + sqlLiteral(window.Namespace) + } + if window.Service != "" { + scope += " AND service=" + sqlLiteral(window.Service) + } + return `SELECT signal,namespace,service,resource,min(first_seen) AS first_seen,max(last_seen) AS last_seen FROM (` + strings.Join(contributions, " UNION ALL ") + `) WHERE true` + scope + ` GROUP BY signal,namespace,service,resource` +} diff --git a/internal/query/resource_inventory_evidence_test.go b/internal/query/resource_inventory_evidence_test.go new file mode 100644 index 000000000..a64a70eba --- /dev/null +++ b/internal/query/resource_inventory_evidence_test.go @@ -0,0 +1,89 @@ +package query + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/queryrows" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" +) + +func TestResourceInventoryFrozenSnapshotPlans(t *testing.T) { + root := os.Getenv("FANOUT_TEST_INVENTORY_SNAPSHOT") + if root == "" { + t.Skip("set FANOUT_TEST_INVENTORY_SNAPSHOT to an extracted telemetry repository") + } + evidence := os.Getenv("FANOUT_TEST_INVENTORY_EVIDENCE") + if evidence == "" { + t.Fatal("FANOUT_TEST_INVENTORY_EVIDENCE is required") + } + data, err := os.ReadFile(filepath.Join(evidence, "panel-sql.json")) + if err != nil { + t.Fatal(err) + } + var statements map[string]string + if err := json.Unmarshal(data, &statements); err != nil { + t.Fatal(err) + } + repo, err := telemetrystore.Open(root) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + cfg := config.Config{DataDir: filepath.Join(evidence, "runtime"), DuckDBMemory: "8GB", DuckDBThreads: 8, DuckDBMaxConns: 4} + d, err := NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + defer d.Close() + w := queryrows.Window{Start: time.Date(2026, 10, 16, 15, 47, 42, 261874001, time.UTC), End: time.Date(2026, 10, 16, 16, 47, 42, 261874001, time.UTC)} + files := w + files.Start = files.Start.Add(-10 * time.Minute) + conn, err := d.DB.Conn(t.Context()) + if err != nil { + t.Fatal(err) + } + tx, err := beginRead(t.Context(), conn) + if err != nil { + t.Fatal(err) + } + defer endRead(conn, tx) + windowMacro := regexp.MustCompile(`\$__window\(([^)]+)\)`) + for name, statement := range statements { + statement = windowMacro.ReplaceAllStringFunc(statement, func(m string) string { return windowPredicate(windowMacro.FindStringSubmatch(m)[1], w) }) + statement = strings.NewReplacer("$__from", timeLiteral(w.Start), "$__to", timeLiteral(w.End)).Replace(statement) + bound, err := d.bindSnapshot(queryrows.WithInventoryWindow(t.Context(), w), tx, statement, files) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(evidence, name+"-bound.sql"), []byte(bound+"\n"), 0600); err != nil { + t.Fatal(err) + } + rows, err := tx.QueryContext(t.Context(), "EXPLAIN (FORMAT JSON) "+bound) + if err != nil { + t.Fatal(err) + } + var plan strings.Builder + for rows.Next() { + var key, value string + if err := rows.Scan(&key, &value); err != nil { + t.Fatal(err) + } + plan.WriteString(value + "\n") + } + err = rows.Err() + rows.Close() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(evidence, name+"-plan.json"), []byte(plan.String()), 0600); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/query/resource_inventory_test.go b/internal/query/resource_inventory_test.go new file mode 100644 index 000000000..0149a0146 --- /dev/null +++ b/internal/query/resource_inventory_test.go @@ -0,0 +1,758 @@ +package query + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/parquet-go/parquet-go" +) + +func inventoryContext(ctx context.Context, w queryrows.Window) context.Context { + return queryrows.WithInventoryWindow(queryrows.WithWindow(ctx, w), w) +} + +// Compare full typed identities, multiplicities and nanosecond extrema inside +// one pinned snapshot; JSON is deliberately not used as resource identity. +func assertInventoryOracle(t *testing.T, d *Duck, w queryrows.Window, want int) { + t.Helper() + var raw []string + for _, signal := range []string{"spans", "logs", "metrics"} { + clock := "time" + if signal == "spans" { + clock = "start_time" + } + raw = append(raw, `SELECT `+sqlLiteral(signal)+` AS signal,namespace,service,resource,min(`+clock+`) AS first_seen,max(`+clock+`) AS last_seen FROM `+signal+` WHERE `+windowPredicate(clock, w)+` GROUP BY namespace,service,resource`) + } + statement := `WITH expected AS (` + strings.Join(raw, " UNION ALL ") + `), differences AS ((SELECT * FROM resource_inventory EXCEPT ALL SELECT * FROM expected) UNION ALL (SELECT * FROM expected EXCEPT ALL SELECT * FROM resource_inventory)) SELECT (SELECT count(*) FROM differences),(SELECT count(*) FROM resource_inventory)` + var differences, count int + if err := d.QueryRowScan(inventoryContext(t.Context(), w), []any{&differences, &count}, statement); err != nil || differences != 0 || want >= 0 && count != want { + t.Fatalf("inventory/raw differences=%d rows=%d want=%d: %v", differences, count, want, err) + } +} + +func resourceBatch(id string, times []int64, resource map[string]any) telemetrystore.Batch { + b := telemetrystore.Batch{ID: id} + for i, n := range times { + // Deliberately different event/start/end/observed/ingest clocks. + b.Spans = append(b.Spans, telemetry.Span{Namespace: "prod", ServiceName: "api", TraceID: id, SpanID: fmt.Sprint(i), StartUnixNanos: n, EndUnixNanos: n + int64(time.Hour), IngestedAt: 1, Resource: resource}) + b.Logs = append(b.Logs, telemetry.Log{Namespace: "prod", ServiceName: "api", TimeUnixNanos: n, EventUnixNanos: n, ObservedTimeNanos: n + int64(time.Hour), IngestedAt: 1, Resource: resource}) + b.Metrics = append(b.Metrics, telemetry.Metric{Namespace: "prod", ServiceName: "api", TimeUnixNanos: n, EventUnixNanos: n, StartTimeUnixNanos: n - int64(time.Hour), IngestedAt: 1, Resource: resource, Type: "gauge"}) + } + return b +} + +func TestResourceInventoryExactWindowsAndTypedIdentity(t *testing.T) { + for _, at := range []int64{-100, 0, 1791370800000000001} { + t.Run(fmt.Sprint(at), func(t *testing.T) { + d, repo := newBatchCacheTest(t) + resources := []map[string]any{nil, {}, {"n": int64(9007199254740993), "b": false, "bytes": []byte{0, 255}, "nested": []any{int64(1), map[string]any{"x": true}}, "dotted.key": "literal"}, {"n": "9007199254740993"}, {"n": true}} + for i, r := range resources { + if err := commitRestoredFixture(repo, t.Context(), resourceBatch(fmt.Sprint("typed", i), []int64{at - 1, at, at + 9, at + 10}, r)); err != nil { + t.Fatal(err) + } + } + if err := repo.Commit(t.Context(), resourceBatch("equal", []int64{at + 2}, resources[2])); err != nil { + t.Fatal(err) + } + want := 15 + if at < 0 { + want = 5 + } + w := queryrows.Window{Start: time.Unix(0, at), End: time.Unix(0, at+10)} + assertInventoryOracle(t, d, w, want) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, want) + // Widening the window makes every cached contribution interior. + w.Start = time.Unix(0, at-2) + w.End = time.Unix(0, at+11) + assertInventoryOracle(t, d, w, want) + if at <= 0 { + return + } + conn, err := d.DB.Conn(t.Context()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + bound, err := d.bindSnapshot(inventoryContext(t.Context(), w), conn, `SELECT * FROM resource_inventory`, w) + if err != nil || strings.Contains(bound, "read_parquet") || strings.Contains(bound, "read_trace") { + t.Fatalf("interior source: %s %v", bound, err) + } + rows, err := conn.QueryContext(t.Context(), "EXPLAIN "+bound) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var key, plan string + if err := rows.Scan(&key, &plan); err != nil { + t.Fatal(err) + } + if strings.Contains(plan, "READ_PARQUET") { + t.Fatal(plan) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestResourceInventoryGapsClippedExtremaAndExpandedWindow(t *testing.T) { + d, repo := newBatchCacheTest(t) + at := time.Date(2026, 10, 1, 12, 0, 0, 1, time.UTC) + w := queryrows.Window{Start: at, End: at.Add(time.Minute)} + for _, b := range []telemetrystore.Batch{ + resourceBatch("gap", []int64{at.Add(-time.Minute).UnixNano(), w.End.UnixNano()}, map[string]any{"id": "gap"}), + resourceBatch("clipped", []int64{at.Add(-time.Minute).UnixNano(), at.Add(5 * time.Second).UnixNano(), at.Add(10 * time.Second).UnixNano(), w.End.UnixNano()}, map[string]any{"id": "clipped"}), + resourceBatch("lookback", []int64{at.Add(-time.Second).UnixNano()}, map[string]any{"id": "lookback"}), + } { + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + } + assertInventoryOracle(t, d, w, 3) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 3) + expanded := w + expanded.Start = at.Add(-10 * time.Minute) + ctx := queryrows.WithWindow(queryrows.WithInventoryWindow(t.Context(), w), expanded) + var first, last int64 + if err := d.QueryRowScan(ctx, []any{&first, &last}, `SELECT min(epoch_ns(first_seen)),max(epoch_ns(last_seen)) FROM resource_inventory`); err != nil || first != at.Add(5*time.Second).UnixNano() || last != at.Add(10*time.Second).UnixNano() { + t.Fatalf("expanded first/last=%d/%d: %v", first, last, err) + } +} + +func TestResourceInventorySignalSpecificUnknownBoundsAndPruning(t *testing.T) { + d, repo := newBatchCacheTest(t) + at := time.Unix(0, 100) + b := resourceBatch("mixed", []int64{110}, map[string]any{"host.name": "local"}) + b.Logs = append(b.Logs, telemetry.Log{TimeUnixNanos: 90, EventUnixNanos: 90, IngestedAt: 1}) + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + w := queryrows.Window{Start: at, End: time.Unix(0, 120)} + assertInventoryOracle(t, d, w, 3) + batches := repo.Parquet.BatchMetadata() + source := d.resourceInventorySource(batches, map[string]bool{"mixed": true}, w) + if strings.Contains(source, "spans.parquet") || !strings.Contains(source, "logs.parquet") || strings.Contains(source, "metrics.parquet") { + t.Fatal(source) + } + batches[0].SpanTime = telemetry.TimeRange{} + source = d.resourceInventorySource(batches, map[string]bool{"mixed": true}, w) + if !strings.Contains(source, "spans.parquet") { + t.Fatal("unknown bounds omitted raw signal") + } + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM (` + source + `)`).Scan(&count); err != nil || count != 3 { + t.Fatalf("unknown rows=%d %v", count, err) + } + // A span-only lookup must prune log and metric raw/cache branches. + conn, err := d.DB.Conn(t.Context()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + bound, err := d.bindSnapshot(inventoryContext(t.Context(), w), conn, `SELECT resource['host.name']::VARCHAR FROM resource_inventory WHERE signal='spans'`, w) + if err != nil { + t.Fatal(err) + } + rows, err := conn.QueryContext(t.Context(), "EXPLAIN "+bound) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var key, plan string + if err := rows.Scan(&key, &plan); err != nil { + t.Fatal(err) + } + if strings.Contains(plan, "READ_PARQUET") || strings.Contains(plan, "read_trace") { + t.Fatal(plan) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } +} + +func TestResourceInventoryLatePublicationAndReadTransaction(t *testing.T) { + d, repo := newBatchCacheTest(t) + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 100)} + if err := repo.Commit(t.Context(), resourceBatch("initial", []int64{10}, nil)); err != nil { + t.Fatal(err) + } + conn, err := d.DB.Conn(t.Context()) + if err != nil { + t.Fatal(err) + } + tx, err := beginRead(t.Context(), conn) + if err != nil { + t.Fatal(err) + } + defer endRead(conn, tx) + // Fix the MVCC snapshot before the writer acknowledges the file. + if _, err := batchMarkers(t.Context(), tx); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + bound, err := d.bindSnapshot(inventoryContext(t.Context(), w), tx, `SELECT count(*) FROM resource_inventory`, w) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(bound, "read_parquet") { + t.Fatal("reader used a marker committed after its snapshot") + } + var count int + if err := tx.QueryRow(bound).Scan(&count); err != nil || count != 3 { + t.Fatalf("pinned reader rows=%d %v", count, err) + } + for _, b := range []telemetrystore.Batch{{ID: "late-log", Logs: resourceBatch("", []int64{5}, map[string]any{"late": true}).Logs}, {ID: "late-metric", Metrics: resourceBatch("", []int64{6}, map[string]any{"late": true}).Metrics}} { + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + } + assertInventoryOracle(t, d, w, 5) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 5) + // Already captured membership remains unchanged after late publication. + if err := tx.QueryRow(bound).Scan(&count); err != nil || count != 3 { + t.Fatalf("captured reader changed=%d %v", count, err) + } +} + +func TestResourceInventoryAtomicSignalAndMarkerFailures(t *testing.T) { + for _, failure := range []string{"spans", "logs", "metrics", "marker"} { + t.Run(failure, func(t *testing.T) { + d, repo := newBatchCacheTest(t) + if err := repo.Commit(t.Context(), resourceBatch("atomic", []int64{10}, nil)); err != nil { + t.Fatal(err) + } + constraint := `CHECK (signal<>` + sqlLiteral(failure) + `)` + if failure == "marker" { + if _, err := d.DB.Exec(`DROP TABLE read_batches; CREATE TABLE read_batches(batch_id VARCHAR PRIMARY KEY CHECK(batch_id<>'atomic'),replacement_id VARCHAR)`); err != nil { + t.Fatal(err) + } + } else { + if _, err := d.DB.Exec(`DROP TABLE read_resource_inventory; CREATE TABLE read_resource_inventory(batch_id VARCHAR,signal VARCHAR ` + constraint + `,namespace VARCHAR,service VARCHAR,resource VARIANT,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS)`); err != nil { + t.Fatal(err) + } + } + if _, err := d.RefreshReadCaches(t.Context()); err == nil { + t.Fatal("injected failure succeeded") + } + for _, table := range []string{"read_batches", "read_resource_inventory", "read_trace_parts", "read_trace_candidates"} { + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM ` + table).Scan(&count); err != nil || count != 0 { + t.Fatalf("partial %s=%d %v", table, count, err) + } + } + if _, err := d.DB.Exec(`DROP TABLE read_batches; DROP TABLE read_resource_inventory`); err != nil { + t.Fatal(err) + } + if err := createBatchCaches(d.DB); err != nil { + t.Fatal(err) + } + for range 2 { + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + assertInventoryOracle(t, d, queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 20)}, 3) + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&count); err != nil || count != 3 { + t.Fatalf("retry inventory=%d %v", count, err) + } + }) + } +} + +func TestResourceInventoryFailedAndIncompleteReplacement(t *testing.T) { + d, repo := newBatchCacheTest(t) + if err := repo.Commit(t.Context(), resourceBatch("input", []int64{10}, nil)); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if err := d.transferBatchCache(t.Context(), "incomplete", []string{"input", "missing"}); err != nil { + t.Fatal(err) + } + markers, err := batchMarkers(t.Context(), d.DB) + if err != nil || markers["incomplete"] { + t.Fatalf("incomplete acknowledged: %v %v", markers, err) + } + failed := errors.New("publication failed") + if err := d.PublishParquetReplacement(t.Context(), telemetry.BatchMetadata{ID: "prepared"}, []string{"input"}, func(context.Context) error { return failed }); !errors.Is(err, failed) { + t.Fatal(err) + } + assertInventoryOracle(t, d, queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 20)}, 3) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory WHERE batch_id IN ('incomplete','prepared')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("inactive rows=%d %v", count, err) + } +} + +func TestResourceInventoryCompactionRetentionAndNamespaces(t *testing.T) { + for _, complete := range []bool{false, true} { + t.Run(fmt.Sprint(complete), func(t *testing.T) { + d, repo := newBatchCacheTest(t) + for i := range 8 { + b := resourceBatch(fmt.Sprint("part", i), []int64{int64(10 + i)}, map[string]any{"shared": true}) + if i%2 == 0 { + b.Spans[0].Namespace = "other" + b.Logs[0].Namespace = "other" + b.Metrics[0].Namespace = "other" + } + if err := commitRestoredFixture(repo, t.Context(), b); err != nil { + t.Fatal(err) + } + } + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 100)} + if complete { + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + assertInventoryOracle(t, d, w, 6) + if n, err := repo.CompactParquet(t.Context(), d, 8); err != nil || n != 8 { + t.Fatalf("compaction=%d %v", n, err) + } + assertInventoryOracle(t, d, w, 6) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 6) + var rows int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&rows); err != nil || rows != 6 { + t.Fatalf("merged inventory=%d %v", rows, err) + } + if n, err := repo.PruneParquet(t.Context(), d, 2, 8); err != nil || n != 1 { + t.Fatalf("retention=%d %v", n, err) + } + assertInventoryOracle(t, d, w, 0) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&rows); err != nil || rows != 0 { + t.Fatalf("retired inventory=%d %v", rows, err) + } + }) + } +} + +func TestResourceInventoryRetentionChangesExtremaBeforeCleanup(t *testing.T) { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprint(reverse), func(t *testing.T) { + d, repo := newBatchCacheTest(t) + for i := range 3 { + event := int64(10 + i) + if reverse { + event = int64(12 - i) + } + b := resourceBatch(fmt.Sprint("retain", i), []int64{event}, map[string]any{"shared": true}) + b.Spans[0].IngestedAt = int64(i + 1) + b.Logs[0].IngestedAt = int64(i + 1) + b.Metrics[0].IngestedAt = int64(i + 1) + if err := commitRestoredFixture(repo, t.Context(), b); err != nil { + t.Fatal(err) + } + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 100)} + assertInventoryOracle(t, d, w, 3) + if n, err := repo.PruneParquet(t.Context(), d, 2, 8); err != nil || n != 1 { + t.Fatalf("retention=%d %v", n, err) + } + assertInventoryOracle(t, d, w, 3) + var first, last int64 + wantFirst, wantLast := int64(11), int64(12) + if reverse { + wantFirst, wantLast = 10, 11 + } + if err := d.QueryRowScan(inventoryContext(t.Context(), w), []any{&first, &last}, `SELECT min(epoch_ns(first_seen)),max(epoch_ns(last_seen)) FROM resource_inventory`); err != nil || first != wantFirst || last != wantLast { + t.Fatalf("retained first/last=%d/%d %v", first, last, err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 3) + + }) + } +} + +func TestResourceInventoryLogAndMetricOversizedProgress(t *testing.T) { + for _, signal := range []string{"logs", "metrics"} { + t.Run(signal, func(t *testing.T) { + d, repo := newBatchCacheTest(t) + b := telemetrystore.Batch{ID: "oversized"} + if signal == "logs" { + b.Logs = make([]telemetry.Log, readCacheRowBudget+1) + for i := range b.Logs { + b.Logs[i] = telemetry.Log{TimeUnixNanos: 1} + } + } else { + b.Metrics = make([]telemetry.Metric, readCacheRowBudget+1) + for i := range b.Metrics { + b.Metrics[i] = telemetry.Metric{TimeUnixNanos: 1} + } + } + if err := repo.Commit(t.Context(), b); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + markers, err := batchMarkers(t.Context(), d.DB) + if err != nil || !markers[b.ID] { + t.Fatalf("no oversized progress: %v %v", markers, err) + } + var rows int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&rows); err != nil || rows != 1 { + t.Fatalf("inventory=%d %v", rows, err) + } + }) + } +} + +func TestResourceInventoryRequiresLogicalScopeAndReleasesCancelledReaders(t *testing.T) { + d, repo := newBatchCacheTest(t) + if err := repo.Commit(t.Context(), resourceBatch("boundary", []int64{1, 10, 20}, nil)); err != nil { + t.Fatal(err) + } + statement := `SELECT count(*) FROM resource_inventory` + w := queryrows.Window{Start: time.Unix(0, 5), End: time.Unix(0, 15)} + for _, ctx := range []context.Context{t.Context(), queryrows.WithWindow(t.Context(), w)} { + if rows, err := d.QueryContext(ctx, statement); err == nil { + rows.Close() + t.Fatal("missing logical scope accepted") + } + } + ctx, cancel := context.WithTimeout(inventoryContext(t.Context(), w), 20*time.Millisecond) + defer cancel() + rows, err := d.QueryContext(ctx, `SELECT sum(i) FROM resource_inventory CROSS JOIN range(1000000000000) t(i)`) + if err == nil { + for rows.Next() { + var v any + _ = rows.Scan(&v) + } + err = rows.Err() + rows.Close() + } + if err == nil { + t.Fatal("long inventory query was not interrupted") + } + assertInventoryOracle(t, d, w, 3) + ctx, stop := context.WithTimeout(t.Context(), time.Second) + defer stop() + if err := d.PublishParquet(ctx, func(context.Context) error { return nil }); err != nil { + t.Fatalf("cancelled reader leaked publication lease: %v", err) + } +} + +func TestResourceInventoryRestartDiscardsInactivePreparation(t *testing.T) { + d, repo := newBatchCacheTest(t) + if err := repo.Commit(t.Context(), resourceBatch("active", []int64{10}, nil)); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if err := d.transferBatchCache(t.Context(), "prepared", []string{"active"}); err != nil { + t.Fatal(err) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } + reopened, err := NewDuck(t.Context(), d.cfg, repo) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 20)} + assertInventoryOracle(t, reopened, w, 3) + if _, err := reopened.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + var count int + if err := reopened.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory`).Scan(&count); err != nil || count != 3 { + t.Fatalf("restarted rows=%d %v", count, err) + } +} + +func TestResourceInventoryTransferFailureLeavesOutputUnacknowledged(t *testing.T) { + d, repo := newBatchCacheTest(t) + if err := repo.Commit(t.Context(), resourceBatch("input", []int64{10}, nil)); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := d.DB.Exec(`CREATE TABLE inventory_backup AS SELECT * FROM read_resource_inventory; DROP TABLE read_resource_inventory; CREATE TABLE read_resource_inventory(batch_id VARCHAR CHECK(batch_id<>'output'),signal VARCHAR,namespace VARCHAR,service VARCHAR,resource VARIANT,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS); INSERT INTO read_resource_inventory SELECT * FROM inventory_backup; DROP TABLE inventory_backup`); err != nil { + t.Fatal(err) + } + if err := d.transferBatchCache(t.Context(), "output", []string{"input"}); err == nil { + t.Fatal("injected transfer succeeded") + } + markers, err := batchMarkers(t.Context(), d.DB) + if err != nil || markers["output"] { + t.Fatalf("failed transfer acknowledged: %v %v", markers, err) + } + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_resource_inventory WHERE batch_id='output'`).Scan(&count); err != nil || count != 0 { + t.Fatalf("partial output=%d %v", count, err) + } + assertInventoryOracle(t, d, queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 20)}, 3) +} + +func TestResourceInventoryTransferFailureKeepsPublishedOutputRaw(t *testing.T) { + d, repo := newBatchCacheTest(t) + for i := range 8 { + if err := commitRestoredFixture(repo, t.Context(), resourceBatch(fmt.Sprint("part", i), []int64{int64(10 + i)}, nil)); err != nil { + t.Fatal(err) + } + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := d.DB.Exec(`CREATE TABLE inventory_backup AS SELECT * FROM read_resource_inventory; DROP TABLE read_resource_inventory; CREATE TABLE read_resource_inventory(batch_id VARCHAR CHECK(NOT starts_with(batch_id,'compact')),signal VARCHAR,namespace VARCHAR,service VARCHAR,resource VARIANT,first_seen TIMESTAMPTZ_NS,last_seen TIMESTAMPTZ_NS); INSERT INTO read_resource_inventory SELECT * FROM inventory_backup; DROP TABLE inventory_backup`); err != nil { + t.Fatal(err) + } + if n, err := repo.CompactParquet(t.Context(), d, 8); err != nil || n != 8 { + t.Fatalf("publication=%d %v", n, err) + } + batches := repo.Parquet.BatchMetadata() + if len(batches) != 1 { + t.Fatalf("active files=%d", len(batches)) + } + markers, err := batchMarkers(t.Context(), d.DB) + if err != nil || markers[batches[0].ID] { + t.Fatalf("failed output acknowledged: %v %v", markers, err) + } + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 100)} + assertInventoryOracle(t, d, w, 3) + if !strings.Contains(d.resourceInventorySource(batches, markers, w), "read_parquet") { + t.Fatal("unacknowledged output did not use raw files") + } + if _, err := d.DB.Exec(`DROP TABLE read_resource_inventory`); err != nil { + t.Fatal(err) + } + if err := createBatchCaches(d.DB); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 3) +} + +func TestResourceInventorySignedPhysicalEventClocks(t *testing.T) { + // Canonical OTLP admission substitutes absent/nonpositive log/metric clocks. + // Rewrite only physical event clocks under the exact production schema so + // the reader's signed timestamp contract is exercised for every signal. + for _, at := range []int64{-100, 0} { + t.Run(fmt.Sprint(at), func(t *testing.T) { + d, repo := newBatchCacheTest(t) + b := resourceBatch("signed", []int64{999, 1000, 1009, 1010}, map[string]any{"typed": int64(7)}) + if err := commitRestoredFixture(repo, t.Context(), b); err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + path := filepath.Join(repo.Parquet.BatchesDir(), "signed.batch", signal+".parquet") + signedInventoryFixture(t, path, at-1000) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } + if err := repo.Close(); err != nil { + t.Fatal(err) + } + reopened, err := telemetrystore.Open(d.cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + d, err = NewDuck(t.Context(), d.cfg, reopened) + if err != nil { + t.Fatal(err) + } + defer d.Close() + w := queryrows.Window{Start: time.Unix(0, at), End: time.Unix(0, at+10)} + assertInventoryOracle(t, d, w, 3) + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + assertInventoryOracle(t, d, w, 3) + var first, last int64 + if err := d.QueryRowScan(inventoryContext(t.Context(), w), []any{&first, &last}, `SELECT min(epoch_ns(first_seen)),max(epoch_ns(last_seen)) FROM resource_inventory`); err != nil || first != at || last != at+9 { + t.Fatalf("signed extrema=%d/%d %v", first, last, err) + } + w.Start = time.Unix(0, at-2) + w.End = time.Unix(0, at+11) + assertInventoryOracle(t, d, w, 3) + }) + } +} + +func signedInventoryFixture(t *testing.T, path string, offset int64) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + t.Fatal(err) + } + file, err := parquet.OpenFile(f, stat.Size()) + if err != nil { + t.Fatal(err) + } + out, err := os.Create(path + ".signed") + if err != nil { + t.Fatal(err) + } + defer out.Close() + writer := parquet.NewGenericWriter[any](out, file.Schema()) + columns := file.Schema().Columns() + for _, group := range file.RowGroups() { + reader := group.Rows() + for { + rows := make([]parquet.Row, 1) + n, err := reader.ReadRows(rows) + if err != nil && err != io.EOF { + t.Fatal(err) + } + if n == 0 { + break + } + row := rows[0].Clone() + for i, value := range row { + name := columns[value.Column()][0] + if name == "start_time" || name == "start_unix_nano" || name == "log_time" || name == "metric_time" || name == "time_unix_nano" { + row[i] = parquet.Int64Value(value.Int64()+offset).Level(value.RepetitionLevel(), value.DefinitionLevel(), value.Column()) + } + } + if _, err := writer.WriteRows([]parquet.Row{row}); err != nil { + t.Fatal(err) + } + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + if err := os.Rename(path+".signed", path); err != nil { + t.Fatal(err) + } +} + +func TestResourceInventoryPanelSQLBoundary(t *testing.T) { + d, _ := newBatchCacheTest(t) + for _, relation := range []string{"resource_inventory", "main.resource_inventory"} { + statement := "SELECT signal,namespace,service,resource,first_seen,last_seen FROM " + relation + if _, _, err := d.PrepareTelemetrySQL(t.Context(), statement, statement, 10); err != nil { + t.Fatalf("panel relation %s: %v", relation, err) + } + if result := d.ExecuteSQL(t.Context(), SQLRequest{Query: statement}); result.Error == "" { + t.Fatalf("unscoped inventory accepted: %+v", result) + } + } + for _, statement := range []string{ + "SELECT * FROM telemetry.resource_inventory", "SELECT * FROM memory.main.resource_inventory", + "SELECT * FROM read_resource_inventory", "SELECT * FROM read_batches", + "SELECT * FROM read_parquet('private.parquet')", "SELECT * FROM query_table('read_resource_inventory')", + "SELECT * FROM duckdb_tables()", "SELECT 1; SELECT 2", "DELETE FROM resource_inventory", + } { + if _, _, err := d.PrepareTelemetrySQL(t.Context(), statement, statement, 10); err == nil { + t.Fatalf("unsafe panel SQL accepted: %s", statement) + } + } +} + +func TestResourceInventoryProjectionPreservesTypesAndQualifiedBindings(t *testing.T) { + d, repo := newBatchCacheTest(t) + resource := map[string]any{"n": int64(9007199254740993), "b": false, "bytes": []byte{0, 255}, "nested": []any{int64(1), map[string]any{"x": true}}, "dotted.key": "literal"} + if err := repo.Commit(t.Context(), resourceBatch("types", []int64{10}, resource)); err != nil { + t.Fatal(err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + w := queryrows.Window{Start: time.Unix(0, 1), End: time.Unix(0, 20)} + statement := `SELECT main.resource_inventory.resource,resource['n']::BIGINT AS n,resource['b']::BOOLEAN AS b,resource['dotted.key']::VARCHAR AS dotted,epoch_ns(main.resource_inventory.first_seen) AS first FROM main.resource_inventory WHERE signal='spans'` + prepared, serialized, err := d.PrepareTelemetrySQL(t.Context(), statement, statement, 10) + if err != nil || len(serialized) != 5 || !serialized[0] { + t.Fatalf("projection=%v %v", serialized, err) + } + rows, err := d.QueryContext(inventoryContext(t.Context(), w), prepared) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var payload, dotted string + var n, first int64 + var b bool + if !rows.Next() { + t.Fatal(rows.Err()) + } + if err := rows.Scan(&payload, &n, &b, &dotted, &first); err != nil || n != 9007199254740993 || b || dotted != "literal" || first != 10 || !strings.Contains(payload, `"n":9007199254740993`) || !strings.Contains(payload, `"nested":[1,{"x":true}]`) { + t.Fatalf("typed row=%s %d %v %s %d: %v", payload, n, b, dotted, first, err) + } + if rows.Next() || rows.Err() != nil { + t.Fatalf("unexpected extra row/error: %v", rows.Err()) + } +} + +func TestResourceInventoryTransferPreservesNullAndEmptyGroups(t *testing.T) { + d, _ := newBatchCacheTest(t) + for _, statement := range []string{ + `INSERT INTO read_batches(batch_id) VALUES ('a'),('b')`, + `INSERT INTO read_resource_inventory SELECT id,'spans',namespace,service,resource,make_timestamp_ns(1)::TIMESTAMPTZ_NS,make_timestamp_ns(2)::TIMESTAMPTZ_NS FROM (VALUES ('a'),('b')) b(id) CROSS JOIN (VALUES (NULL::VARCHAR,NULL::VARCHAR,NULL::VARIANT),('','',NULL::VARIANT),('','',map()::VARIANT)) r(namespace,service,resource)`, + } { + if _, err := d.DB.Exec(statement); err != nil { + t.Fatal(err) + } + } + if err := d.transferBatchCache(t.Context(), "out", []string{"a", "b"}); err != nil { + t.Fatal(err) + } + var total, nulls, empty int + if err := d.DB.QueryRow(`SELECT count(*),count(*) FILTER (WHERE namespace IS NULL AND service IS NULL AND resource IS NULL),count(*) FILTER (WHERE namespace='' AND service='' AND resource IS NOT NULL) FROM read_resource_inventory WHERE batch_id='out'`).Scan(&total, &nulls, &empty); err != nil || total != 3 || nulls != 1 || empty != 1 { + t.Fatalf("null/empty groups=%d/%d/%d %v", total, nulls, empty, err) + } +} diff --git a/internal/query/sql_boundary.go b/internal/query/sql_boundary.go index bd07d65b6..27a7e8e7e 100644 --- a/internal/query/sql_boundary.go +++ b/internal/query/sql_boundary.go @@ -46,7 +46,7 @@ func (d *Duck) querySQL(ctx context.Context, query string, maxRows int, explain release() } }() - if err := validateSQLAST(ctx, conn, query); err != nil { + if err := validateSQLAST(ctx, conn, query, false); err != nil { return nil, nil, err } query = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";")) @@ -101,12 +101,12 @@ func parseStatement(ctx context.Context, db rowQueryer, query string) (map[strin return parsed.Statements[0].Node, nil } -func validateSQLAST(ctx context.Context, conn *sql.Conn, query string) error { +func validateSQLAST(ctx context.Context, conn *sql.Conn, query string, panel bool) error { node, err := parseStatement(ctx, conn, query) if err != nil { return err } - return validateSQLNode(node, nil) + return validateSQLNode(node, nil, panel) } // describeProjection describes a query and returns, per result column, the @@ -182,7 +182,7 @@ func containsLogicalType(expression, name string) bool { return strings.EqualFold(token.String(), name) } -func validateSQLNode(value any, ctes map[string]bool) error { +func validateSQLNode(value any, ctes map[string]bool, panel bool) error { switch node := value.(type) { case map[string]any: // Definitions see prior and outer CTEs. A nonrecursive definition cannot @@ -203,7 +203,7 @@ func validateSQLNode(value any, ctes map[string]bool) error { if !ok || !defined || definition["query_node"] == nil { return fmt.Errorf("invalid CTE definition") } - if err := validateSQLNode(definition, scope); err != nil { + if err := validateSQLNode(definition, scope, panel); err != nil { return err } scope[strings.ToLower(name)] = true @@ -230,6 +230,12 @@ func validateSQLNode(value any, ctes map[string]bool) error { schema, _ := node["schema_name"].(string) catalog, _ := node["catalog_name"].(string) allowed := map[string]bool{"spans": true, "logs": true, "metrics": true, "service_rollup": true, "edge_rollup": true} + if strings.EqualFold(name, "resource_inventory") { + if !panel || (schema != "" && schema != "main") { + return fmt.Errorf("resource_inventory is available only to panel SQL in its logical window") + } + allowed["resource_inventory"] = true + } cte := catalog == "" && ((schema == "" && ctes[strings.ToLower(name)]) || (schema == "recurring" && ctes["recurring:"+strings.ToLower(name)])) if !cte && (catalog != "" || (schema != "" && schema != "main" && schema != "telemetry") || !allowed[strings.ToLower(name)]) { @@ -261,13 +267,13 @@ func validateSQLNode(value any, ctes map[string]bool) error { scope[strings.ToLower(name)] = true scope["recurring:"+strings.ToLower(name)] = true } - if err := validateSQLNode(child, scope); err != nil { + if err := validateSQLNode(child, scope, panel); err != nil { return err } } case []any: for _, child := range node { - if err := validateSQLNode(child, ctes); err != nil { + if err := validateSQLNode(child, ctes, panel); err != nil { return err } } diff --git a/internal/query/sql_boundary_test.go b/internal/query/sql_boundary_test.go index 274619e1f..9eca7e6e7 100644 --- a/internal/query/sql_boundary_test.go +++ b/internal/query/sql_boundary_test.go @@ -39,7 +39,7 @@ func TestSQLBoundaryRejectsHiddenReadsAndMutations(t *testing.T) { "SELECT getvariable('operator_secret')", "WITH x AS MATERIALIZED (COPY (SELECT 1) TO '" + output + "') SELECT * FROM x", } { - if err := validateSQLAST(ctx, conn, query); err == nil { + if err := validateSQLAST(ctx, conn, query, false); err == nil { t.Errorf("accepted %s", query) } } @@ -50,7 +50,7 @@ func TestSQLBoundaryRejectsHiddenReadsAndMutations(t *testing.T) { "WITH x AS (SELECT 1 AS n) SELECT * FROM x", "WITH RECURSIVE x(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM x WHERE n<3) SELECT * FROM x", } { - if err := validateSQLAST(ctx, conn, query); err != nil { + if err := validateSQLAST(ctx, conn, query, false); err != nil { t.Errorf("rejected %s: %v", query, err) } } diff --git a/internal/query/views.go b/internal/query/views.go index b6a624f0d..0084b4384 100644 --- a/internal/query/views.go +++ b/internal/query/views.go @@ -149,6 +149,9 @@ const macroAttr = ` CREATE OR REPLACE MACRO attr(attributes, key) AS variant_extract(attributes, key);` +// A binding descriptor for panel validation, never the execution source. +const emptyResourceInventory = `SELECT NULL::VARCHAR AS signal,NULL::VARCHAR AS namespace,NULL::VARCHAR AS service,NULL::VARIANT AS resource,NULL::TIMESTAMPTZ_NS AS first_seen,NULL::TIMESTAMPTZ_NS AS last_seen WHERE false` + // CreateCacheTables creates only DuckDB's rebuildable query accelerators. The // production telemetry rows themselves live in immutable Parquet batches. func CreateCacheTables(db *sql.DB) error { @@ -200,7 +203,7 @@ func CreateParquetViews(db *sql.DB, parquetDir string) error { // CreateViews creates stable clean-name views plus the attr() macro. func CreateViews(db *sql.DB) error { - for _, stmt := range []string{macroAttr, viewSpans, viewLogs, viewMetrics} { + for _, stmt := range []string{macroAttr, viewSpans, viewLogs, viewMetrics, `CREATE OR REPLACE VIEW resource_inventory AS ` + emptyResourceInventory} { if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("create view/macro: %w", err) } diff --git a/internal/query/views_test.go b/internal/query/views_test.go index e1b1500ff..ceeb3e6de 100644 --- a/internal/query/views_test.go +++ b/internal/query/views_test.go @@ -56,7 +56,7 @@ func TestCreateTables_CacheTablesIncludePartitionColumns(t *testing.T) { rows, err := db.Query(` SELECT table_name, column_name FROM duckdb_columns() -WHERE table_name IN ('service_rollup', 'edge_rollup', 'read_trace_parts', 'rollup_state') +WHERE table_name IN ('service_rollup', 'edge_rollup', 'read_trace_parts', 'read_resource_inventory', 'rollup_state') ORDER BY table_name, column_name`) if err != nil { t.Fatalf("query duckdb_columns failed: %v", err) @@ -79,10 +79,11 @@ ORDER BY table_name, column_name`) } required := map[string][]string{ - "service_rollup": {"namespace", "bucket", "service"}, - "edge_rollup": {"namespace", "bucket", "caller", "callee", "edge_type"}, - "read_trace_parts": {"batch_id", "namespace", "service", "trace_id", "min_start", "max_start", "max_end", "has_error"}, - "rollup_state": {"cache_key", "last_ingested_unix_nano", "updated_at"}, + "service_rollup": {"namespace", "bucket", "service"}, + "edge_rollup": {"namespace", "bucket", "caller", "callee", "edge_type"}, + "read_trace_parts": {"batch_id", "namespace", "service", "trace_id", "min_start", "max_start", "max_end", "has_error"}, + "read_resource_inventory": {"batch_id", "signal", "namespace", "service", "resource", "first_seen", "last_seen"}, + "rollup_state": {"cache_key", "last_ingested_unix_nano", "updated_at"}, } for table, columns := range required { for _, column := range columns { diff --git a/internal/queryrows/window.go b/internal/queryrows/window.go index 0e5669068..e2805407c 100644 --- a/internal/queryrows/window.go +++ b/internal/queryrows/window.go @@ -14,6 +14,18 @@ type Window struct { } type windowKey struct{} +type inventoryWindowKey struct{} + +// WithInventoryWindow records the logical panel interval before file lookback +// expansion. It is independent of the raw-file read window and read kind. +func WithInventoryWindow(ctx context.Context, window Window) context.Context { + return context.WithValue(ctx, inventoryWindowKey{}, window) +} + +func ReadInventoryWindow(ctx context.Context) (Window, bool) { + window, ok := ctx.Value(inventoryWindowKey{}).(Window) + return window, ok && window.Start.Before(window.End) +} func WithWindow(ctx context.Context, window Window) context.Context { return context.WithValue(ctx, windowKey{}, window) diff --git a/site/src/content/docs/guides/build-dashboards.mdx b/site/src/content/docs/guides/build-dashboards.mdx index 31021e581..c8a2f9592 100644 --- a/site/src/content/docs/guides/build-dashboards.mdx +++ b/site/src/content/docs/guides/build-dashboards.mdx @@ -90,10 +90,62 @@ the cause of a change. For queries that need it, a panel can use guarded SQL: one read-only `SELECT` over approved telemetry relations, with a required time-window filter. Logs, -log patterns, traces, scatter and state timelines use structured queries. +log patterns, traces and scatter use structured queries. SQL state timelines +return timestamp `time`, a string item, one numeric measure and optional string +`state`, with thresholds. SQL is an authoring escape hatch. **Spec** is a read-only inspection view; there is no SQL or chart-style editor there. +### Discover resources observed in the window + +`get_telemetry_schema` and `/api/telemetry/schema` list SQL-only relations in +`sql_relations`, separately from the three telemetry signals. In panel SQL, +`resource_inventory` returns one row per distinct full resource, signal, +namespace and service observed inside the selected half-open window +`[start,end)`. It exposes `signal`, `namespace`, `service`, typed `resource`, +`first_seen` and `last_seen`. The timestamps are actual event observations +inside that window, including when the resource also existed earlier. + +```sql +SELECT namespace, service, resource['service.version']::VARCHAR AS version, + first_seen, last_seen +FROM resource_inventory +WHERE signal = 'spans' AND $__window(first_seen) +ORDER BY namespace, service, version, first_seen, last_seen +``` + +Select the required signal (`spans`, `logs` or `metrics`). Dotted resource keys +are literal keys. Full resources with different runtime or SDK attributes +remain distinct even if their displayed version matches. `count(*)` counts +resource identities per signal, not events or instances. This relation is +scoped to the logical panel window, independently of metric lookbacks; it is +not a historical registry. Structured `query.from`, variables and unscoped +arbitrary SQL cannot read it. The [SQL relation reference](/reference/mcp-tools#panel-sql-relations) +lists column types and semantics. + +### Update existing System overview panels deliberately + +New defaults use `resource_inventory` for **What changed?** (`versions`) and +the service-host input to **Is it reachable?** (`check_states`). Existing +saved dashboards keep their SQL. Deploy the supporting binary before editing +those saved definitions. + +1. Use `list_dashboards` and `get_dashboard` to select the owner's dashboard by + ID and capture its current `version` and spec. Do not identify a board by + title alone. Compare both panel SQL fields with the known shipped definitions + being upgraded; skip and report customized or missing definitions. +2. Build replacements from the new default SQL, changing only those SQL fields. + Run `preview_panels` with the saved time and variables and inspect the complete + frames, including first-seen times and reachability warnings. +3. Call `edit_dashboard` with the captured `base_version` and two + `update_panel` operations, each with its panel `id` and `set: {sql: ...}`. + Preserve every other field, layout and variable. If the base version is + stale, reread and review the diff before retrying. +4. Retain the prior SQL and version history. Before rolling back to a binary + without this relation, restore both prior SQL fields through reviewed edits + at the then-current base version. The disposable cache rebuilds independently; + the telemetry format remains unchanged. + ## Choose metric measures by kind `get_telemetry_schema` lists each metric's name, unit, semantic `type` and diff --git a/site/src/content/docs/reference/mcp-tools.mdx b/site/src/content/docs/reference/mcp-tools.mdx index 74e2b91a3..dd6440de8 100644 --- a/site/src/content/docs/reference/mcp-tools.mdx +++ b/site/src/content/docs/reference/mcp-tools.mdx @@ -61,7 +61,7 @@ AG-UI `mcp-app` activity content uses exactly `resource_uri`, `tool_name`, ## create_dashboard -**Create dashboard** — Create a dashboard for the authenticated user from a complete spec. Read get_telemetry_schema first and preview_panels until every panel is ok or deliberately empty. The result lists any panel that is still empty or failing. Successful saves return a receipt for that committed version: base_version, version, changes, layout_changed, dashboard_fields and save_check. Changes describe authored panel fields and position_changed; physical packing is only layout_changed. Order chips mark the minimal moved set. save_check reports checked, reason when unchecked, total elapsed_ms and one status/rows result per saved panel with elapsed_ms when executed. checked means every panel was executed, not that error or empty panels are healthy. Text rows identify nonempty content, not telemetry. Use the structured receipt rather than prose warnings as check evidence. Create receipts list added panels without layout or dashboard edit chips. No-op saves still create a new version with no change chips. Per-panel elapsed_ms is omitted when the panel did not run. A dashboard is \{name, description, time:\{range}, variables, panels}. Each panel: \{id (lowercase), title, viz, width 1-12, query or sql}. viz: stat or gauge (one measure; gauge needs min and max), timeseries (bucket auto, at most one by), bar (one or two by), table (up to three by), text (content in Markdown), heatmap (bucket auto, no by), histogram (at most one by), scatter (exactly two measures in x then y order, one item dimension and an optional colour dimension), state_timeline (one measure, one item dimension, bucket auto and thresholds required). Rollup panel types are service_map (dependency graph) and health (overview metrics, error trend and health distribution). They require query.from=spans, no measures, by, bucket or sort, and only namespace/service equality filters with a literal or single-value variable; explicit casts and values with surrounding whitespace are rejected. Reads default to 400 services/edges and are capped at 400 even when query.limit is larger. Service scope applies before the limit; map nodes carry p95_ms and edges carry average_ms; error_rate is percent. Row panel types are logs, log_patterns and traces. Logs and traces take no measures, by or bucket. Patterns use count() by body_template. Patterns group only by body_template. Severity is the most frequent severity, ties choose the higher severity. Service is the most frequent service, ties choose ascending service name. Both describe the scoped, redacted events, not extra grouping dimensions. Each pattern returns dominant severity, top service, count and a dense trend, computed in the same bounded query after redaction and scoped filters. Redaction applies before filtering, grouping, display and drill-down. Row panels cannot use SQL. "erroring" means slowest erroring traces: filter to erroring traces, then rank by root duration. Trace candidate ranking is approximate when filters exclude the root span, as with exemplars. Categorical charts use fixed-order colour slots. Categorical options.top defaults to 6, maximum 6. Structured panels: the server computes Other; series are chosen worst-first by confidence (Wilson lower bound for error rates; at least 20 samples for latency); the rest fold into Other (N). Count, rate, sum and share rank largest first. Error rates require at least 20 requests for eligible Wilson ranking (z=1.96). Percentiles, quantile, avg, max and min require at least 20 non-null numeric samples for eligible value ranking and use better (explicit or inferred). Ineligible series follow eligible ones by sample count descending. Ties use sample count descending, then name ascending. SQL panels: series past six are left out, with a note. state_timeline defaults to 8 rows, maximum 20; other noncategorical panels use the same limits. Scatter supports options.x_scale and options.y_scale (log or linear), with distinct x_unit for x and unit for y, each inferred independently when omitted. Rows returned use the count unit. State timelines preserve null and absent buckets as unknown and keep the top items without merging them into Other. Scatter requires a structured query. SQL state_timeline requires timestamp time, a string item, one numeric measure and optional string state; its interval is one minute and thresholds are required. Items rank by volume (row count) for top N. Previous-period comparison is not drawn for scatter or state_timeline. Distribution panels use a structured query with one count() measure and histogram: \{field: duration_ms, buckets: log2} for spans or \{field: value, buckets: explicit} for metrics. Negative span durations clamp to [0,1); +Inf goes to overflow; NaN is excluded because it is not a measurement. Metric histograms use stored temporality and shared per-series increases; resets contribute current counts. Delta histograms sum observation weights. query: \{from: spans|logs|metrics, where: [filter expressions], measures: [fn(field) as alias], by: [fields], bucket, sort, limit}. Measures: count(), rate() per second, error_rate() percent, share() percent, avg/min/max/sum(field), last(value) for metrics, p50/p75/p90/p95/p99(field), quantile(field, 0.999), count_distinct(field). Fields are columns from get_telemetry_schema or attributes['key'] / resource['key'] with the key written literally. Filters are SQL boolean expressions: service = $service, kind = 'SPAN_KIND_SERVER', status = 'STATUS_CODE_ERROR', http_route IN $routes, attributes['http.response.status_code']::INTEGER >= 500. Variables: \{name, kind: query|custom|constant|text, from, field, default, include_all}; $__all selects All and removes filters that use the variable. sql: one SELECT over spans, logs, metrics, service_rollup or edge_rollup that filters time with $__window(column). Use values exactly as get_telemetry_schema lists them. Example panel: \{"id":"latency","title":"Checkout latency","viz":"timeseries","width":8,"query":\{"from":"spans","where":["service = 'checkout'","kind = 'SPAN_KIND_SERVER'"],"measures":["p50(duration_ms)","p95(duration_ms)","p99(duration_ms)"],"bucket":"auto"},"unit":"ms","thresholds":[\{"value":1500,"status":"bad"}]}. Metrics: gauge/updown | counter | histogram. count(): points; count_distinct(): distinct. rate(): undefined | series increase/elapsed summed | count rate summed. sum(value): series means summed | increases | sum increases. avg(value): mean series means | undefined | sum/count increases. Gauge/updown sum/avg carry earlier points within 5m. Deltas spread across covered buckets; rate alone carries within 5m. rate() is per-second increase over covered elapsed time; auto >=1m. min/max: points | points | undefined. last(value): latest strictly before bucket/window end, reported within 5m, summed | same | undefined. pNN/quantile: points | undefined | interpolated bucket increases. summary: no value aggregates. Undefined: NULL. Fix name in its own where entry: one literal or single-value variable without All or multi-value; or name in by. Read type/temporality from get_telemetry_schema. state_timeline needs at least one threshold marked warn or bad, rather than only ok. Annotations attach only to timeseries, heatmap and state_timeline panels, scoped by service and namespace filters. drill: logs needs query.from=logs. Drill is not available on metrics. Every table format's field must exist in the returned frame; formatting cannot reference an absent column. A structured-table sparkline field must be a measure alias from that table's query. service_link needs variable to name the query or custom variable selected by the link. highlight applies to logs and log_patterns, at most 200 characters. split: deploy needs a service equality filter, as a literal or a single-value variable; other filters are allowed. When the service is All, a split: deploy panel shows the plain bar with a note explaining the missing service scope. Use service_map for dependency context, health for service health; neither proves causality. Example 1: Reference shapes; use only the types the question needs. \{"name":"Investigation shapes","annotations":\{"deploys":true,"anomalies":true},"variables":[\{"name":"service","kind":"query","from":"spans","field":"service"}],"panels":[\{"id":"latency_heat","title":"Latency over time","viz":"heatmap","query":\{"from":"spans","where":["service = $service"],"measures":["count()"],"bucket":"auto","histogram":\{"field":"duration_ms","buckets":"log2"}},"drill":"traces"},\{"id":"latency_hist","title":"Latency by operation","viz":"histogram","query":\{"from":"spans","where":["service = $service"],"measures":["count()"],"by":["operation"],"histogram":\{"field":"duration_ms","buckets":"log2"}},"drill":"traces"},\{"id":"items","title":"Calls versus tail latency","viz":"scatter","query":\{"from":"spans","where":["service = $service"],"measures":["count() as rows","p95(duration_ms) as tail_latency"],"by":["operation"]},"drill":"traces"},\{"id":"states","title":"Error states","viz":"state_timeline","query":\{"from":"spans","where":["service = $service"],"measures":["error_rate()"],"by":["operation"],"bucket":"auto"},"thresholds":[\{"value":5,"status":"bad"}],"drill":"traces"},\{"id":"log_rows","title":"Recent logs","viz":"logs","query":\{"from":"logs","where":["service = $service"],"sort":"time","limit":100},"drill":"logs"},\{"id":"patterns","title":"Repeated log patterns","viz":"log_patterns","query":\{"from":"logs","where":["service = $service"],"measures":["count()"],"by":["body_template"],"bucket":"auto","limit":50},"drill":"logs"},\{"id":"trace_rows","title":"Slow traces","viz":"traces","query":\{"from":"spans","where":["service = $service"],"sort":"duration_ms","limit":20},"drill":"traces"},\{"id":"map","title":"Service calls","viz":"service_map","query":\{"from":"spans"}},\{"id":"health","title":"Service health","viz":"health","query":\{"from":"spans"}}]} Example 2: Reference shapes; use only the types the question needs. \{"name":"Interaction shapes","variables":[\{"name":"service","kind":"query","from":"spans","field":"service"}],"panels":[\{"id":"hist","title":"Metric distribution","viz":"histogram","query":\{"from":"metrics","where":["service = $service"],"measures":["count()"],"histogram":\{"field":"value","buckets":"explicit"}}},\{"id":"table","title":"Service calls and trends","viz":"table","query":\{"from":"spans","measures":["count() as calls"],"by":["service"]},"options":\{"columns":[\{"field":"service","format":"service_link","variable":"service"},\{"field":"calls","format":"sparkline"}]}},\{"id":"prior","title":"Service calls one day earlier","viz":"bar","query":\{"from":"spans","measures":["count()"],"by":["service"]},"click":\{"set_variable":"service"},"time":\{"shift":"1d"}}]} Metric examples; use when the discovered type fits. \{"name":"Metric measures","panels":[\{"id":"network","title":"Network bytes per second","viz":"timeseries","query":\{"from":"metrics","where":["name = 'system.network.io'"],"measures":["rate()"],"by":["attributes['direction']"],"bucket":"auto"}},\{"id":"duration","title":"Request duration p95","viz":"timeseries","query":\{"from":"metrics","where":["name = 'http.server.request.duration'"],"measures":["p95(value)"],"bucket":"auto"},"unit":"s"}]} +**Create dashboard** — Create a dashboard for the authenticated user from a complete spec. Read get_telemetry_schema first and preview_panels until every panel is ok or deliberately empty. The result lists any panel that is still empty or failing. Successful saves return a receipt for that committed version: base_version, version, changes, layout_changed, dashboard_fields and save_check. Changes describe authored panel fields and position_changed; physical packing is only layout_changed. Order chips mark the minimal moved set. save_check reports checked, reason when unchecked, total elapsed_ms and one status/rows result per saved panel with elapsed_ms when executed. checked means every panel was executed, not that error or empty panels are healthy. Text rows identify nonempty content, not telemetry. Use the structured receipt rather than prose warnings as check evidence. Create receipts list added panels without layout or dashboard edit chips. No-op saves still create a new version with no change chips. Per-panel elapsed_ms is omitted when the panel did not run. A dashboard is \{name, description, time:\{range}, variables, panels}. Each panel: \{id (lowercase), title, viz, width 1-12, query or sql}. viz: stat or gauge (one measure; gauge needs min and max), timeseries (bucket auto, at most one by), bar (one or two by), table (up to three by), text (content in Markdown), heatmap (bucket auto, no by), histogram (at most one by), scatter (exactly two measures in x then y order, one item dimension and an optional colour dimension), state_timeline (one measure, one item dimension, bucket auto and thresholds required). Rollup panel types are service_map (dependency graph) and health (overview metrics, error trend and health distribution). They require query.from=spans, no measures, by, bucket or sort, and only namespace/service equality filters with a literal or single-value variable; explicit casts and values with surrounding whitespace are rejected. Reads default to 400 services/edges and are capped at 400 even when query.limit is larger. Service scope applies before the limit; map nodes carry p95_ms and edges carry average_ms; error_rate is percent. Row panel types are logs, log_patterns and traces. Logs and traces take no measures, by or bucket. Patterns use count() by body_template. Patterns group only by body_template. Severity is the most frequent severity, ties choose the higher severity. Service is the most frequent service, ties choose ascending service name. Both describe the scoped, redacted events, not extra grouping dimensions. Each pattern returns dominant severity, top service, count and a dense trend, computed in the same bounded query after redaction and scoped filters. Redaction applies before filtering, grouping, display and drill-down. Row panels cannot use SQL. "erroring" means slowest erroring traces: filter to erroring traces, then rank by root duration. Trace candidate ranking is approximate when filters exclude the root span, as with exemplars. Categorical charts use fixed-order colour slots. Categorical options.top defaults to 6, maximum 6. Structured panels: the server computes Other; series are chosen worst-first by confidence (Wilson lower bound for error rates; at least 20 samples for latency); the rest fold into Other (N). Count, rate, sum and share rank largest first. Error rates require at least 20 requests for eligible Wilson ranking (z=1.96). Percentiles, quantile, avg, max and min require at least 20 non-null numeric samples for eligible value ranking and use better (explicit or inferred). Ineligible series follow eligible ones by sample count descending. Ties use sample count descending, then name ascending. SQL panels: series past six are left out, with a note. state_timeline defaults to 8 rows, maximum 20; other noncategorical panels use the same limits. Scatter supports options.x_scale and options.y_scale (log or linear), with distinct x_unit for x and unit for y, each inferred independently when omitted. Rows returned use the count unit. State timelines preserve null and absent buckets as unknown and keep the top items without merging them into Other. Scatter requires a structured query. SQL state_timeline requires timestamp time, a string item, one numeric measure and optional string state; its interval is one minute and thresholds are required. Items rank by volume (row count) for top N. Previous-period comparison is not drawn for scatter or state_timeline. Distribution panels use a structured query with one count() measure and histogram: \{field: duration_ms, buckets: log2} for spans or \{field: value, buckets: explicit} for metrics. Negative span durations clamp to [0,1); +Inf goes to overflow; NaN is excluded because it is not a measurement. Metric histograms use stored temporality and shared per-series increases; resets contribute current counts. Delta histograms sum observation weights. query: \{from: spans|logs|metrics, where: [filter expressions], measures: [fn(field) as alias], by: [fields], bucket, sort, limit}. Measures: count(), rate() per second, error_rate() percent, share() percent, avg/min/max/sum(field), last(value) for metrics, p50/p75/p90/p95/p99(field), quantile(field, 0.999), count_distinct(field). Fields are columns from get_telemetry_schema or attributes['key'] / resource['key'] with the key written literally. Filters are SQL boolean expressions: service = $service, kind = 'SPAN_KIND_SERVER', status = 'STATUS_CODE_ERROR', http_route IN $routes, attributes['http.response.status_code']::INTEGER >= 500. Variables: \{name, kind: query|custom|constant|text, from, field, default, include_all}; $__all selects All and removes filters that use the variable. sql: one SELECT over spans, logs, metrics, service_rollup, edge_rollup or resource_inventory that filters time with $__window(column). Use values exactly as get_telemetry_schema lists them. Example panel: \{"id":"latency","title":"Checkout latency","viz":"timeseries","width":8,"query":\{"from":"spans","where":["service = 'checkout'","kind = 'SPAN_KIND_SERVER'"],"measures":["p50(duration_ms)","p95(duration_ms)","p99(duration_ms)"],"bucket":"auto"},"unit":"ms","thresholds":[\{"value":1500,"status":"bad"}]}. Metrics: gauge/updown | counter | histogram. count(): points; count_distinct(): distinct. rate(): undefined | series increase/elapsed summed | count rate summed. sum(value): series means summed | increases | sum increases. avg(value): mean series means | undefined | sum/count increases. Gauge/updown sum/avg carry earlier points within 5m. Deltas spread across covered buckets; rate alone carries within 5m. rate() is per-second increase over covered elapsed time; auto >=1m. min/max: points | points | undefined. last(value): latest strictly before bucket/window end, reported within 5m, summed | same | undefined. pNN/quantile: points | undefined | interpolated bucket increases. summary: no value aggregates. Undefined: NULL. Fix name in its own where entry: one literal or single-value variable without All or multi-value; or name in by. Read type/temporality from get_telemetry_schema. Panel SQL also supports resource_inventory: signal, namespace, service, full typed resource, first_seen and last_seen. One row per distinct full resource per signal/namespace/service observed inside the logical panel window; first_seen/last_seen are actual in-window event times. Select signal='spans', 'logs' or 'metrics' and use resource['key'] with literal dotted keys. count(*) counts resource identities per signal, not events or instances. Declare $__window(first_seen). This relation is SQL-only: query.from and variables remain spans|logs|metrics; unscoped SQL cannot read it. state_timeline needs at least one threshold marked warn or bad, rather than only ok. Annotations attach only to timeseries, heatmap and state_timeline panels, scoped by service and namespace filters. drill: logs needs query.from=logs. Drill is not available on metrics. Every table format's field must exist in the returned frame; formatting cannot reference an absent column. A structured-table sparkline field must be a measure alias from that table's query. service_link needs variable to name the query or custom variable selected by the link. highlight applies to logs and log_patterns, at most 200 characters. split: deploy needs a service equality filter, as a literal or a single-value variable; other filters are allowed. When the service is All, a split: deploy panel shows the plain bar with a note explaining the missing service scope. Use service_map for dependency context, health for service health; neither proves causality. Example 1: Reference shapes; use only the types the question needs. \{"name":"Investigation shapes","annotations":\{"deploys":true,"anomalies":true},"variables":[\{"name":"service","kind":"query","from":"spans","field":"service"}],"panels":[\{"id":"latency_heat","title":"Latency over time","viz":"heatmap","query":\{"from":"spans","where":["service = $service"],"measures":["count()"],"bucket":"auto","histogram":\{"field":"duration_ms","buckets":"log2"}},"drill":"traces"},\{"id":"latency_hist","title":"Latency by operation","viz":"histogram","query":\{"from":"spans","where":["service = $service"],"measures":["count()"],"by":["operation"],"histogram":\{"field":"duration_ms","buckets":"log2"}},"drill":"traces"},\{"id":"items","title":"Calls versus tail latency","viz":"scatter","query":\{"from":"spans","where":["service = $service"],"measures":["count() as rows","p95(duration_ms) as tail_latency"],"by":["operation"]},"drill":"traces"},\{"id":"states","title":"Error states","viz":"state_timeline","query":\{"from":"spans","where":["service = $service"],"measures":["error_rate()"],"by":["operation"],"bucket":"auto"},"thresholds":[\{"value":5,"status":"bad"}],"drill":"traces"},\{"id":"log_rows","title":"Recent logs","viz":"logs","query":\{"from":"logs","where":["service = $service"],"sort":"time","limit":100},"drill":"logs"},\{"id":"patterns","title":"Repeated log patterns","viz":"log_patterns","query":\{"from":"logs","where":["service = $service"],"measures":["count()"],"by":["body_template"],"bucket":"auto","limit":50},"drill":"logs"},\{"id":"trace_rows","title":"Slow traces","viz":"traces","query":\{"from":"spans","where":["service = $service"],"sort":"duration_ms","limit":20},"drill":"traces"},\{"id":"map","title":"Service calls","viz":"service_map","query":\{"from":"spans"}},\{"id":"health","title":"Service health","viz":"health","query":\{"from":"spans"}}]} Example 2: Reference shapes; use only the types the question needs. \{"name":"Interaction shapes","variables":[\{"name":"service","kind":"query","from":"spans","field":"service"}],"panels":[\{"id":"hist","title":"Metric distribution","viz":"histogram","query":\{"from":"metrics","where":["service = $service"],"measures":["count()"],"histogram":\{"field":"value","buckets":"explicit"}}},\{"id":"table","title":"Service calls and trends","viz":"table","query":\{"from":"spans","measures":["count() as calls"],"by":["service"]},"options":\{"columns":[\{"field":"service","format":"service_link","variable":"service"},\{"field":"calls","format":"sparkline"}]}},\{"id":"prior","title":"Service calls one day earlier","viz":"bar","query":\{"from":"spans","measures":["count()"],"by":["service"]},"click":\{"set_variable":"service"},"time":\{"shift":"1d"}}]} Metric examples; use when the discovered type fits. \{"name":"Metric measures","panels":[\{"id":"network","title":"Network bytes per second","viz":"timeseries","query":\{"from":"metrics","where":["name = 'system.network.io'"],"measures":["rate()"],"by":["attributes['direction']"],"bucket":"auto"}},\{"id":"duration","title":"Request duration p95","viz":"timeseries","query":\{"from":"metrics","where":["name = 'http.server.request.duration'"],"measures":["p95(value)"],"bucket":"auto"},"unit":"s"}]} Observed-resource SQL example. \{"name":"Observed resources","panels":[\{"id":"resources","title":"Span resources observed in this window","viz":"table","sql":"SELECT namespace,service,resource['service.version']::VARCHAR AS version,first_seen,last_seen FROM resource_inventory WHERE signal='spans' AND $__window(first_seen) ORDER BY namespace,service,version,first_seen,last_seen"}]} **Additive.** It creates something new and alters nothing that exists. @@ -166,7 +166,7 @@ Read-only. Calling it changes nothing. ## get_telemetry_schema -**Telemetry schema** — List the signals, columns with their common values, attribute keys seen per service, metric names, services, measure functions and units. Read this before drafting panels so filters use values that exist. See create_dashboard for the spec guide. +**Telemetry schema** — List the signals, columns with their common values, attribute keys seen per service, metric names, services, measure functions, units and SQL-only relations in sql_relations. Read this before drafting panels so filters use values that exist. See create_dashboard for the spec guide. Read-only. Calling it changes nothing. @@ -332,6 +332,21 @@ Read-only. Calling it changes nothing. | `to` | `string` | no | Absolute end, RFC3339Nano; requires from and excludes window | | `window` | `string` | no | Time window such as 15m, 1h, 24h, 168h, or 720h; defaults to 1h | +## Panel SQL relations + +### resource_inventory + +Distinct full resource identities per signal, namespace and service observed inside the logical panel window [start,end). Panel SQL only; unavailable to structured query.from, variables and unscoped SQL. count(*) counts resource identities per signal, not events or instances. Resource extrema describe actual observations inside the selected window, not a historical registry. + +| Column | SQL type | Means | +|---|---|---| +| `signal` | `VARCHAR` | spans, logs or metrics; select the required signal | +| `namespace` | `VARCHAR` | Original resource namespace, preserving NULL and empty values | +| `service` | `VARCHAR` | Original service name, preserving NULL and empty values | +| `resource` | `VARIANT` | Complete typed resource; use resource['key'], with dotted keys written literally | +| `first_seen` | `TIMESTAMPTZ_NS` | Earliest actual event observation inside the selected panel window | +| `last_seen` | `TIMESTAMPTZ_NS` | Latest actual event observation inside the selected panel window | + ## Authentication MCP uses OAuth, not a static key, and **not the ingest token** — an agent