diff --git a/internal/agent/fragment_content_test.go b/internal/agent/fragment_content_test.go index f28b14a86..c684e5ca6 100644 --- a/internal/agent/fragment_content_test.go +++ b/internal/agent/fragment_content_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" fanoutmcp "github.com/labstack/fanout/internal/mcp" "github.com/labstack/fanout/internal/panel" + "github.com/labstack/fanout/internal/toolerror" "reflect" "strings" "testing" @@ -142,31 +143,38 @@ func TestAppSummariesBoundedOnSuccessAndError(t *testing.T) { } func TestAppToolErrorHasNoActivity(t *testing.T) { - messages := []agtypes.Message{{ID: "user", Role: agtypes.RoleUser, Content: "show"}} - provider := &scriptedProvider{steps: [][]StreamEvent{{{Type: EventToolUse, ToolCall: &ToolCall{ID: "call", Name: "query_telemetry", Input: `{}`}}, {Type: EventStop, StopReason: "tool_calls"}}, {{Type: EventStop, StopReason: "end_turn"}}}} - runtime := NewRuntime(provider, &fakeTools{execution: ToolExecution{Content: "Invalid telemetry window", IsError: true, AppResourceURI: "ui://fanout/panels.html"}}, nil) - emitter, _ := newTestEmitter() - if _, err := runtime.execute(t.Context(), "thread", "run", &messages, emitter); err != nil { - t.Fatal(err) - } - found := false - for _, message := range messages { - if message.Role == agtypes.RoleActivity { - t.Fatal("error emitted an app activity") - } - if message.Role == agtypes.RoleTool { - found = true - var payload struct { - Error struct{ Code, Message string } - IsError bool + for _, tc := range []struct{ name, tool, content, code, message string }{ + {"plain", "query_telemetry", "Invalid telemetry window", "tool_failed", "Invalid telemetry window"}, + {"unknown service", "get_service_performance", toolerror.JSON("unknown_service", "kafka is a messaging system used by checkout, fraud-detection, accounting"), "unknown_service", "kafka is a messaging system used by checkout, fraud-detection, accounting"}, + } { + t.Run(tc.name, func(t *testing.T) { + messages := []agtypes.Message{{ID: "user", Role: agtypes.RoleUser, Content: "show"}} + provider := &scriptedProvider{steps: [][]StreamEvent{{{Type: EventToolUse, ToolCall: &ToolCall{ID: "call", Name: tc.tool, Input: `{}`}}, {Type: EventStop, StopReason: "tool_calls"}}, {{Type: EventStop, StopReason: "end_turn"}}}} + runtime := NewRuntime(provider, &fakeTools{execution: ToolExecution{Content: tc.content, IsError: true, AppResourceURI: "ui://fanout/panels.html"}}, nil) + emitter, _ := newTestEmitter() + if _, err := runtime.execute(t.Context(), "thread", "run", &messages, emitter); err != nil { + t.Fatal(err) } - if json.Unmarshal([]byte(messageText(message.Content)), &payload) != nil || payload.Error.Code != "tool_failed" || payload.Error.Message != "Invalid telemetry window" || !payload.IsError || message.Error == "" { - t.Fatal("tool error lost") + found := false + for _, message := range messages { + if message.Role == agtypes.RoleActivity { + t.Fatal("error emitted an app activity") + } + if message.Role == agtypes.RoleTool { + found = true + var payload struct { + Error struct{ Code, Message string } + IsError bool + } + if json.Unmarshal([]byte(messageText(message.Content)), &payload) != nil || payload.Error.Code != tc.code || payload.Error.Message != tc.message || !payload.IsError || message.Error == "" { + t.Fatal("tool error lost") + } + } } - } - } - if !found { - t.Fatal("tool error missing") + if !found { + t.Fatal("tool error missing") + } + }) } } diff --git a/internal/agent/service_input_test.go b/internal/agent/service_input_test.go new file mode 100644 index 000000000..49df9bfcf --- /dev/null +++ b/internal/agent/service_input_test.go @@ -0,0 +1,73 @@ +package agent + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/labstack/fanout/internal/config" + fanoutmcp "github.com/labstack/fanout/internal/mcp" + "github.com/labstack/fanout/internal/observability" + "github.com/labstack/fanout/internal/panel" + "github.com/labstack/fanout/internal/query" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/labstack/fanout/internal/toolerror" +) + +func TestToolRegistryUnknownServiceBoundsOversizedAliases(t *testing.T) { + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 4, RollupInterval: time.Hour} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + duck, err := query.NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + defer duck.Close() + to := time.Now().UTC() + from := to.Add(-time.Hour) + batch := telemetrystore.Batch{ID: "oversized-aliases"} + for i := range 30 { + batch.Spans = append(batch.Spans, telemetry.Span{ + Namespace: "shop", ServiceName: fmt.Sprintf("user%02d-", i) + strings.Repeat("<\\\"\n界", 512), + TraceID: fmt.Sprint(i), SpanID: fmt.Sprint(i), + StartUnixNanos: from.Add(time.Minute).UnixNano(), EndUnixNanos: from.Add(2 * time.Minute).UnixNano(), + Attributes: map[string]any{"messaging.system": "kafka"}, + }) + } + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + executor := panel.NewExecutor(duck, 7) + server := fanoutmcp.NewWithIntelligence(observability.New(duck, duck, 7), nil, executor, nil, "test") + registry, err := NewToolRegistry(t.Context(), server.MCP()) + if err != nil { + t.Fatal(err) + } + defer registry.Close() + // Keep the service filter executable; the alias list above supplies the + // oversized guidance, and this name also exercises JSON/UTF-8 escaping. + for _, service := range []string{"kafka", strings.Repeat("<\\\"\n界", 64)} { + input, err := json.Marshal(map[string]any{"service": service, "namespace": "shop", "from": from, "to": to}) + if err != nil { + t.Fatal(err) + } + result, err := registry.Execute(t.Context(), ToolCall{Name: "get_service_performance", Input: string(input)}) + if err != nil || !result.IsError || result.Structured != nil { + t.Fatalf("unknown service result: %+v %v", result, err) + } + var failure toolerror.Result + if len(result.Content) > 16*1024 || !utf8.ValidString(result.Content) || json.Unmarshal([]byte(result.Content), &failure) != nil || failure.Error == nil || failure.Error.Code != "unknown_service" || failure.Error.Message == "" { + t.Fatalf("invalid error contract: bytes=%d valid_json=%v", len(result.Content), json.Valid([]byte(result.Content))) + } + if service == "kafka" && !strings.Contains(failure.Error.Message, "kafka is a messaging system used by user00-") { + t.Fatalf("alias guidance missing: %q", failure.Error.Message) + } + } +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 2135b5ae1..028c32249 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -93,6 +93,8 @@ type panelExecutor interface { Schema(context.Context, panel.SchemaRequest) (*panel.Schema, error) Exemplars(context.Context, panel.ExemplarRequest) (panel.ExemplarResponse, error) ResolveVariables(context.Context, panel.ResolveRequest) (map[string][]panel.Option, error) + ResolveWindow(time.Time, time.Time) (time.Time, time.Time, error) + ServiceGuidance(context.Context, time.Time, time.Time, string, string) (string, error) } type Server struct { @@ -386,6 +388,11 @@ func (s *Server) dependencies(ctx context.Context, _ *mcp.CallToolRequest, input if err != nil { return nil, output, safePanelToolError(err) } + if len(output.Data.Nodes) == 0 { + if err := s.checkService(ctx, scope, strings.TrimSpace(input.Service)); err != nil { + return nil, observability.Result[observability.Dependencies]{}, err + } + } return summary(output.Summary), output, nil } @@ -404,7 +411,16 @@ func (s *Server) presetFragment(ctx context.Context, input QueryInput, kind, ser } d := fragmentPreset(kind, service, input.Namespace, severity, search, input.Limit) d.Time = panel.Time{From: &scope.Start, To: &scope.End, Refresh: "off"} - return s.runFragment(ctx, panel.RunRequest{Dashboard: d}, "preset") + result, out, err := s.runFragment(ctx, panel.RunRequest{Dashboard: d}, "preset") + if err != nil { + return nil, PanelFragment{}, err + } + if allResultsEmpty(out.Results) { + if err := s.checkService(ctx, scope, service); err != nil { + return nil, PanelFragment{}, err + } + } + return result, out, nil } func (s *Server) overview(ctx context.Context, _ *mcp.CallToolRequest, input QueryInput) (*mcp.CallToolResult, PanelFragment, error) { return s.presetFragment(ctx, input, "overview", "", "", "") @@ -440,6 +456,11 @@ func (s *Server) trace(ctx context.Context, _ *mcp.CallToolRequest, input TraceI if err != nil { return nil, PanelFragment{}, err } + if allResultsEmpty(out.Results) && len(detail.Data.Spans) == 0 && len(detail.Data.Logs) == 0 { + if err := s.checkService(ctx, scope, strings.TrimSpace(input.Service)); err != nil { + return nil, PanelFragment{}, err + } + } if detail.Data.Spans == nil { detail.Data.Spans = []observability.TraceSpan{} } @@ -458,6 +479,44 @@ func (s *Server) trace(ctx context.Context, _ *mcp.CallToolRequest, input TraceI return summary(fragmentSummary(out)), out, nil } +func allResultsEmpty(results []panel.Result) bool { + if len(results) == 0 { + return false + } + for _, result := range results { + if result.Status != panel.StatusEmpty { + return false + } + } + return true +} + +// Called only after a service-scoped tool succeeds with entirely empty results. +// Unknown names return a product tool error without a typed success payload. +func (s *Server) checkService(ctx context.Context, scope observability.Scope, service string) error { + if service == "" || s.panels == nil { + return nil + } + message, err := s.panels.ServiceGuidance(ctx, scope.Start, scope.End, scope.Namespace, service) + if err != nil { + return safePanelToolError(err) + } + if message == "" { + return nil + } + // JSON escaping expands a byte by at most six bytes. Reserve the envelope + // and bound before encoding so the agent's 16 KiB app cap cannot cut JSON. + const maxMessageBytes = (16*1024 - 128) / 6 + if len(message) > maxMessageBytes { + message = message[:maxMessageBytes-len("…")] + for !utf8.ValidString(message) { + message = message[:len(message)-1] + } + message += "…" + } + return errors.New(toolerror.JSON("unknown_service", message)) +} + func (s *Server) scope(input QueryInput) (observability.Scope, error) { if input.Limit < 0 || input.Limit > 500 { return observability.Scope{}, errors.New("limit must be from 1 to 500") @@ -465,26 +524,35 @@ func (s *Server) scope(input QueryInput) (observability.Scope, error) { if (input.From == nil) != (input.To == nil) { return observability.Scope{}, errors.New("from and to must be supplied together") } + var start, end time.Time if input.From != nil { if strings.TrimSpace(input.Window) != "" { return observability.Scope{}, errors.New("from/to cannot coexist with window") } - start, end := input.From.UTC(), input.To.UTC() + start, end = input.From.UTC(), input.To.UTC() if !end.After(start) { return observability.Scope{}, errors.New("to must be after from") } - return observability.Scope{Namespace: input.Namespace, Start: start, End: end}, nil + } else { + window := time.Hour + if strings.TrimSpace(input.Window) != "" { + parsed, err := time.ParseDuration(input.Window) + if err != nil || parsed <= 0 { + return observability.Scope{}, fmt.Errorf("window must be a positive duration such as 15m or 1h") + } + window = parsed + } + end = s.now().UTC() + start = end.Add(-window) } - window := time.Hour - if strings.TrimSpace(input.Window) != "" { - parsed, err := time.ParseDuration(input.Window) - if err != nil || parsed <= 0 { - return observability.Scope{}, fmt.Errorf("window must be a positive duration such as 15m or 1h") + if s.panels != nil { + var err error + start, end, err = s.panels.ResolveWindow(start, end) + if err != nil { + return observability.Scope{}, safePanelToolError(err) } - window = parsed } - end := s.now().UTC() - return observability.Scope{Namespace: input.Namespace, Start: end.Add(-window), End: end}, nil + return observability.Scope{Namespace: input.Namespace, Start: start, End: end}, nil } func summary(text string) *mcp.CallToolResult { diff --git a/internal/mcp/service_input_test.go b/internal/mcp/service_input_test.go new file mode 100644 index 000000000..ed0cc566f --- /dev/null +++ b/internal/mcp/service_input_test.go @@ -0,0 +1,510 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/observability" + "github.com/labstack/fanout/internal/panel" + "github.com/labstack/fanout/internal/query" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/labstack/fanout/internal/toolerror" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func serviceInputFixture(t testing.TB) (*Server, *query.Duck, *telemetrystore.Repository, time.Time, time.Time) { + t.Helper() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 4, RollupInterval: time.Hour} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = repo.Close() }) + duck, err := query.NewDuck(t.Context(), cfg, repo) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = duck.Close() }) + to := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) + from := to.Add(-time.Hour) + batch := telemetrystore.Batch{ID: "service-input"} + for i, service := range []string{"checkout", "fraud-detection", "accounting"} { + for j := range 6 - 2*i { + n := from.Add(time.Minute).UnixNano() + int64(j) + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "shop", ServiceName: service, TraceID: service, SpanID: fmt.Sprint(j), Name: "orders process", Kind: "SPAN_KIND_CONSUMER", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, IngestedAt: n, DBSystem: "postgresql", PeerService: "broker", Attributes: map[string]any{"messaging.system": "kafka"}}) + } + } + batch.Logs = []telemetry.Log{{Namespace: "shop", ServiceName: "logger", TimeUnixNanos: from.Add(time.Minute).UnixNano(), Severity: "INFO", Body: "log only"}} + batch.Metrics = []telemetry.Metric{{Namespace: "shop", ServiceName: "collector", Name: "load", Type: "gauge", TimeUnixNanos: from.Add(time.Minute).UnixNano(), Value: 1}} + // These names are valid, but have no traffic in the requested window. + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "shop", ServiceName: "idle", TraceID: "idle", SpanID: "idle", StartUnixNanos: from.Add(-time.Hour).UnixNano(), EndUnixNanos: from.Add(-time.Hour).UnixNano() + 1}, telemetry.Span{Namespace: "other", ServiceName: "elsewhere", TraceID: "elsewhere", SpanID: "elsewhere", StartUnixNanos: from.Add(time.Minute).UnixNano(), EndUnixNanos: from.Add(time.Minute).UnixNano() + 1}) + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + executor := panel.NewExecutor(duck, 30) + queries := observability.New(duck, duck, 30) + executor.SetRollupReader(queries) + s := NewWithIntelligence(queries, nil, executor, nil, "test") + s.now = func() time.Time { return to } + return s, duck, repo, from, to +} + +func TestServiceInputsRejectUnknownBeforeRendering(t *testing.T) { + s, _, _, from, to := serviceInputFixture(t) + client := connectTestClient(t, s, &mcp.ClientCapabilities{Extensions: map[string]any{mcpUIExtension: map[string]any{"mimeTypes": []string{mcpAppMIME}}}}) + for _, name := range []string{"get_service_performance", "get_service_dependencies", "search_logs", "inspect_trace"} { + t.Run(name, func(t *testing.T) { + for _, service := range []string{"missing", "kafka", "postgresql", "broker", "elsewhere", "Checkout", "checkout' OR true --"} { + args := map[string]any{"service": service, "namespace": "shop"} + if name == "get_service_dependencies" { + args["window"] = "1h" + } else { + args["from"], args["to"] = from, to + } + result, err := client.CallTool(t.Context(), &mcp.CallToolParams{Name: name, Arguments: args}) + if err != nil || result == nil || !result.IsError || result.StructuredContent != nil { + t.Fatalf("%s: result=%+v err=%v", service, result, err) + } + var failure toolerror.Result + if err := json.Unmarshal([]byte(result.Content[0].(*mcp.TextContent).Text), &failure); err != nil || failure.Error == nil || failure.Error.Code != "unknown_service" { + t.Fatalf("%s: error=%+v decode=%v", service, failure, err) + } + message := failure.Error.Message + if !strings.Contains(message, "checkout, fraud-detection, accounting, collector, logger") || strings.Contains(message, "idle,") || strings.Contains(message, "elsewhere,") { + t.Fatalf("incorrect traffic/namespace guidance: %s", message) + } + kind := map[string]string{"kafka": "messaging system", "postgresql": "database system", "broker": "peer/callee"}[service] + if kind != "" && !strings.Contains(message, service+" is a "+kind+" used by checkout, fraud-detection, accounting") { + t.Fatalf("missing alias users: %s", message) + } + } + }) + } +} + +func TestServiceInputsKeepKnownAndIdleServices(t *testing.T) { + s, duck, _, from, to := serviceInputFixture(t) + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + for _, service := range []string{"checkout", "idle", "logger", "collector"} { + result, out, err := s.performance(t.Context(), nil, PerformanceInput{Service: service, Namespace: "shop", From: &from, To: &to}) + if err != nil || result.IsError || len(out.Results) != 4 { + t.Fatalf("cached=%v %s: result=%+v output=%+v err=%v", cached, service, result, out, err) + } + want := panel.StatusEmpty + if service == "checkout" { + want = panel.StatusOK + } + if out.Results[3].Status != want { + t.Fatalf("%s operations=%+v want %s", service, out.Results[3], want) + } + if service != "checkout" && !allResultsEmpty(out.Results) { + t.Fatalf("known empty service changed response: %s %+v", service, out.Results) + } + } + } + result, out, err := s.logs(t.Context(), nil, LogsInput{Service: "logger", Namespace: "shop", From: &from, To: &to}) + if err != nil || result.IsError || len(out.Results) == 0 || out.Results[0].Status != panel.StatusOK { + t.Fatalf("log-only service lost: %+v %+v %v", result, out, err) + } + result, out, err = s.trace(t.Context(), nil, TraceInput{Service: "checkout", Namespace: "shop", From: &from, To: &to}) + if err != nil || result.IsError || out.Trace == nil || len(out.Trace.Data.Spans) != 6 { + t.Fatalf("known trace lost: %+v %+v %v", result, out, err) + } +} + +type serviceGuidanceSpy struct { + panelExecutor + calls int +} + +func (e *serviceGuidanceSpy) ServiceGuidance(ctx context.Context, start, end time.Time, namespace, service string) (string, error) { + e.calls++ + return e.panelExecutor.ServiceGuidance(ctx, start, end, namespace, service) +} + +func TestServiceInputsWithResultsSkipMembership(t *testing.T) { + s, duck, _, from, to := serviceInputFixture(t) + if _, err := duck.DB.Exec(`INSERT INTO edge_rollup VALUES ('shop',?,'checkout','broker',6,1,0,'call')`, from.Add(time.Minute)); err != nil { + t.Fatal(err) + } + spy := &serviceGuidanceSpy{panelExecutor: s.panels} + s.panels = spy + client := connectTestClient(t, s, nil) + for _, tc := range []struct { + name string + args map[string]any + }{ + {"get_service_performance", map[string]any{"service": "checkout", "namespace": "shop", "from": from, "to": to}}, + {"get_service_dependencies", map[string]any{"service": "checkout", "namespace": "shop", "window": "1h"}}, + // An observed dependency is a result even if the root is only a callee. + {"get_service_dependencies", map[string]any{"service": "broker", "namespace": "shop", "window": "1h"}}, + {"search_logs", map[string]any{"service": "logger", "namespace": "shop", "from": from, "to": to}}, + {"inspect_trace", map[string]any{"service": "checkout", "namespace": "shop", "from": from, "to": to}}, + // An exact trace ID still returns its detail when the service filter has + // no list rows; that is not an entirely empty result. + {"inspect_trace", map[string]any{"service": "missing", "trace_id": "checkout", "namespace": "shop", "from": from, "to": to}}, + } { + t.Run(tc.name+"/"+tc.args["service"].(string), func(t *testing.T) { + result, err := client.CallTool(t.Context(), &mcp.CallToolParams{Name: tc.name, Arguments: tc.args}) + if err != nil || result.IsError || result.StructuredContent == nil { + t.Fatalf("result=%+v err=%v", result, err) + } + if spy.calls != 0 { + t.Fatalf("nonempty tool performed %d membership lookups", spy.calls) + } + }) + } +} + +func TestServiceInputsValidateOnlyAfterAllResultsEmpty(t *testing.T) { + s, _, _, from, to := serviceInputFixture(t) + executor := s.panels + for _, tc := range []struct { + name string + service string + statuses []string + runErr error + lookups int + unknown bool + }{ + {name: "unknown", service: "missing", statuses: []string{panel.StatusEmpty, panel.StatusEmpty}, lookups: 1, unknown: true}, + {name: "known empty", service: "idle", statuses: []string{panel.StatusEmpty, panel.StatusEmpty}, lookups: 1}, + {name: "no service", statuses: []string{panel.StatusEmpty, panel.StatusEmpty}}, + {name: "one populated", service: "missing", statuses: []string{panel.StatusEmpty, panel.StatusOK}}, + {name: "panel error", service: "missing", statuses: []string{panel.StatusEmpty, panel.StatusError}}, + {name: "execution error", service: "missing", runErr: errors.New("execution failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + ran := false + spy := &serviceGuidanceSpy{panelExecutor: receiptExecutor{panelExecutor: executor, run: func(context.Context, panel.RunRequest) ([]panel.Result, error) { + ran = true + if spyCalls := s.panels.(*serviceGuidanceSpy).calls; spyCalls != 0 { + t.Errorf("membership ran before preset: %d lookups", spyCalls) + } + var results []panel.Result + for i, status := range tc.statuses { + results = append(results, panel.Result{ID: fmt.Sprint(i), Status: status}) + } + return results, tc.runErr + }}} + s.panels = spy + result, out, err := s.performance(t.Context(), nil, PerformanceInput{Service: tc.service, Namespace: "shop", From: &from, To: &to}) + if !ran || spy.calls != tc.lookups { + t.Fatalf("ran=%v membership=%d want %d", ran, spy.calls, tc.lookups) + } + if tc.unknown { + if err == nil || !strings.Contains(err.Error(), `"unknown_service"`) || result != nil || len(out.Results) != 0 { + t.Fatalf("unknown result=%+v out=%+v err=%v", result, out, err) + } + } else if (err != nil) != (tc.runErr != nil) || tc.runErr == nil && (result == nil || len(out.Results) != len(tc.statuses)) { + t.Fatalf("result=%+v out=%+v err=%v", result, out, err) + } + }) + } +} + +func TestServiceGuidanceBoundsOversizedName(t *testing.T) { + s, _, _, from, to := serviceInputFixture(t) + // Such a name exceeds the preset filter limit now that execution runs first. + // Exercise the shared error bound directly to retain that regression check. + err := s.checkService(t.Context(), observability.Scope{Namespace: "shop", Start: from, End: to}, strings.Repeat("<\\\"\n界", 4096)) + var failure toolerror.Result + if err == nil || len(err.Error()) > 16*1024 || !utf8.ValidString(err.Error()) || json.Unmarshal([]byte(err.Error()), &failure) != nil || failure.Error == nil || failure.Error.Code != "unknown_service" || !strings.HasSuffix(failure.Error.Message, "…") { + t.Fatalf("invalid bounded error: %v", err) + } +} + +func TestServiceInputsClampWindowBeforeValidation(t *testing.T) { + s, duck, repo, _, to := serviceInputFixture(t) + executor := panel.NewExecutor(duck, 7) + queries := observability.New(duck, duck, 7) + executor.SetRollupReader(queries) + s.panels, s.queries = executor, queries + old := to.Add(-10 * 24 * time.Hour).UnixNano() + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "outside-effective-window", Logs: []telemetry.Log{{Namespace: "shop", ServiceName: "old-window-only", TimeUnixNanos: old}}}); err != nil { + t.Fatal(err) + } + client := connectTestClient(t, s, nil) + wantWindow := to.Add(-7*24*time.Hour).Format(time.RFC3339Nano) + "/" + to.Format(time.RFC3339Nano) + for _, name := range []string{"get_service_performance", "get_service_dependencies", "search_logs", "inspect_trace"} { + t.Run(name, func(t *testing.T) { + result, err := client.CallTool(t.Context(), &mcp.CallToolParams{Name: name, Arguments: map[string]any{"service": "checkout", "namespace": "shop", "window": "720h"}}) + if err != nil || result.IsError { + t.Fatalf("clamped known service: %+v %v", result, err) + } + encoded, err := json.Marshal(result.StructuredContent) + if err != nil { + t.Fatal(err) + } + if name == "get_service_dependencies" { + var out observability.Result[observability.Dependencies] + if err := json.Unmarshal(encoded, &out); err != nil || out.Provenance.Window != wantWindow { + t.Fatalf("dependency scope: %s %v", encoded, err) + } + } else { + var out PanelFragment + if err := json.Unmarshal(encoded, &out); err != nil || len(out.Results) == 0 { + t.Fatalf("fragment: %s %v", encoded, err) + } + for _, panelResult := range out.Results { + if panelResult.FromMS != to.Add(-7*24*time.Hour).UnixMilli() || panelResult.ToMS != to.UnixMilli() { + t.Fatalf("panel scope: %+v", panelResult) + } + } + if out.Trace != nil && out.Trace.Provenance.Window != wantWindow { + t.Fatalf("trace scope: %+v", out.Trace.Provenance) + } + } + result, err = client.CallTool(t.Context(), &mcp.CallToolParams{Name: name, Arguments: map[string]any{"service": "missing", "namespace": "shop", "window": "720h"}}) + var failure toolerror.Result + if err != nil || !result.IsError || json.Unmarshal([]byte(result.Content[0].(*mcp.TextContent).Text), &failure) != nil || failure.Error == nil || failure.Error.Code != "unknown_service" || strings.Contains(failure.Error.Message, "old-window-only") { + t.Fatalf("effective-window guidance: %+v %+v %v", result, failure, err) + } + }) + } +} + +func TestServiceInputsKeepRetainedEventTimes(t *testing.T) { + s, duck, repo, from, to := serviceInputFixture(t) + for _, offset := range []struct { + name string + at time.Time + }{{"late", to.Add(-365 * 24 * time.Hour)}, {"future", to.Add(365 * 24 * time.Hour)}} { + for _, signal := range []string{"log", "metric"} { + service := offset.name + "-" + signal + batch := telemetrystore.Batch{ID: service} + if signal == "log" { + batch.Logs = []telemetry.Log{{Namespace: "shop", ServiceName: service, TimeUnixNanos: offset.at.UnixNano(), Body: "retained"}} + } else { + batch.Metrics = []telemetry.Metric{{Namespace: "shop", ServiceName: service, TimeUnixNanos: offset.at.UnixNano(), Name: "retained", Type: "gauge", Value: 1}} + } + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + } + } + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + for _, service := range []string{"late-log", "late-metric", "future-log", "future-metric"} { + for _, namespace := range []string{"shop", "", "other"} { + message, err := s.panels.ServiceGuidance(t.Context(), from, to, namespace, service) + if err != nil || (message == "") != (namespace != "other") { + t.Errorf("cached=%v namespace=%q service=%q: %q %v", cached, namespace, service, message, err) + } + } + result, out, err := s.performance(t.Context(), nil, PerformanceInput{Service: service, Namespace: "shop", From: &from, To: &to}) + if err != nil || result.IsError || !allResultsEmpty(out.Results) { + t.Fatalf("retained service lost its empty response: cached=%v service=%s out=%+v err=%v", cached, service, out, err) + } + } + message, err := s.panels.ServiceGuidance(t.Context(), from, to, "shop", "missing") + if err != nil || strings.Contains(message, "late-") || strings.Contains(message, "future-") { + t.Fatalf("event time leaked into suggestions: %q %v", message, err) + } + } +} + +func TestServiceGuidanceLimitsSuggestionsAndScopesAliases(t *testing.T) { + s, duck, repo, from, to := serviceInputFixture(t) + batch := telemetrystore.Batch{ID: "more-services"} + for i := range 12 { + for j := range 20 - i { + n := from.Add(2*time.Minute).UnixNano() + int64(j) + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "shop", ServiceName: fmt.Sprintf("rank%02d", i), TraceID: fmt.Sprint(i), SpanID: fmt.Sprint(j), StartUnixNanos: n, EndUnixNanos: n + 1, PeerService: "alias", DBSystem: "alias", Attributes: map[string]any{"messaging.system": "alias"}}) + } + } + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + if _, err := duck.DB.Exec(`INSERT INTO edge_rollup VALUES ('shop',?,'checkout','external',9,1,0,'call'),('other',?,'elsewhere','external',99,1,0,'call')`, from.Add(time.Minute), from.Add(time.Minute)); err != nil { + t.Fatal(err) + } + message, err := s.panels.ServiceGuidance(t.Context(), from, to, "shop", "external") + if err != nil || !strings.Contains(message, "external is a peer/callee used by checkout.") || strings.Contains(message, "elsewhere") || strings.Contains(message, "rank10") || strings.Contains(message, "rank11") { + t.Fatalf("guidance=%s err=%v", message, err) + } + for i := range 10 { + if !strings.Contains(message, fmt.Sprintf("rank%02d", i)) { + t.Fatal(message) + } + } + message, err = s.panels.ServiceGuidance(t.Context(), from, to, "shop", "alias") + if err != nil || strings.Contains(message, "rank10") || strings.Contains(message, "rank11") { + t.Fatalf("unbounded aliases: %s %v", message, err) + } + for _, kind := range []string{"messaging system", "database system", "peer/callee"} { + prefix := "alias is a " + kind + " used by " + _, names, ok := strings.Cut(message, prefix) + if !ok || strings.Count(strings.SplitN(names, ".", 2)[0], ", ") != 9 { + t.Fatalf("expected ten users for %s: %s", kind, message) + } + } + // Validity is independent of the ten suggestions, and namespace is exact. + for _, tc := range []struct{ namespace, service string }{{"shop", "rank11"}, {"other", "elsewhere"}, {"", "elsewhere"}} { + message, err := s.panels.ServiceGuidance(t.Context(), from, to, tc.namespace, tc.service) + if err != nil || message != "" { + t.Fatalf("known %+v: %s %v", tc, message, err) + } + } + message, err = s.panels.ServiceGuidance(t.Context(), to.Add(time.Hour), to.Add(2*time.Hour), "shop", "kafka") + if err != nil || strings.Contains(message, "used by") || !strings.Contains(message, "No known services have traffic") { + t.Fatalf("alias leaked outside window: %s %v", message, err) + } +} + +func BenchmarkServiceInput(b *testing.B) { + s, duck, repo, from, to := serviceInputFixture(b) + // Synthetic demo-sized traffic; no private investigation data is needed. + batch := telemetrystore.Batch{ID: "benchmark-traffic"} + services := []string{"checkout", "fraud-detection", "accounting"} + for i := range 54000 { + n := from.Add(time.Duration(i) * time.Second / 15).UnixNano() + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "shop", ServiceName: services[i%len(services)], TraceID: fmt.Sprint(i), SpanID: fmt.Sprint(i), Name: "orders process", Kind: "SPAN_KIND_CONSUMER", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, IngestedAt: n, Attributes: map[string]any{"messaging.system": "kafka"}}) + } + if err := repo.Commit(b.Context(), batch); err != nil { + b.Fatal(err) + } + if _, err := duck.DB.Exec(`INSERT INTO service_rollup VALUES ('shop',?,'checkout',6,6,1,1,0,0,0)`, from.Add(time.Minute)); err != nil { + b.Fatal(err) + } + if _, err := duck.RefreshReadCaches(b.Context()); err != nil { + b.Fatal(err) + } + d := fragmentPreset("performance", "checkout", "shop", "", "", 0) + d.Time = panel.Time{From: &from, To: &to, Refresh: "off"} + for _, name := range []string{"rollup_check", "inventory_check", "unknown_guidance", "preset_baseline", "preset_lazy"} { + b.Run(name, func(b *testing.B) { + for b.Loop() { + var err error + switch name { + case "rollup_check": + _, err = s.panels.ServiceGuidance(b.Context(), from, to, "shop", "checkout") + case "inventory_check": + _, err = s.panels.ServiceGuidance(b.Context(), from, to, "shop", "accounting") + case "unknown_guidance": + _, err = s.panels.ServiceGuidance(b.Context(), from, to, "shop", "kafka") + case "preset_baseline": + _, _, err = s.runFragment(b.Context(), panel.RunRequest{Dashboard: d}, "preset") + case "preset_lazy": + _, _, err = s.performance(b.Context(), nil, PerformanceInput{Service: "checkout", Namespace: "shop", From: &from, To: &to}) + } + if err != nil { + b.Fatal(err) + } + } + }) + } +} + +// Each measured cold operation gets a fresh DuckDB engine and executor, with +// no completed-batch cache populated. Files remain in the OS page cache; this +// measures application/native cache coldness without requiring privileged I/O. +func BenchmarkServiceInputRetainedBatches(b *testing.B) { + cfg := config.Config{DataDir: b.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 4, RollupInterval: time.Hour} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = repo.Close() }) + to := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) + from := to.Add(-time.Hour) + const batches = 256 + for i := range batches { + batch := telemetrystore.Batch{ID: fmt.Sprintf("retained-%03d", i)} + for j := range 64 { + n := from.Add(time.Duration(i*64+j) * time.Second / 8).UnixNano() + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: fmt.Sprint(i, j), SpanID: fmt.Sprint(j), Name: "process", Kind: "SPAN_KIND_CONSUMER", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, Attributes: map[string]any{"messaging.system": "kafka"}}) + batch.Logs = append(batch.Logs, telemetry.Log{Namespace: "shop", ServiceName: "logger", TimeUnixNanos: n, Body: "retained"}) + batch.Metrics = append(batch.Metrics, telemetry.Metric{Namespace: "shop", ServiceName: "collector", TimeUnixNanos: n, Name: "load", Type: "gauge", Value: 1}) + } + if err := repo.Commit(b.Context(), batch); err != nil { + b.Fatal(err) + } + } + for _, name := range []string{"cold_inventory", "cold_unknown_guidance", "cold_preset_baseline", "cold_preset_lazy", "cold_unknown_baseline", "cold_unknown_lazy"} { + b.Run(name, func(b *testing.B) { + for b.Loop() { + b.StopTimer() + duck, err := query.NewDuck(b.Context(), cfg, repo) + if err != nil { + b.Fatal(err) + } + var cached int + if err := duck.DB.QueryRow(`SELECT count(*) FROM read_batches`).Scan(&cached); err != nil || cached != 0 { + b.Fatalf("expected cold batch cache: %d %v", cached, err) + } + executor := panel.NewExecutor(duck, 7) + queries := observability.New(duck, duck, 7) + executor.SetRollupReader(queries) + s := NewWithIntelligence(queries, nil, executor, nil, "test") + service := "checkout" + if name == "cold_unknown_baseline" || name == "cold_unknown_lazy" { + service = "kafka" + } + d := fragmentPreset("performance", service, "shop", "", "", 0) + d.Time = panel.Time{From: &from, To: &to, Refresh: "off"} + var out PanelFragment + var message string + b.StartTimer() + switch name { + case "cold_inventory": + message, err = executor.ServiceGuidance(b.Context(), from, to, "shop", "checkout") + case "cold_unknown_guidance": + message, err = executor.ServiceGuidance(b.Context(), from, to, "shop", "kafka") + case "cold_preset_baseline", "cold_unknown_baseline": + _, out, err = s.runFragment(b.Context(), panel.RunRequest{Dashboard: d}, "preset") + case "cold_preset_lazy", "cold_unknown_lazy": + _, out, err = s.performance(b.Context(), nil, PerformanceInput{Service: service, Namespace: "shop", From: &from, To: &to}) + } + b.StopTimer() + if closeErr := duck.Close(); closeErr != nil { + b.Fatal(closeErr) + } + if name == "cold_unknown_lazy" { + var failure toolerror.Result + if err == nil || json.Unmarshal([]byte(err.Error()), &failure) != nil || failure.Error == nil || failure.Error.Code != "unknown_service" || !strings.Contains(failure.Error.Message, "messaging system used by checkout") || len(out.Results) != 0 { + b.Fatalf("unexpected unknown result: %+v %v", out, err) + } + } else if err != nil { + b.Fatal(err) + } + if name == "cold_inventory" && message != "" || name == "cold_unknown_guidance" && !strings.Contains(message, "messaging system used by checkout") { + b.Fatalf("unexpected guidance: %q", message) + } + if strings.Contains(name, "preset") || name == "cold_unknown_baseline" { + if len(out.Results) != 4 { + b.Fatalf("expected four panel results: %+v", out.Results) + } + for _, result := range out.Results { + want := panel.StatusOK + if name == "cold_unknown_baseline" { + want = panel.StatusEmpty + } + if result.Status != want { + b.Fatalf("preset failed: %+v", result) + } + } + } + b.StartTimer() + } + b.ReportMetric(batches, "batches") + }) + } +} diff --git a/internal/panel/service_input.go b/internal/panel/service_input.go new file mode 100644 index 000000000..1cac606c2 --- /dev/null +++ b/internal/panel/service_input.go @@ -0,0 +1,143 @@ +package panel + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/labstack/fanout/internal/queryrows" +) + +// ServiceGuidance returns an empty message for an exact observed service name. +// Retained names remain valid even when the requested window has no traffic. +// Rollups are the common, cheap path; inventory covers publication/rollup lag +// and services that emit only logs or metrics. Neither uses a capped name list. +func (e *Executor) ServiceGuidance(ctx context.Context, start, end time.Time, namespace, service string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, e.timeout) + defer cancel() + if !start.Before(end) || end.Sub(start) > e.maxWindow { + return "", Problems{{Path: "time", Message: "window must be positive and within retention"}} + } + known, err := e.observedService(ctx, `SELECT EXISTS ( + SELECT 1 FROM service_rollup WHERE service=? AND (?='' OR namespace=?) +)`, service, namespace, namespace) + if err != nil || known { + return "", err + } + + // Retention follows ingestion time. Membership includes all event times in + // the pinned retained batches, including late and future-dated events. + history := queryrows.Window{Start: start, End: end, Namespace: namespace, Service: service, RetainedInventory: true} + historyCtx := queryrows.WithInventoryWindow(queryrows.WithWindow(ctx, history), history) + known, err = e.observedService(historyCtx, `SELECT EXISTS ( + SELECT 1 FROM resource_inventory WHERE service=? AND (?='' OR namespace=?) +)`, service, namespace, namespace) + if err != nil || known { + return "", err + } + + ctx = queryrows.WithWindow(ctx, queryrows.Window{Start: start, End: end, Namespace: namespace}) + rows, err := e.engine.QueryContext(ctx, serviceGuidanceSQL, start, end, namespace, service) + if err != nil { + return "", fmt.Errorf("service guidance: %w", err) + } + defer rows.Close() + var knownNames []string + users := map[string][]string{} + for rows.Next() { + var kind, ns, name string + var traffic int64 + if err := rows.Scan(&kind, &ns, &name, &traffic); err != nil { + return "", err + } + label := name + if namespace == "" && ns != "" { + label += " (namespace " + ns + ")" + } + if kind == "service" { + knownNames = append(knownNames, label) + } else { + users[kind] = append(users[kind], label) + } + } + if err := rows.Err(); err != nil { + return "", err + } + message := fmt.Sprintf("Unknown service %q", service) + if namespace != "" { + message += fmt.Sprintf(" in namespace %q", namespace) + } + message += "." + for _, kind := range []string{"messaging system", "database system", "peer/callee"} { + if names := users[kind]; len(names) > 0 { + message += fmt.Sprintf(" %s is a %s used by %s.", service, kind, strings.Join(names, ", ")) + } + } + if len(knownNames) == 0 { + message += " No known services have traffic in the requested window. Try a wider window or check the namespace." + } else { + message += " Known services by traffic in the requested window: " + strings.Join(knownNames, ", ") + ". Use an exact service name." + } + return message, nil +} + +func (e *Executor) observedService(ctx context.Context, sql string, args ...any) (bool, error) { + rows, err := e.engine.QueryContext(ctx, sql, args...) + if err != nil { + return false, fmt.Errorf("observe service: %w", err) + } + defer rows.Close() + var known bool + if rows.Next() { + if err := rows.Scan(&known); err != nil { + return false, err + } + } + return known, rows.Err() +} + +// Guidance is only queried after rejecting a name. Count actual event traffic +// rather than resource identities, which are not proportional to traffic. +// The read window pins/prunes Parquet, while predicates keep nanosecond bounds. +const serviceGuidanceSQL = ` +WITH span_rows AS MATERIALIZED ( + SELECT namespace,service,messaging_system,db_system,peer_service FROM spans + WHERE start_time >= $1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < $2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$3) +), traffic AS ( + SELECT namespace,service,count(*) AS n FROM span_rows GROUP BY namespace,service + UNION ALL + SELECT namespace,service,count(*) FROM logs + WHERE time >= $1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < $2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$3) GROUP BY namespace,service + UNION ALL + SELECT namespace,service,count(*) FROM metrics + WHERE time >= $1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < $2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$3) GROUP BY namespace,service +), known AS ( + SELECT 'service' AS kind,namespace,service,sum(n)::BIGINT AS n FROM traffic + WHERE service IS NOT NULL AND service<>'' GROUP BY namespace,service + ORDER BY n DESC,namespace,service LIMIT 10 +), aliases AS ( + SELECT 'messaging system' AS kind,namespace,service,count(*) AS n FROM span_rows + WHERE messaging_system=$4 GROUP BY namespace,service + UNION ALL + SELECT 'database system',namespace,service,count(*) FROM span_rows + WHERE db_system=$4 GROUP BY namespace,service + UNION ALL + SELECT 'peer/callee',namespace,service,count(*) FROM span_rows + WHERE peer_service=$4 GROUP BY namespace,service + UNION ALL + SELECT 'peer/callee',namespace,caller,sum(calls)::BIGINT FROM edge_rollup + WHERE bucket >= $1 AND bucket < $2 AND ($3='' OR namespace=$3) AND callee=$4 AND calls>0 + GROUP BY namespace,caller +), alias_users AS ( + SELECT kind,namespace,service,sum(n)::BIGINT AS n, + row_number() OVER (PARTITION BY kind ORDER BY sum(n) DESC,namespace,service) AS rank + FROM aliases GROUP BY kind,namespace,service +) +SELECT * FROM known +UNION ALL +SELECT kind,namespace,service,n FROM alias_users WHERE rank<=10 +ORDER BY kind,n DESC,namespace,service` diff --git a/internal/panel/timerange.go b/internal/panel/timerange.go index d55bff306..3bdc5239d 100644 --- a/internal/panel/timerange.go +++ b/internal/panel/timerange.go @@ -7,6 +7,12 @@ import ( "time" ) +// ResolveWindow applies the executor's time bounds before callers validate or +// execute a tool, so guidance and all subsequent queries use the same interval. +func (e *Executor) ResolveWindow(start, end time.Time) (time.Time, time.Time, error) { + return resolveWindow(Time{From: &start, To: &end}, nil, e.now(), e.maxWindow) +} + // resolveWindow turns a dashboard time, optionally overridden by a panel, // into a concrete UTC range no longer than retention. A range longer than // retention keeps its end and loses its oldest part rather than failing. diff --git a/internal/query/resource_inventory.go b/internal/query/resource_inventory.go index 36679d752..f2b434556 100644 --- a/internal/query/resource_inventory.go +++ b/internal/query/resource_inventory.go @@ -11,16 +11,17 @@ import ( // are exact only inside the logical window; boundary and unacknowledged files // must supply actual observations, rather than overlapping resource envelopes. func (d *Duck) resourceInventorySource(batches []telemetry.BatchMetadata, markers map[string]bool, window queryrows.Window) string { + retained := window.RetainedInventory var contributions []string for _, signal := range []string{"spans", "logs", "metrics"} { var interior []string var raw []telemetry.BatchMetadata for _, batch := range batches { - if !overlapping(batch, signal, window) { + bounds, count := batchTime(batch, signal) + if count == 0 || !retained && !overlapping(batch, signal, window) { continue } - bounds, _ := batchTime(batch, signal) - if markers[batch.ID] && bounds.Known && bounds.MinNanos >= window.Start.UnixNano() && bounds.MaxNanos < window.End.UnixNano() { + if markers[batch.ID] && (retained || bounds.Known && bounds.MinNanos >= window.Start.UnixNano() && bounds.MaxNanos < window.End.UnixNano()) { interior = append(interior, batch.ID) } else { raw = append(raw, batch) @@ -34,7 +35,11 @@ func (d *Duck) resourceInventorySource(batches []telemetry.BatchMetadata, marker if signal == "spans" { clock = "start_time" } - contributions = append(contributions, `SELECT `+sqlLiteral(signal)+` AS signal,namespace,service,resource,min(`+clock+`) AS first_seen,max(`+clock+`) AS last_seen FROM (`+cleanSource(signal, d.snapshotSource(signal, raw, batches))+`) WHERE `+windowPredicate(clock, window)+` GROUP BY namespace,service,resource`) + predicate := "true" + if !retained { + predicate = windowPredicate(clock, window) + } + contributions = append(contributions, `SELECT `+sqlLiteral(signal)+` AS signal,namespace,service,resource,min(`+clock+`) AS first_seen,max(`+clock+`) AS last_seen FROM (`+cleanSource(signal, d.snapshotSource(signal, raw, batches))+`) WHERE `+predicate+` GROUP BY namespace,service,resource`) } } if len(contributions) == 0 { diff --git a/internal/queryrows/window.go b/internal/queryrows/window.go index e2805407c..095c86b89 100644 --- a/internal/queryrows/window.go +++ b/internal/queryrows/window.go @@ -11,6 +11,9 @@ type Window struct { Start, End time.Time Namespace, Service string Kind ReadKind + // RetainedInventory ignores event time only for resource_inventory. Raw + // relations still require the caller's explicit event-time predicates. + RetainedInventory bool } type windowKey struct{}