Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/agent/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/labstack/fanout/internal/toolerror"
)

const systemPrompt = `You are Fanout's observability assistant. When a view is attached to your reply it is the picture: never draw diagrams, trees, or charts in text, never use code fences to draw boxes, arrows, or trees, and never add a table or list that restates what an attached view already shows; your prose adds only what the view omits. Use get_observability_overview first for broad health questions, get_intelligence_snapshot for the latest precomputed anomalies and recurring log patterns, get_service_topology for direct dependency edges, get_service_dependencies for bounded upstream or downstream reachability from a service, get_service_performance for activity/latency/operations/comparisons, inspect_trace for trace or root-cause inspection, and search_logs for log questions. Treat structured outputs as authoritative. You build and change the user's dashboards. Build one whenever the user asks for an overview, asks why something is slow, failing or changing, asks to compare, break down or track telemetry, or asks for anything they would want to look at again; answer a single factual question with a view instead. To build one, read get_telemetry_schema, draft a complete spec of panels that answer the request, run preview_panels, fix every invalid panel, replace or explain every empty one, and only then call create_dashboard. Cover every part of the request: when a part has no data, keep its panel and say why in the panel description instead of dropping it. Title each panel with exactly what it measures. Prefer a few precise panels over many vague ones: headline stats first, then the time series that explain them, then a table of the worst offenders. After saving, reply in two or three sentences with what the dashboard shows and what stands out. Use filter values exactly as the schema lists them. To change a dashboard, get_dashboard first and use edit_dashboard so unrelated panels stay as they are; replace only when the user asks for a redesign. State the time window you used, distinguish missing data from healthy behavior, and never invent services, metrics, or causal claims. Keep answers concise because attached views provide interactive details. Never expose implementation details to the user: do not mention protocol names, tool names, schemas, query IDs, data-source names, storage engines, providers, or internal execution steps. Refer to attached interactive content simply as a view.` + dashboardAnalysisGuidance
const systemPrompt = `You are Fanout's observability assistant. When a view is attached to your reply it is the picture: never draw diagrams, trees, or charts in text, never use code fences to draw boxes, arrows, or trees, and never add a table or list that restates what an attached view already shows; your prose adds only what the view omits. Use get_observability_overview first for broad health questions, get_intelligence_snapshot for the latest precomputed anomalies and recurring log patterns, get_service_topology for direct dependency edges, get_service_dependencies for bounded upstream or downstream reachability from a service, get_service_performance for activity/latency/operations/comparisons, inspect_trace for trace or root-cause inspection, and search_logs for log questions. Treat structured outputs as authoritative. You build and change the user's dashboards. Build one whenever the user asks for an overview, asks why something is slow, failing or changing, asks to compare, break down or track telemetry, or asks for anything they would want to look at again; answer a single factual question with a view instead. To build one, read get_telemetry_schema, draft a complete spec of panels that answer the request, run preview_panels, fix every invalid panel, replace or drop every empty one, and only then call create_dashboard. Cover every part of the request: when a part has no data, state that finding in the answer and use the signals the service actually has. Title each panel with exactly what it measures. Prefer a few precise panels over many vague ones: headline stats first, then the time series that explain them, then a table of the worst offenders. After saving, reply in two or three sentences with what the dashboard shows and what stands out. Use filter values exactly as the schema lists them. To change a dashboard, get_dashboard first and use edit_dashboard so unrelated panels stay as they are; replace only when the user asks for a redesign. State the time window you used, distinguish missing data from healthy behavior, and never invent services, metrics, or causal claims. Do not present empty results. Dashboards contain only panels that return data; text panels are exempt. Keep answers concise because attached views provide interactive details. Never expose implementation details to the user: do not mention protocol names, tool names, schemas, query IDs, data-source names, storage engines, providers, or internal execution steps. Refer to attached interactive content simply as a view.` + dashboardAnalysisGuidance

const dashboardAnalysisGuidance = ` For analysis dashboards, use only the types the question needs; do not fill a dashboard with all of them. Use heatmap for latency changes, histogram for distributions, scatter for relationships, state_timeline for threshold states, logs for events, log_patterns for repeated messages, traces for slow or erroring traces, service_map for dependencies, and health for service health. Use drill for span or log evidence; keep checked filters. Include annotations for change investigations; deploys and detector findings do not prove causes. Use a deploy split for scoped before/since comparisons; retain missing-deploy explanations. A definition, explanation, or single fact is an answer intent; do not create or replace a dashboard for it. Preserve every requested facet and explain absent telemetry without inventing it. Saved panels must keep working as new telemetry arrives: never filter on trace, span or request IDs or other values copied from one result; show recent evidence with a traces panel and drill to its logs. In replies, never name schema fields to the user. Explain in chat is answer intent: use the observed absolute panel window and resolved variables, explain errors and suggest corrections without creating, editing, replacing or restoring dashboards.`

Expand Down
2 changes: 1 addition & 1 deletion internal/agent/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ func TestSystemPromptForbidsTextDiagrams(t *testing.T) {
func TestSystemPromptBuildsDashboardsForDashboardShapedRequests(t *testing.T) {
for _, want := range []string{
"Build one whenever the user asks for an overview, asks why something is slow, failing or changing",
"when a part has no data, keep its panel and say why",
"when a part has no data, state that finding in the answer",
"Title each panel with exactly what it measures",
} {
if !strings.Contains(systemPrompt, want) {
Expand Down
52 changes: 52 additions & 0 deletions internal/agent/service_input_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ import (
"unicode/utf8"

"github.com/labstack/fanout/internal/config"
"github.com/labstack/fanout/internal/dashboard"
fanoutmcp "github.com/labstack/fanout/internal/mcp"
"github.com/labstack/fanout/internal/observability"
"github.com/labstack/fanout/internal/panel"
"github.com/labstack/fanout/internal/query"
controlstore "github.com/labstack/fanout/internal/store"
"github.com/labstack/fanout/internal/telemetry"
telemetrystore "github.com/labstack/fanout/internal/telemetry/store"
"github.com/labstack/fanout/internal/toolerror"
Expand Down Expand Up @@ -71,3 +73,53 @@ func TestToolRegistryUnknownServiceBoundsOversizedAliases(t *testing.T) {
}
}
}

func TestToolRegistryNoDataAndDashboardPanelErrors(t *testing.T) {
cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 4, RollupInterval: time.Hour}
repo, err := telemetrystore.Open(cfg.TelemetryDir())
if err != nil {
t.Fatal(err)
}
defer repo.Close()
duck, err := query.NewDuck(t.Context(), cfg, repo)
if err != nil {
t.Fatal(err)
}
defer duck.Close()
to := time.Now().UTC()
from := to.Add(-time.Hour)
if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "signals", Metrics: []telemetry.Metric{{ServiceName: "collector", Name: "load", Type: "gauge", Value: 1, TimeUnixNanos: from.Add(time.Minute).UnixNano()}}, Logs: []telemetry.Log{{ServiceName: "logger", Severity: "INFO", Body: "hello", TimeUnixNanos: from.Add(time.Minute).UnixNano()}}}); err != nil {
t.Fatal(err)
}
db, err := controlstore.NewSQLite(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.DB.Exec(`INSERT INTO users(id,email) VALUES ('owner','owner@example.test')`); err != nil {
t.Fatal(err)
}
executor := panel.NewExecutor(duck, 7)
server := fanoutmcp.NewWithIntelligence(observability.New(duck, duck, 7), dashboard.New(db.DB, executor), executor, nil, "test")
registry, err := NewToolRegistry(t.Context(), server.MCP())
if err != nil {
t.Fatal(err)
}
defer registry.Close()
spec := panel.Dashboard{Name: "Empty", Time: panel.Time{From: &from, To: &to}, Panels: []panel.Panel{{ID: "p", Title: "Collector requests", Viz: "stat", Query: &panel.Query{From: "spans", Where: []string{"service = 'collector'"}, Measures: []string{"count()"}}}}}
for _, tc := range []struct {
name, code, want string
args any
}{
{"get_service_performance", "no_data", "spans=0, logs=0, metrics=1", map[string]any{"service": "collector", "from": from, "to": to}},
{"search_logs", "no_data", "INFO (1)", map[string]any{"service": "logger", "severity": "ERROR", "from": from, "to": to}},
{"create_dashboard", "tool_failed", "panels[0]: Collector requests", map[string]any{"dashboard": spec}},
} {
input, _ := json.Marshal(tc.args)
result, err := registry.Execute(dashboard.WithOwner(t.Context(), "owner"), ToolCall{Name: tc.name, Input: string(input)})
var failure toolerror.Result
if err != nil || !result.IsError || result.Structured != nil || json.Unmarshal([]byte(result.Content), &failure) != nil || failure.Error == nil || failure.Error.Code != tc.code || !strings.Contains(failure.Error.Message, tc.want) {
t.Fatalf("%s: %+v %v", tc.name, result, err)
}
}
}
13 changes: 13 additions & 0 deletions internal/dashboard/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,14 @@ func decodeRecord(row generated.Dashboard) (Record, error) {
return record, nil
}

type saveCheckKey struct{}

// WithSaveCheck adds a transport-specific check of the exact spec before each
// write attempt, including edits retried against a concurrent version.
func WithSaveCheck(ctx context.Context, check func(context.Context, panel.Dashboard) error) context.Context {
return context.WithValue(ctx, saveCheckKey{}, check)
}

// prepare normalizes, validates and packs a spec for storage.
func (s *Service) prepare(ctx context.Context, spec *panel.Dashboard, repack bool) error {
panel.Normalize(spec)
Expand All @@ -199,6 +207,11 @@ func (s *Service) prepare(ctx context.Context, spec *panel.Dashboard, repack boo
if err := s.validator.Validate(validationCtx, spec); err != nil {
return err
}
if check, ok := ctx.Value(saveCheckKey{}).(func(context.Context, panel.Dashboard) error); ok {
if err := check(validationCtx, *spec); err != nil {
return err
}
}
if repack {
for i := range spec.Panels {
spec.Panels[i].Grid = nil
Expand Down
57 changes: 51 additions & 6 deletions internal/mcp/dashboards.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,17 +120,17 @@ var dashboardTools = [...]mcp.Tool{
},
{
Name: "create_dashboard", Title: "Create dashboard",
Description: "Create a dashboard for the authenticated user from a complete spec. Read get_telemetry_schema first and preview_panels until every panel is ok or deliberately empty. The result lists any panel that is still empty or failing. " + saveReceiptGuide + specGuide,
Description: "Create a dashboard for the authenticated user from a complete spec. Read get_telemetry_schema first and preview_panels until every non-text panel returns data. Empty or failing panels are rejected with their path, title and diagnosis. " + saveReceiptGuide + specGuide,
Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false)},
},
{
Name: "replace_dashboard", Title: "Replace dashboard",
Description: "Replace a dashboard's whole spec; omitted panels are removed. Use only for a redesign the user asked for; use edit_dashboard to change a few panels. " + saveReceiptGuide + "See create_dashboard for the spec guide.",
Description: "Every non-text panel must return data in the preview or the save is rejected. Replace a dashboard's whole spec; omitted panels are removed. Use only for a redesign the user asked for; use edit_dashboard to change a few panels. " + saveReceiptGuide + "See create_dashboard for the spec guide.",
Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)},
},
{
Name: "edit_dashboard", Title: "Edit dashboard",
Description: "Change a dashboard with typed operations, applied in order and saved as one version: add_panel, update_panel (set replaces the named fields), remove_panel, move_panel, set_variable, remove_variable, set_time, rename. Panels not named are left unchanged. Only edit when the user asks to change that dashboard. " + saveReceiptGuide + "See create_dashboard for the spec guide.",
Description: "Every non-text panel must return data in the preview or the save is rejected. Change a dashboard with typed operations, applied in order and saved as one version: add_panel, update_panel (set replaces the named fields), remove_panel, move_panel, set_variable, remove_variable, set_time, rename. Panels not named are left unchanged. Only edit when the user asks to change that dashboard. " + saveReceiptGuide + "See create_dashboard for the spec guide.",
Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPtr(true), OpenWorldHint: boolPtr(false)},
},
{
Expand Down Expand Up @@ -222,7 +222,7 @@ func (s *Server) dashboardCreate(ctx context.Context, req *mcp.CallToolRequest,
ctx = dashboard.WithBuildOrigin(ctx, origin)
}
}
record, err := s.dashboards.CreateWithChanges(ctx, owner, input.Dashboard, agentAuthor(owner))
record, err := s.dashboards.CreateWithChanges(dashboard.WithSaveCheck(ctx, s.checkDashboardPanels), owner, input.Dashboard, agentAuthor(owner))
if err != nil {
return nil, dashboardOutput{}, dashboardToolError(err)
}
Expand All @@ -234,7 +234,7 @@ func (s *Server) dashboardReplace(ctx context.Context, req *mcp.CallToolRequest,
if err != nil {
return nil, dashboardOutput{}, err
}
record, err := s.dashboards.ReplaceWithChanges(ctx, owner, strings.TrimSpace(input.ID), input.Dashboard, input.BaseVersion, agentAuthor(owner), input.Message)
record, err := s.dashboards.ReplaceWithChanges(dashboard.WithSaveCheck(ctx, s.checkDashboardPanels), owner, strings.TrimSpace(input.ID), input.Dashboard, input.BaseVersion, agentAuthor(owner), input.Message)
if err != nil {
return nil, dashboardOutput{}, dashboardToolError(err)
}
Expand All @@ -246,13 +246,58 @@ func (s *Server) dashboardEdit(ctx context.Context, req *mcp.CallToolRequest, in
if err != nil {
return nil, dashboardOutput{}, err
}
record, err := s.dashboards.EditWithChanges(ctx, owner, strings.TrimSpace(input.ID), input.Operations, input.BaseVersion, agentAuthor(owner), input.Message)
record, err := s.dashboards.EditWithChanges(dashboard.WithSaveCheck(ctx, s.checkDashboardPanels), owner, strings.TrimSpace(input.ID), input.Operations, input.BaseVersion, agentAuthor(owner), input.Message)
if err != nil {
return nil, dashboardOutput{}, dashboardToolError(err)
}
return s.saved(ctx, "Updated", record)
}

func (s *Server) checkDashboardPanels(ctx context.Context, spec panel.Dashboard) error {
telemetry := false
for _, p := range spec.Panels {
telemetry = telemetry || p.Viz != "text"
}
if !telemetry {
return nil
}
if s.panels == nil {
return errors.New("panels are unavailable")
}
results, err := s.panels.Run(ctx, panel.RunRequest{Dashboard: spec})
if err != nil {
return err
}
byID := make(map[string]panel.Result, len(results))
for _, r := range results {
byID[r.ID] = r
}
var problems panel.Problems
for i, p := range spec.Panels {
if p.Viz == "text" {
continue
}
r, exists := byID[p.ID]
// Execution status records data presence before the shared batch budget
// truncates frames, which can remove every row from a populated panel.
if exists && r.Status == panel.StatusOK {
continue
}
diagnosis := r.Diagnosis
if diagnosis == "" {
diagnosis = r.Error
}
if diagnosis == "" {
diagnosis = "Preview returned no rows."
}
problems = append(problems, panel.Problem{Path: fmt.Sprintf("panels[%d]", i), Message: fmt.Sprintf("%s: %s", p.Title, panel.RedactPaths(diagnosis)), Hint: "Replace or drop this panel; dashboards contain only panels that return data."})
}
if len(problems) > 0 {
return problems
}
return nil
}

func (s *Server) dashboardVersions(ctx context.Context, req *mcp.CallToolRequest, input DashboardIDInput) (*mcp.CallToolResult, dashboardVersionsOutput, error) {
owner, err := dashboardOwner(req)
if err != nil {
Expand Down
18 changes: 5 additions & 13 deletions internal/mcp/dashboards_tools_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func textDashboard(name string) panel.Dashboard {
}

func TestCreateDashboardAcceptsTheDocumentedShape(t *testing.T) {
s := newPanelServer(t)
s := panelServerFixture(t, true)
session := connectTestClient(t, s, nil)
// Metadata carries the in-process owner, as the agent runtime does.
call := func(args string) *mcp.CallToolResult {
Expand Down Expand Up @@ -162,21 +162,13 @@ func TestOtherOwnersDashboardIsNotFound(t *testing.T) {
}
}

func TestSaveReportsPanelsItCouldNotCheck(t *testing.T) {
// The structural validator accepts a filter the executor then rejects, so
// the post-save run fails after the dashboard is already stored.
func TestSaveRejectsPanelsItCannotPreview(t *testing.T) {
panels := newPanelServer(t).panels
s := newToolServer(t, structural{}, panels)
spec := panel.Dashboard{Name: "Unchecked", Panels: []panel.Panel{{ID: "requests", Title: "Requests", Viz: "stat", Query: &panel.Query{From: "spans", Where: []string{"service = = 'x'"}, Measures: []string{"count()"}}}}}
result, out, err := s.dashboardCreate(t.Context(), requestFor("owner"), DashboardCreateInput{Dashboard: spec})
if err != nil {
t.Fatal(err)
}
if out.Dashboard.Version != 1 {
t.Fatalf("dashboard not saved: %+v", out.Dashboard)
}
if text := result.Content[0].(*mcp.TextContent).Text; !strings.Contains(text, "Panels were not checked:") {
t.Fatalf("summary hides the failed check: %q", text)
_, out, err := s.dashboardCreate(t.Context(), requestFor("owner"), DashboardCreateInput{Dashboard: spec})
if err == nil || out.Dashboard != nil || out.Receipt != nil {
t.Fatalf("save=%+v err=%v", out, err)
}
}

Expand Down
4 changes: 2 additions & 2 deletions internal/mcp/fragment_budget_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func TestLargeTraceKeepsRootErrorsAndSlowest(t *testing.T) {
}
}
func TestNullArgumentsRawJSONRPC(t *testing.T) {
s := newPanelServer(t)
s := panelServerFixture(t, true)
h := s.HTTPHandler()
for _, name := range []string{"get_telemetry_schema", "list_dashboards", "get_observability_overview"} {
t.Run(name, func(t *testing.T) {
Expand All @@ -127,7 +127,7 @@ func TestNullArgumentsRawJSONRPC(t *testing.T) {
}
}
func TestViewMetadataOnWire(t *testing.T) {
s := newPanelServer(t)
s := panelServerFixture(t, true)
client := connectTestClient(t, s, nil)
for _, name := range []string{"query_telemetry", "get_service_topology"} {
args := map[string]any{}
Expand Down
Loading
Loading