Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ v2026.9.5
These are LabStack's shared API and MCP conventions. Product-specific
resources and protocol-defined contracts keep their own semantics.

- API errors are `{code, message}` with stable snake_case codes; tool errors
are `error: {code, message}`. Protocol-defined error bodies keep their spelling.
- Application HTTP routes use `/api`, lowercase words, meaningful slash-separated
domains/resources/actions, and plural collections. Do not use hyphens in
product-defined static path segments. Opaque resource IDs may contain hyphens.
Expand Down
2 changes: 2 additions & 0 deletions cmd/fanout-docgen/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ func renderRoutes(routes []api.RouteDoc) []byte {
b.WriteString("(`/api/settings/ingest`). Neither has a browser page yet.\n")
b.WriteString(":::\n\n")

b.WriteString("## Errors\n\nApplication API errors, including unknown routes and wrong methods, return exactly\n`{\"code\":\"bad_request\",\"message\":\"Safe explanation\"}`. Codes are stable\nsnake_case identifiers; clients branch on `code` rather than message text.\nInvalid dashboard or panel specs also include `problems` with field paths,\nmessages and optional hints. Every error response uses `Cache-Control: no-store`;\nHEAD errors have no body. Server failures return a short, safe message; the\nunderlying error is logged and never returned.\n\nDefault codes by HTTP status:\n\n| Status | Code |\n|---|---|\n| 400 | `bad_request` |\n| 401 | `unauthorized` |\n| 403 | `forbidden` |\n| 404 | `not_found` |\n| 405 | `method_not_allowed` |\n| 409 | `conflict` |\n| 413 | `payload_too_large` |\n| 415 | `unsupported_media_type` |\n| 422 | `unprocessable` |\n| 429 | `rate_limited` |\n| 500 | `internal` |\n| 502 | `bad_gateway` |\n| 503 | `unavailable` |\n| 504 | `timeout` |\n\nOther 4xx statuses use `bad_request`; other 5xx statuses use `internal`.\nSpecific conditions use `already_current`, `dashboard_version_not_found`,\n`dashboard_version_conflict`, `invalid_spec` and `last_active_administrator`.\nTool failures carry `error: {code, message}` inside the tool result.\nPanel execution results keep their own status and error fields.\nOAuth, OTLP/gRPC, MCP JSON-RPC and AG-UI run errors keep their protocol spelling.\nOperational responses at `/healthz`, `/readyz` and `/metrics` keep their bodies.\n\n")

b.WriteString("The **Requires** column is not a description of the rule. It is the answer the\n")
b.WriteString("authorization middleware gives for that exact method and path, so a route\n")
b.WriteString("cannot be documented as public while the server treats it otherwise.\n")
Expand Down
3 changes: 2 additions & 1 deletion cmd/fanout/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ func main() {

// Start Echo HTTP API
e := echo.New()
e.HTTPErrorHandler = api.HTTPErrorHandler
if err := api.ConfigureClientIP(e, cfg.TrustedProxyCIDRs); err != nil {
slog.Error("trusted proxy IP extraction configuration failed", "err", err)
os.Exit(1)
Expand All @@ -222,7 +223,7 @@ func main() {
}
attrs := []any{"method", v.Method, "uri", uri, "status", v.Status, "latency", v.Latency}
if v.Error != nil {
attrs = append(attrs, "err", v.Error)
attrs = []any{"method", v.Method, "route", c.Path(), "status", v.Status, "latency", v.Latency, "err", v.Error}
}
if v.Status >= 500 {
slog.Error("request", attrs...)
Expand Down
2 changes: 1 addition & 1 deletion docs/superpowers/plans/2026-10-04-agent-dashboards-m1.md
Original file line number Diff line number Diff line change
Expand Up @@ -4641,7 +4641,7 @@ func writeProblems(c *echo.Context, err error) (bool, error) {
if !errors.As(err, &problems) {
return false, nil
}
return true, c.JSON(http.StatusBadRequest, map[string]any{"message": "The dashboard spec is invalid.", "problems": problems})
return true, c.JSON(http.StatusBadRequest, map[string]any{"code": "invalid_spec", "message": "The dashboard spec is invalid.", "problems": problems})
}
```

Expand Down
5 changes: 3 additions & 2 deletions internal/agent/committed_panic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,12 @@ func TestRuntimePanicDistinguishesCommittedSave(t *testing.T) {
if json.Unmarshal([]byte(messageText(m.Content)), &p) != nil {
t.Fatal(m)
}
want := "tool execution failed"
failure, ok := p["error"].(map[string]any)
want := "tool_failed"
if commit {
want = "interrupted"
}
if p["error"] != want || p["isError"] != true || commit && m.Error != "interrupted" || !commit && m.Error == "interrupted" {
if !ok || failure["code"] != want || failure["message"] == "" || p["isError"] != true || commit && m.Error != "interrupted" || !commit && m.Error == "interrupted" {
t.Fatalf("commit=%v message=%+v", commit, m)
}
}
Expand Down
7 changes: 5 additions & 2 deletions internal/agent/fragment_content_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,11 @@ func TestAppToolErrorHasNoActivity(t *testing.T) {
}
if message.Role == agtypes.RoleTool {
found = true
var payload map[string]any
if json.Unmarshal([]byte(messageText(message.Content)), &payload) != nil || payload["error"] != "Invalid telemetry window" || payload["isError"] != true || message.Error == "" {
var payload struct {
Error struct{ Code, Message string }
IsError bool
}
if json.Unmarshal([]byte(messageText(message.Content)), &payload) != nil || payload.Error.Code != "tool_failed" || payload.Error.Message != "Invalid telemetry window" || !payload.IsError || message.Error == "" {
t.Fatal("tool error lost")
}
}
Expand Down
14 changes: 6 additions & 8 deletions internal/agent/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/labstack/fanout/internal/api"
"github.com/labstack/fanout/internal/dashboard"
appid "github.com/labstack/fanout/internal/id"
"github.com/labstack/fanout/internal/toolerror"
)

const systemPrompt = `You are Fanout's observability assistant. When a view is attached to your reply it is the picture: never draw diagrams, trees, or charts in text, never use code fences to draw boxes, arrows, or trees, and never add a table or list that restates what an attached view already shows; your prose adds only what the view omits. Use get_observability_overview first for broad health questions, get_intelligence_snapshot for the latest precomputed anomalies and recurring log patterns, get_service_topology for direct dependency edges, get_service_dependencies for bounded upstream or downstream reachability from a service, get_service_performance for activity/latency/endpoints/comparisons, inspect_trace for trace or root-cause inspection, and search_logs for log questions. Treat structured outputs as authoritative. You build and change the user's dashboards. Build one whenever the user asks for an overview, asks why something is slow, failing or changing, asks to compare, break down or track telemetry, or asks for anything they would want to look at again; answer a single factual question with a view instead. To build one, read get_telemetry_schema, draft a complete spec of panels that answer the request, run preview_panels, fix every invalid panel, replace or explain every empty one, and only then call create_dashboard. Cover every part of the request: when a part has no data, keep its panel and say why in the panel description instead of dropping it. Title each panel with exactly what it measures. Prefer a few precise panels over many vague ones: headline stats first, then the time series that explain them, then a table of the worst offenders. After saving, reply in two or three sentences with what the dashboard shows and what stands out. Use filter values exactly as the schema lists them. To change a dashboard, get_dashboard first and use edit_dashboard so unrelated panels stay as they are; replace only when the user asks for a redesign. State the time window you used, distinguish missing data from healthy behavior, and never invent services, metrics, or causal claims. Keep answers concise because attached views provide interactive details. Never expose implementation details to the user: do not mention protocol names, tool names, schemas, query IDs, data-source names, storage engines, providers, or internal execution steps. Refer to attached interactive content simply as a view.` + dashboardAnalysisGuidance
Expand Down Expand Up @@ -463,18 +464,15 @@ func (r *Runtime) execute(ctx context.Context, threadID, runID string, messages
// Relayed to the model as a tool error; log it so repeated
// tool-transport failures are findable server-side.
slog.Warn("agent tool execution failed", "thread_id", threadID, "run_id", runID, "tool", call.Name, "err", err)
execution = ToolExecution{Content: fmt.Sprintf(`{"error":%q}`, err.Error()), IsError: true}
execution = ToolExecution{Content: toolerror.JSON("tool_failed", "Tool execution failed. Please try again."), IsError: true}
}
messageError := errorString(execution.IsError)
if execution.IsError {
// AG-UI's live result event carries content, not Message.Error.
// Keep the error structural on that path as well as on reload.
var payload map[string]any
if json.Unmarshal([]byte(execution.Content), &payload) != nil || payload == nil {
payload = map[string]any{"error": execution.Content}
}
payload := toolerror.Normalize(execution.Content)
payload["isError"] = true
if payload["error"] == "interrupted" {
if failure, ok := payload["error"].(map[string]any); ok && failure["code"] == "interrupted" {
messageError = "interrupted"
}
raw, marshalErr := json.Marshal(payload)
Expand Down Expand Up @@ -533,15 +531,15 @@ func (r *Runtime) execute(ctx context.Context, threadID, runID string, messages
func (r *Runtime) executeTool(ctx context.Context, call ToolCall) (execution ToolExecution, err error) {
if answerOnly, _ := ctx.Value(answerOnlyKey{}).(bool); answerOnly {
if !r.tools.ReadOnly(call.Name) {
return ToolExecution{Content: `{"code":"answer_only","error":"This explanation request is answer-only. Do not save changes; explain the observed evidence and suggest corrections in your answer."}`, IsError: true}, nil
return ToolExecution{Content: toolerror.JSON("answer_only", "This explanation request is answer-only. Do not save changes; explain the observed evidence and suggest corrections in your answer."), IsError: true}, nil
}
}
ctx = dashboard.TrackSave(ctx)
defer func() {
if value := recover(); value != nil {
slog.Error("agent tool panicked", "tool", call.Name, "panic", value, "stack", string(debug.Stack()))
if dashboard.SaveCommitted(ctx) {
execution = ToolExecution{Content: `{"error":"interrupted"}`, IsError: true}
execution = ToolExecution{Content: toolerror.JSON("interrupted", "Tool execution was interrupted. The save may have completed."), IsError: true}
err = nil
} else {
execution = ToolExecution{}
Expand Down
2 changes: 1 addition & 1 deletion internal/agent/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ func TestRuntimeToolExecutionErrorFeedsModel(t *testing.T) {
second := provider.got[1]
var sawErrorResult bool
for _, message := range second {
if message.Role == RoleTool && message.ToolResult != nil && message.ToolResult.IsError && strings.Contains(message.ToolResult.Content, "mcp session closed") {
if message.Role == RoleTool && message.ToolResult != nil && message.ToolResult.IsError && strings.Contains(message.ToolResult.Content, `"code":"tool_failed"`) && !strings.Contains(message.ToolResult.Content, "mcp session closed") {
sawErrorResult = true
}
}
Expand Down
2 changes: 1 addition & 1 deletion internal/agent/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,7 @@ func completeInterruptedToolCalls(messages []agtypes.Message) []agtypes.Message
if answered[call.ID] {
continue
}
repaired = append(repaired, agtypes.Message{ID: message.ID + "-" + call.ID + "-interrupted", Role: agtypes.RoleTool, ToolCallID: call.ID, Content: `{"error":"interrupted"}`, Error: "interrupted"})
repaired = append(repaired, agtypes.Message{ID: message.ID + "-" + call.ID + "-interrupted", Role: agtypes.RoleTool, ToolCallID: call.ID, Content: `{"error":{"code":"interrupted","message":"Tool execution was interrupted. The save may have completed."}}`, Error: "interrupted"})
answered[call.ID] = true
}
}
Expand Down
4 changes: 2 additions & 2 deletions internal/agent/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ func TestStoreStartRunRepairsUnansweredToolCalls(t *testing.T) {
if len(seed[1].ToolCalls) != 1 || seed[1].ToolCalls[0].ID != "call-1" {
t.Fatal("interrupted call was erased")
}
if seed[2].Role != agtypes.RoleTool || seed[2].ToolCallID != "call-1" || seed[2].Content != `{"error":"interrupted"}` {
if seed[2].Role != agtypes.RoleTool || seed[2].ToolCallID != "call-1" || seed[2].Content != `{"error":{"code":"interrupted","message":"Tool execution was interrupted. The save may have completed."}}` {
t.Fatalf("missing interrupted result: %+v", seed)
}
if got := messageIDs(seed); strings.Join(got, ",") != "user-1,assistant-1,assistant-1-call-1-interrupted,run-1-outcome,user-2" {
Expand Down Expand Up @@ -764,7 +764,7 @@ func TestInterruptedBuildEvidenceSurvivesReloadAndTheNextRun(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(before.Messages) < 3 || before.Messages[2].ToolCallID != "create" || before.Messages[2].Content != `{"error":"interrupted"}` {
if len(before.Messages) < 3 || before.Messages[2].ToolCallID != "create" || before.Messages[2].Content != `{"error":{"code":"interrupted","message":"Tool execution was interrupted. The save may have completed."}}` {
t.Fatalf("reload=%+v", before.Messages)
}
next, err := store.StartRun(ctx, "owner", agtypes.RunAgentInput{ThreadID: "thread", RunID: "next", Messages: []agtypes.Message{{ID: "next-user", Role: agtypes.RoleUser, Content: "Again"}}})
Expand Down
16 changes: 15 additions & 1 deletion internal/agent/tool_error_structure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,14 @@ func TestRuntimeToolErrorsAreStructuredLiveAndPersisted(t *testing.T) {
if json.Unmarshal([]byte(messageText(message.Content)), &payload) != nil || payload["error"] == nil && payload["isError"] != true || message.Error == "" {
t.Fatalf("unstructured error: %+v", message)
}
if interrupted && (message.Error != "interrupted" || payload["error"] != "interrupted") {
failure, ok := payload["error"].(map[string]any)
if code, _ := failure["code"].(string); !ok || code == "" {
t.Fatalf("tool error has no code: %v", failure)
}
if message, _ := failure["message"].(string); message == "" {
t.Fatalf("missing error code or message: %+v", message)
}
if interrupted && (message.Error != "interrupted" || failure["code"] != "interrupted") {
t.Fatalf("unknown outcome lost: %+v", message)
}
}
Expand All @@ -130,6 +137,13 @@ func TestRuntimeToolErrorsAreStructuredLiveAndPersisted(t *testing.T) {
if json.Unmarshal([]byte(event.Content), &payload) != nil || payload["error"] == nil && payload["isError"] != true {
t.Fatalf("unstructured live error: %s", event.Content)
}
failure, ok := payload["error"].(map[string]any)
if code, _ := failure["code"].(string); !ok || code == "" {
t.Fatalf("tool error has no code: %v", failure)
}
if message, _ := failure["message"].(string); message == "" {
t.Fatalf("missing live code or message: %s", event.Content)
}
found = true
}
if !found {
Expand Down
3 changes: 2 additions & 1 deletion internal/agent/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/labstack/fanout/internal/dashboard"
fanoutmcp "github.com/labstack/fanout/internal/mcp"
"github.com/labstack/fanout/internal/toolerror"
"github.com/modelcontextprotocol/go-sdk/mcp"
)

Expand Down Expand Up @@ -116,7 +117,7 @@ func (r *ToolRegistry) Execute(ctx context.Context, call ToolCall) (ToolExecutio
}
}
if !allowed {
return ToolExecution{Content: "This tool is unavailable to the model: " + call.Name, IsError: true}, nil
return ToolExecution{Content: toolerror.JSON("tool_failed", "This tool is unavailable to the model: "+call.Name), IsError: true}, nil
}
arguments := json.RawMessage(call.Input)
if len(arguments) == 0 {
Expand Down
8 changes: 5 additions & 3 deletions internal/api/auth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func newTestAuthServerWith(t *testing.T, cfg config.Config, smtp auth.SMTPConfig
sessions := auth.NewBrowserSessions(sqlite.DB, cfg.SessionIdleTTL, cfg.SessionAbsoluteTTL, false)
audit := auth.NewAuditStore(sqlite.DB)
e := echo.New()
e.HTTPErrorHandler = HTTPErrorHandler
RegisterAuthMiddleware(e, users, sessions, audit, cfg)
RegisterAuthRoutes(e, users, codes, setup, settings.NewStore(sqlite.DB), sessions, audit, smtp, cfg)
return &testAuthServer{e: e, db: sqlite, users: users, codes: codes, setup: setup, setupToken: setupToken, sessions: sessions, audit: audit, cfg: cfg}
Expand Down Expand Up @@ -464,14 +465,14 @@ func TestStartDoesNotRevealAccountState(t *testing.T) {
}
}

func TestStartExplainsWhenSMTPIsNotConfigured(t *testing.T) {
func TestLoginWithoutSMTPReturnsUnavailable(t *testing.T) {
s := newTestAuthServer(t)
req := httptest.NewRequest(http.MethodPost, "/api/auth/code/send", strings.NewReader(`{"email":"active@example.com"}`))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
s.e.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "login link") {
t.Fatalf("status = %d body=%s, want 503 with login-link guidance", rec.Code, rec.Body.String())
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), `"code":"unavailable"`) {
t.Fatalf("status = %d body=%s, want 503 with unavailable code", rec.Code, rec.Body.String())
}
}

Expand Down Expand Up @@ -674,6 +675,7 @@ func TestSetupLifecycleAndIngestToken(t *testing.T) {
func TestSetupExpiryAndExistingAdminRetry(t *testing.T) {
t.Run("expired", func(t *testing.T) {
e := echo.New()
e.HTTPErrorHandler = HTTPErrorHandler
h := &AuthHandler{setup: expiredSetupCredential{}, setupLimiter: auth.NewKeyedLimiter(10, 15*time.Minute)}
e.POST("/api/auth/setup", h.Setup)
req := httptest.NewRequest(http.MethodPost, "/api/auth/setup", strings.NewReader(`{"email":"admin@example.com","setup_token":"expired"}`))
Expand Down
6 changes: 3 additions & 3 deletions internal/api/dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,15 +183,15 @@ func dashboardError(c *echo.Context, err error) error {
}
switch {
case errors.Is(err, dashboard.ErrAlreadyCurrent):
return c.JSON(http.StatusConflict, map[string]any{"message": err.Error(), "error_code": dashboard.AlreadyCurrentCode})
return newAPIError(http.StatusConflict, dashboard.AlreadyCurrentCode, err.Error())
case errors.Is(err, dashboard.ErrVersionNotFound):
return c.JSON(http.StatusNotFound, map[string]any{"message": err.Error(), "error_code": dashboard.VersionNotFoundCode})
return newAPIError(http.StatusNotFound, dashboard.VersionNotFoundCode, err.Error())
case errors.Is(err, dashboard.ErrNotFound):
return echo.NewHTTPError(http.StatusNotFound, "dashboard not found")
case errors.Is(err, dashboard.ErrConflict):
return echo.NewHTTPError(http.StatusConflict, "a dashboard with that name already exists")
case errors.Is(err, dashboard.ErrStale):
return echo.NewHTTPError(http.StatusConflict, "The dashboard changed since you opened it. Reload to see the latest version.")
return newAPIError(http.StatusConflict, "dashboard_version_conflict", "The dashboard changed since you opened it. Reload to see the latest version.")
case errors.Is(err, context.DeadlineExceeded):
return echo.NewHTTPError(http.StatusGatewayTimeout, "Checking the dashboard took too long. Try again, or narrow its panels.").Wrap(err)
case errors.Is(err, context.Canceled):
Expand Down
6 changes: 3 additions & 3 deletions internal/api/dashboard_history_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,14 @@ func TestHistoricalDashboardHTTPReadRequiresOwnerAndReturnsSaveChanges(t *testin
message, code string
}{
{cookie, "version 99 of dashboard " + first.ID + " does not exist", "dashboard_version_not_found"},
{s.login(t, other), "dashboard not found", ""},
{s.login(t, other), "dashboard not found", "not_found"},
} {
var wire map[string]any
missing := call("99", tc.cookie)
if err := json.Unmarshal(missing.Body.Bytes(), &wire); err != nil {
t.Fatal(err)
}
if missing.Code != 404 || wire["message"] != tc.message || (tc.code != "" && wire["error_code"] != tc.code) || (tc.code == "" && wire["error_code"] != nil) {
if missing.Code != 404 || wire["message"] != tc.message || (tc.code != "" && wire["code"] != tc.code) || (tc.code == "" && wire["code"] != nil) {
t.Fatalf("missing = %s", missing.Body)
}
}
Expand All @@ -107,7 +107,7 @@ func TestHistoricalDashboardHTTPReadRequiresOwnerAndReturnsSaveChanges(t *testin
if err := json.Unmarshal(current.Body.Bytes(), &conflict); err != nil {
t.Fatal(err)
}
if current.Code != 409 || conflict["error_code"] != "already_current" {
if current.Code != 409 || conflict["code"] != "already_current" {
t.Fatalf("current restore=%d %s", current.Code, current.Body)
}
versions, err := service.Versions(t.Context(), owner.ID, first.ID)
Expand Down
Loading
Loading