From f1bdc58a1075799f3b90683e3976743e6a88c710 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 8 Nov 2026 08:21:25 -0800 Subject: [PATCH 1/2] feat(ingest): set the first ingest token from configuration Ingest stays closed until an administrator issues a token by hand, so an automated install cannot accept telemetry as soon as it starts. FANOUT_INGEST_TOKEN (or its _FILE form) now supplies the first token. Its hash is stored at startup only when no hash exists, so a token rotated in the UI survives restarts. The value must match the ingest token format, and a malformed one fails configuration load naming only the setting. Closes #446 --- .env.example | 6 + cmd/fanout/main.go | 28 +++- cmd/fanout/main_test.go | 46 ++++++ docs/authentication.md | 2 +- docs/operations.md | 4 +- internal/api/auth_test.go | 38 +++++ internal/config/config.go | 9 ++ internal/config/config_test.go | 77 ++++++++++ internal/ingest/auth.go | 8 +- internal/ingest/auth_test.go | 24 ++++ internal/settings/ingest.go | 59 +++++++- internal/settings/store_test.go | 133 ++++++++++++++++++ .../docs/docs/get-started/first-boot.mdx | 3 +- .../docs/docs/get-started/send-telemetry.mdx | 18 ++- site/src/content/docs/docs/guides/deploy.mdx | 7 +- .../docs/docs/reference/configuration.mdx | 1 + .../docs/reference/configuration/ingest.mdx | 11 ++ 17 files changed, 454 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 0c173671e..aa3bb6525 100644 --- a/.env.example +++ b/.env.example @@ -105,6 +105,12 @@ # Default: 268435456 # FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= +# FANOUT_INGEST_TOKEN; string; secret +# Sets the first ingest token when none is stored, so an automated install accepts telemetry as soon as it starts. Use foi_ plus 43 base64url characters. Only its hash is stored. It is ignored once a token exists, so a token rotated in the UI survives restarts. +# Default: (empty) +# FANOUT_INGEST_TOKEN= +# Alternatively set FANOUT_INGEST_TOKEN_FILE to a secret file path; do not set both. + # FANOUT_MAINTENANCE_INTERVAL; duration # Controls Parquet retention and compaction, and query-cache checkpointing. # Default: 1h diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 6e8c83453..3a972df41 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -184,6 +184,14 @@ func main() { } defer sqlite.Close() + settingsStore := settings.NewStore(sqlite.DB) + auditStore := auth.NewAuditStore(sqlite.DB) + // Runs before the alert engine, the control database's first other writer. + if err := applyConfiguredIngestToken(ctx, settingsStore, auditStore, cfg.IngestToken); err != nil { + slog.Error("configured ingest token init failed", "err", err) + os.Exit(1) + } + // Start alert engine var alertStore *alert.Store var alertEngine *alert.Engine @@ -198,11 +206,9 @@ func main() { slog.Info("alert engine enabled", "interval", cfg.AlertEvaluationInterval) } - settingsStore := settings.NewStore(sqlite.DB) userStore := auth.NewUserStore(sqlite.DB) identityStore := auth.NewIdentityStore(sqlite.DB) browserSessions := auth.NewBrowserSessions(sqlite.DB, cfg.SessionIdleTTL, cfg.SessionAbsoluteTTL, cfg.SecureCookies()) - auditStore := auth.NewAuditStore(sqlite.DB) setup := auth.NewSetup() userCount, err := userStore.CountUsers() @@ -713,6 +719,24 @@ func createLoginLink(cfg config.Config, rawEmail string, output io.Writer) error return nil } +// applyConfiguredIngestToken seeds the first ingest token from configuration. +// It logs the outcome by setting name and never the token or its hash. +func applyConfiguredIngestToken(ctx context.Context, store *settings.Store, audit *auth.AuditStore, token string) error { + if token == "" { + return nil + } + applied, err := store.ApplyConfiguredIngestToken(ctx, token, audit) + if err != nil { + return err + } + if applied { + slog.Info("ingest token set from FANOUT_INGEST_TOKEN") + } else { + slog.Info("FANOUT_INGEST_TOKEN ignored because an ingest token already exists") + } + return nil +} + func browserLoginURL(cfg config.Config) *url.URL { if publicURL, err := url.Parse(strings.TrimSpace(cfg.PublicURL)); err == nil && publicURL.Scheme != "" && publicURL.Host != "" { publicURL.Path = "/login" diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index a91915d73..aaca7e724 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -13,6 +13,7 @@ import ( "errors" "flag" "io" + "log/slog" "math/big" "net" "net/http" @@ -28,6 +29,7 @@ import ( "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/server" + "github.com/labstack/fanout/internal/settings" "github.com/labstack/fanout/internal/store" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" @@ -620,3 +622,47 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { }) } } + +func TestApplyConfiguredIngestTokenLogsWithoutToken(t *testing.T) { + sqlite, err := store.NewSQLite(":memory:") + if err != nil { + t.Fatalf("NewSQLite: %v", err) + } + defer sqlite.Close() + settingsStore := settings.NewStore(sqlite.DB) + audit := auth.NewAuditStore(sqlite.DB) + + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + first := "foi_" + strings.Repeat("A", 43) + second := "foi_" + strings.Repeat("B", 43) + if err := applyConfiguredIngestToken(context.Background(), settingsStore, audit, ""); err != nil { + t.Fatalf("apply without setting: %v", err) + } + if logs.Len() != 0 { + t.Fatalf("logs without the setting = %q, want none", logs.String()) + } + got, err := settingsStore.GetIngest(context.Background()) + if err != nil || got.TokenHash != "" { + t.Fatalf("hash without the setting = %q err = %v", got.TokenHash, err) + } + + if err := applyConfiguredIngestToken(context.Background(), settingsStore, audit, first); err != nil { + t.Fatalf("first apply: %v", err) + } + if err := applyConfiguredIngestToken(context.Background(), settingsStore, audit, second); err != nil { + t.Fatalf("second apply: %v", err) + } + + output := logs.String() + hash := settings.HashIngestToken(first) + if strings.Contains(output, first) || strings.Contains(output, second) || strings.Contains(output, hash) { + t.Fatalf("logs exposed token material: %s", output) + } + if strings.Count(output, "FANOUT_INGEST_TOKEN") != 2 { + t.Fatalf("logs must report both outcomes by setting name: %s", output) + } +} diff --git a/docs/authentication.md b/docs/authentication.md index 81e58cce3..f6fa9ae14 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -259,7 +259,7 @@ access, including browser sessions and OAuth tokens, through | Setup URL | Creates the first administrator | One-time, expires in 1 hour | | Login URL | Authenticates one existing active local user | One-time, expires in 15 minutes; mint with `fanout login-link` | | Browser session | Interactive access | Idle and absolute TTL; revoked on role, email, or status change | -| Ingest token | OTLP ingest | Settings page | +| Ingest token | OTLP ingest | Settings page; `FANOUT_INGEST_TOKEN` sets only the first token | | Metrics token | `/metrics` scrape | `FANOUT_METRICS_TOKEN` | | MCP OAuth tokens | MCP clients | 30-day access tokens, checked on every request; 90-day refresh tokens renewed on each use. Refresh rotation with replay-family revocation, admitting the same client's rotated token for one access-token lifetime so processes sharing a credential can refresh from stale copies. Users disconnect individual agents in Settings; admins can still revoke all of a user's access | diff --git a/docs/operations.md b/docs/operations.md index 83a637303..57783a90d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -31,6 +31,7 @@ Every secret accepts either its value or its file path: | `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | | `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | | `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_INGEST_TOKEN` | `FANOUT_INGEST_TOKEN_FILE` | | `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | | `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | | `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | @@ -135,7 +136,8 @@ includes `/v1`, while Anthropic's excludes it. Both native and container deployments use one listener. Native installs default to loopback; set `FANOUT_ADDR=:7520` for remote access. The image supplies `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. Every OTLP request -requires the ingest token created during first-admin setup; browser sessions +requires the ingest token created during first-admin setup or set by +`FANOUT_INGEST_TOKEN` on an empty install; browser sessions and MCP tokens do not authorize ingest. Do not embed the ingest token in a browser or shipped mobile application. diff --git a/internal/api/auth_test.go b/internal/api/auth_test.go index a949981ba..bac5eaeb7 100644 --- a/internal/api/auth_test.go +++ b/internal/api/auth_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "errors" "net/http" @@ -753,3 +754,40 @@ func TestSetupConsumesTokenBeforeFalliblePostSetupWork(t *testing.T) { t.Fatalf("setup token state = %v, want unset: a committed administrator must not leave a live setup credential", got) } } + +func TestSetupKeepsTokenAppliedFromConfiguration(t *testing.T) { + s := newTestAuthServer(t) + store := settings.NewStore(s.db.DB) + token := "foi_" + strings.Repeat("A", 43) + if applied, err := store.ApplyConfiguredIngestToken(context.Background(), token, s.audit); err != nil || !applied { + t.Fatalf("applied = %v err = %v", applied, err) + } + + req := httptest.NewRequest(http.MethodPost, "/api/auth/setup", strings.NewReader(`{"email":"admin@example.com","setup_token":"`+s.setupToken+`"}`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("setup = %d %s", rec.Code, rec.Body.String()) + } + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + if body["status"] != "authenticated" { + t.Fatalf("status = %q", body["status"]) + } + if _, issued := body["ingest_token"]; issued { + t.Fatal("setup must not issue a token when configuration already supplied one") + } + if strings.Contains(rec.Body.String(), token) { + t.Fatal("setup response exposed the configured token") + } + got, err := store.GetIngest(context.Background()) + if err != nil { + t.Fatalf("GetIngest: %v", err) + } + if !settings.CheckIngestToken(token, got.TokenHash) { + t.Fatal("setup replaced the configured token") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 74f2f5e42..e6290efa1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,7 @@ import ( "time" appauth "github.com/labstack/fanout/internal/auth" + "github.com/labstack/fanout/internal/secret" ) const maxIngestBatchSize = 50_000 @@ -60,6 +61,11 @@ type Config struct { // until durable publication. Zero is unbounded. A single request is admitted // when no other request holds budget, even if it exceeds the limit. IngestMaxInFlightBytes int `group:"ingest" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` + // IngestToken sets the first ingest token when none is stored, so an automated + // install accepts telemetry as soon as it starts. Use foi_ plus 43 base64url + // characters. Only its hash is stored. It is ignored once a token exists, so a + // token rotated in the UI survives restarts. + IngestToken string `group:"ingest" env:"FANOUT_INGEST_TOKEN" secret:"true"` // RollupInterval schedules analytical rollups; use a Go duration of at least 1s. RollupInterval time.Duration `group:"storage" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` // MCPEnabled serves the MCP endpoint at /mcp. @@ -282,6 +288,9 @@ func (c Config) Validate() error { if c.IngestBatchSize <= 0 || c.IngestBatchSize > maxIngestBatchSize { return fmt.Errorf("FANOUT_INGEST_BATCH_SIZE must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) } + if c.IngestToken != "" && !secret.ValidSecret(c.IngestToken, "foi_") { + return fmt.Errorf("FANOUT_INGEST_TOKEN must be foi_ followed by 43 base64url characters") + } if c.RollupInterval < time.Second { return fmt.Errorf("FANOUT_ROLLUP_INTERVAL must be at least 1s, got %s", c.RollupInterval) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ab31b48fb..ab35b8853 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -497,3 +497,80 @@ func TestSecureCookies(t *testing.T) { }) } } + +func TestLoadIngestToken(t *testing.T) { + const name = "FANOUT_INGEST_TOKEN" + valid := "foi_" + strings.Repeat("A", 43) + + t.Run("absent by default", func(t *testing.T) { + cfg, err := Load(LoadOptions{Environ: validEnvironment()}) + if err != nil { + t.Fatal(err) + } + if cfg.IngestToken != "" { + t.Fatal("ingest token must default to empty") + } + }) + + t.Run("environment value", func(t *testing.T) { + cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"="+valid)}) + if err != nil { + t.Fatal(err) + } + if cfg.IngestToken != valid { + t.Fatal("ingest token was not loaded from the environment") + } + }) + + t.Run("file value", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ingest-token") + if err := os.WriteFile(path, []byte(valid+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"_FILE="+path)}) + if err != nil { + t.Fatal(err) + } + if cfg.IngestToken != valid { + t.Fatal("ingest token was not loaded from its file") + } + }) + + for _, malformed := range []string{ + "synthetic-private-value", + "foi_" + strings.Repeat("A", 42), + "foi_" + strings.Repeat("A", 44), + "foi_" + strings.Repeat("A", 42) + "+", + "fo_" + strings.Repeat("A", 43), + strings.Repeat("A", 43), + " " + valid, + } { + t.Run("rejects malformed environment value", func(t *testing.T) { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"="+malformed)}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), malformed) { + t.Fatal("expected redacted error naming the setting") + } + }) + t.Run("rejects malformed file value", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ingest-token") + if err := os.WriteFile(path, []byte(malformed), 0o600); err != nil { + t.Fatal(err) + } + _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"_FILE="+path)}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), malformed) { + t.Fatal("expected redacted error naming the setting") + } + }) + } + + t.Run("rejects both forms", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ingest-token") + if err := os.WriteFile(path, []byte(valid), 0o600); err != nil { + t.Fatal(err) + } + _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"="+valid, name+"_FILE="+path)}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), valid) { + t.Fatal("expected redacted conflict error naming the setting") + } + }) +} diff --git a/internal/ingest/auth.go b/internal/ingest/auth.go index e279840d3..19bc06935 100644 --- a/internal/ingest/auth.go +++ b/internal/ingest/auth.go @@ -46,8 +46,8 @@ func (a *ingestAuthorizer) Unary() grpc.UnaryServerInterceptor { } } -// authorize requires a valid ingest token on every request (set up during -// admin creation, rotatable from the settings page). Peer IP is not +// authorize requires a valid ingest token on every request (set from +// configuration or during admin creation, rotatable from the settings page). Peer IP is not // considered — operators decide who reaches the port. func (a *ingestAuthorizer) authorize(ctx context.Context) error { err := a.authorizeToken(ctx, ingestTokenFromContext(ctx)) @@ -82,8 +82,8 @@ func (a *ingestAuthorizer) authorizeToken(ctx context.Context, token string) err return fmt.Errorf("%w: %v", errIngestAuthUnavailable, err) } if ingestCfg.TokenHash == "" { - // Pre-setup (no admin yet) — no token exists to check against. Reject - // all requests until setup completes; collectors must wait. + // No token exists to check against: none was configured and setup has + // not issued one. Reject all requests; collectors must wait. slog.Warn("ingest: rejecting request — no ingest token configured (setup not complete)") return errIngestNotInitialized } diff --git a/internal/ingest/auth_test.go b/internal/ingest/auth_test.go index a3b7ab8c4..5f839d837 100644 --- a/internal/ingest/auth_test.go +++ b/internal/ingest/auth_test.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/settings" appstore "github.com/labstack/fanout/internal/store" ) @@ -143,3 +144,26 @@ func TestMalformedIngestSecretsSkipSettingsLookup(t *testing.T) { } } } + +func TestAuthorize_AcceptsTokenAppliedFromConfiguration(t *testing.T) { + sqlite, err := appstore.NewSQLite(":memory:") + if err != nil { + t.Fatalf("NewSQLite: %v", err) + } + t.Cleanup(func() { _ = sqlite.Close() }) + store := settings.NewStore(sqlite.DB) + token := "foi_" + strings.Repeat("A", 43) + if applied, err := store.ApplyConfiguredIngestToken(context.Background(), token, auth.NewAuditStore(sqlite.DB)); err != nil || !applied { + t.Fatalf("applied = %v err = %v", applied, err) + } + + authorizer := newIngestAuthorizer(store) + valid := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+token)) + if err := authorizer.authorize(valid); err != nil { + t.Fatalf("authorize configured token: %v", err) + } + other := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer foi_"+strings.Repeat("B", 43))) + if status.Code(authorizer.authorize(other)) != codes.Unauthenticated { + t.Fatal("a different token must be rejected") + } +} diff --git a/internal/settings/ingest.go b/internal/settings/ingest.go index ab5a9b53e..e37287a83 100644 --- a/internal/settings/ingest.go +++ b/internal/settings/ingest.go @@ -2,7 +2,9 @@ package settings import ( "context" + "database/sql" "encoding/json" + "errors" "fmt" "time" @@ -13,10 +15,10 @@ import ( const ingestKey = "ingest" -// Ingest holds the ingest-auth configuration. TokenHash is set during -// first-admin setup and rotatable from the Settings page; when unset -// (pre-setup), the authorizer rejects all OTLP requests. Collectors -// present the token via `Authorization: Bearer`. +// Ingest holds the ingest-auth configuration. TokenHash is set from +// FANOUT_INGEST_TOKEN at startup or during first-admin setup, and is rotatable +// from the Settings page; when unset, the authorizer rejects all OTLP requests. +// Collectors present the token via `Authorization: Bearer`. type Ingest struct { TokenHash string `json:"token_hash"` } @@ -52,10 +54,55 @@ func (s *Store) SetIngestWithAudit(ctx context.Context, ingest Ingest, audit *au return fmt.Errorf("settings: begin ingest update: %w", err) } defer func() { _ = tx.Rollback() }() - if _, err := generated.New(tx).UpsertSetting(writeCtx, generated.UpsertSettingParams{Key: ingestKey, Value: string(payload), UpdatedAt: time.Now().UTC().Format(time.RFC3339)}); err != nil { + return commitIngest(writeCtx, tx, payload, audit, event) +} + +// ApplyConfiguredIngestToken stores the hash of a token supplied by +// configuration when no token hash exists, and reports whether it did. An +// existing hash always wins, so rotation survives restarts that still carry the +// setting. Call it before any other writer uses the control database: the check +// and the write share a transaction, which a concurrent writer would abort. +func (s *Store) ApplyConfiguredIngestToken(ctx context.Context, token string, audit *auth.AuditStore) (bool, error) { + if audit == nil { + return false, fmt.Errorf("settings: audit store is required for ingest updates") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("settings: begin ingest update: %w", err) + } + defer func() { _ = tx.Rollback() }() + + current := DefaultIngest() + row, err := generated.New(tx).GetSetting(ctx, ingestKey) + switch { + case errors.Is(err, sql.ErrNoRows): + case err != nil: + return false, fmt.Errorf("settings: get %s: %w", ingestKey, err) + default: + if err := json.Unmarshal([]byte(row.Value), ¤t); err != nil { + return false, fmt.Errorf("settings: decode %s: %w", ingestKey, err) + } + } + if current.TokenHash != "" { + return false, nil + } + + payload, err := json.Marshal(Ingest{TokenHash: HashIngestToken(token)}) + if err != nil { + return false, fmt.Errorf("settings: encode ingest: %w", err) + } + event := auth.AuditEvent{EventType: "ingest_key.rotated", Outcome: "success", TargetType: "ingest", TargetID: "default", Metadata: map[string]any{"source": "configuration"}} + if err := commitIngest(ctx, tx, payload, audit, event); err != nil { + return false, err + } + return true, nil +} + +func commitIngest(ctx context.Context, tx *sql.Tx, payload []byte, audit *auth.AuditStore, event auth.AuditEvent) error { + if _, err := generated.New(tx).UpsertSetting(ctx, generated.UpsertSettingParams{Key: ingestKey, Value: string(payload), UpdatedAt: time.Now().UTC().Format(time.RFC3339)}); err != nil { return fmt.Errorf("settings: upsert ingest: %w", err) } - if err := audit.RecordTx(writeCtx, tx, event); err != nil { + if err := audit.RecordTx(ctx, tx, event); err != nil { return err } if err := tx.Commit(); err != nil { diff --git a/internal/settings/store_test.go b/internal/settings/store_test.go index dfcd15c4b..fa9312f5a 100644 --- a/internal/settings/store_test.go +++ b/internal/settings/store_test.go @@ -100,3 +100,136 @@ func TestIngestRejectsRetiredTokenEvenWithMatchingDigest(t *testing.T) { t.Fatal("retired ingest token accepted") } } + +func configuredIngestToken(fill string) string { + return "foi_" + strings.Repeat(fill, 43) +} + +func ingestAuditCount(t *testing.T, store *Store) int { + t.Helper() + var count int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM auth_audit_events WHERE event_type = 'ingest_key.rotated'`).Scan(&count); err != nil { + t.Fatalf("count audit events: %v", err) + } + return count +} + +func TestApplyConfiguredIngestTokenStoresOnlyHashOnEmptyInstall(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + token := configuredIngestToken("A") + + applied, err := store.ApplyConfiguredIngestToken(ctx, token, auth.NewAuditStore(store.db)) + if err != nil { + t.Fatalf("ApplyConfiguredIngestToken: %v", err) + } + if !applied { + t.Fatal("token was not applied to an empty install") + } + got, err := store.GetIngest(ctx) + if err != nil { + t.Fatalf("GetIngest: %v", err) + } + if got.TokenHash != HashIngestToken(token) || !CheckIngestToken(token, got.TokenHash) { + t.Fatal("stored hash does not verify the configured token") + } + + var value, metadata string + if err := store.db.QueryRow(`SELECT value FROM settings WHERE key = ?`, ingestKey).Scan(&value); err != nil { + t.Fatalf("read setting: %v", err) + } + if err := store.db.QueryRow(`SELECT metadata_json FROM auth_audit_events WHERE event_type = 'ingest_key.rotated'`).Scan(&metadata); err != nil { + t.Fatalf("read audit event: %v", err) + } + if strings.Contains(value, token) || strings.Contains(metadata, token) { + t.Fatal("raw token was persisted") + } + if !strings.Contains(metadata, `"configuration"`) { + t.Fatalf("audit metadata = %s, want configuration source", metadata) + } +} + +func TestApplyConfiguredIngestTokenIgnoredOnceHashExists(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + audit := auth.NewAuditStore(store.db) + first := configuredIngestToken("A") + + if _, err := store.ApplyConfiguredIngestToken(ctx, first, audit); err != nil { + t.Fatalf("first apply: %v", err) + } + + // A later restart with a different value changes nothing. + applied, err := store.ApplyConfiguredIngestToken(ctx, configuredIngestToken("B"), audit) + if err != nil { + t.Fatalf("second apply: %v", err) + } + got, err := store.GetIngest(ctx) + if err != nil { + t.Fatalf("GetIngest: %v", err) + } + if applied || got.TokenHash != HashIngestToken(first) { + t.Fatal("a different configured value replaced the stored hash") + } + if count := ingestAuditCount(t, store); count != 1 { + t.Fatalf("audit events = %d, want 1", count) + } + + // Rotation from the UI survives restarts that still carry the setting. + _, rotated, err := GenerateIngestToken() + if err != nil { + t.Fatalf("GenerateIngestToken: %v", err) + } + if err := store.SetIngest(ctx, Ingest{TokenHash: rotated}); err != nil { + t.Fatalf("SetIngest: %v", err) + } + applied, err = store.ApplyConfiguredIngestToken(ctx, first, audit) + if err != nil { + t.Fatalf("apply after rotation: %v", err) + } + got, err = store.GetIngest(ctx) + if err != nil { + t.Fatalf("GetIngest: %v", err) + } + if applied || got.TokenHash != rotated { + t.Fatal("configured value replaced a rotated token") + } +} + +func TestApplyConfiguredIngestTokenFillsEmptyStoredHash(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + token := configuredIngestToken("A") + if err := store.SetIngest(ctx, Ingest{}); err != nil { + t.Fatalf("SetIngest: %v", err) + } + + applied, err := store.ApplyConfiguredIngestToken(ctx, token, auth.NewAuditStore(store.db)) + if err != nil { + t.Fatalf("ApplyConfiguredIngestToken: %v", err) + } + got, err := store.GetIngest(ctx) + if err != nil { + t.Fatalf("GetIngest: %v", err) + } + if !applied || got.TokenHash != HashIngestToken(token) { + t.Fatal("an empty stored hash must count as no token") + } +} + +func TestApplyConfiguredIngestTokenFailsOnCorruptSetting(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + token := configuredIngestToken("A") + if err := store.Upsert(ctx, ingestKey, "not an ingest object"); err != nil { + t.Fatalf("Upsert: %v", err) + } + + applied, err := store.ApplyConfiguredIngestToken(ctx, token, auth.NewAuditStore(store.db)) + if err == nil || applied || strings.Contains(err.Error(), token) { + t.Fatalf("applied = %v err = %v, want redacted failure", applied, err) + } + if count := ingestAuditCount(t, store); count != 0 { + t.Fatalf("audit events = %d, want 0", count) + } +} diff --git a/site/src/content/docs/docs/get-started/first-boot.mdx b/site/src/content/docs/docs/get-started/first-boot.mdx index 9a0c21b4b..f70ca44cc 100644 --- a/site/src/content/docs/docs/get-started/first-boot.mdx +++ b/site/src/content/docs/docs/get-started/first-boot.mdx @@ -43,7 +43,8 @@ docker logs fanout 2>&1 | grep -A8 "FANOUT SETUP" Open the URL, create the administrator, and **save the ingest token shown after setup**. The raw token is displayed once; Fanout stores only its hash and cannot -show it again. +show it again. If `FANOUT_INGEST_TOKEN` already set the token, setup shows no +token and leaves that one in place. An administrator can issue a replacement in **Connect telemetry → Ingest token** in the account menu, or with `POST /api/settings/ingest/token/rotate`. Save the new token when it is diff --git a/site/src/content/docs/docs/get-started/send-telemetry.mdx b/site/src/content/docs/docs/get-started/send-telemetry.mdx index 3fa1f78be..0f6431f16 100644 --- a/site/src/content/docs/docs/get-started/send-telemetry.mdx +++ b/site/src/content/docs/docs/get-started/send-telemetry.mdx @@ -25,14 +25,28 @@ with `POST /api/settings/ingest/token/rotate`. Save the new token when it is shown. Rotation invalidates the old token immediately and cuts off existing exporters until you update their credentials. +An automated install can accept telemetry as soon as it starts, before any +administrator exists. Generate a token in the same format and pass it as +`FANOUT_INGEST_TOKEN`, or `FANOUT_INGEST_TOKEN_FILE` for a mounted secret: + +```sh +printf 'foi_%s\n' "$(openssl rand -base64 32 | tr '+/' '-_' | tr -d '=')" +``` + +Fanout stores the token's hash at startup only when no token exists, and setup +then shows no token. Once a token exists the setting is ignored, so a token +rotated in the UI survives restarts and changing the setting replaces nothing. +A value in any other format fails startup with an error naming the setting. + Send it as a bearer token; this is the only accepted ingest credential form: ```text Authorization: Bearer foi_ ``` -Ingest is available after administrator setup. Exporters require the ingest -token; browser sessions and MCP OAuth tokens cannot authorize exports. +Ingest is available after administrator setup, or as soon as Fanout starts when +`FANOUT_INGEST_TOKEN` set the token. Exporters require the ingest token; browser +sessions and MCP OAuth tokens cannot authorize exports. ## Configure an SDK diff --git a/site/src/content/docs/docs/guides/deploy.mdx b/site/src/content/docs/docs/guides/deploy.mdx index a66c6e630..d77239ddc 100644 --- a/site/src/content/docs/docs/guides/deploy.mdx +++ b/site/src/content/docs/docs/guides/deploy.mdx @@ -47,9 +47,10 @@ The port mapping above exposes it only on the host's loopback interface. Use [TLS](/docs/guides/tls) and a restricted network route for remote clients. The ingest token is issued during administrator setup and stored in control -state. There is no ingest-token environment setting or `_FILE` form. Give the -issued token to your exporters through their secret delivery mechanism; see -[send telemetry](/docs/get-started/send-telemetry). +state as a hash. An automated install can set the first token instead with +`FANOUT_INGEST_TOKEN` or `FANOUT_INGEST_TOKEN_FILE`; see +[send telemetry](/docs/get-started/send-telemetry). Give the token to your +exporters through their secret delivery mechanism. A named volume preserves data across container replacement. Do not remove it when upgrading. The healthcheck marks an unhealthy container; Docker does not diff --git a/site/src/content/docs/docs/reference/configuration.mdx b/site/src/content/docs/docs/reference/configuration.mdx index 49cbf2abd..6474476ff 100644 --- a/site/src/content/docs/docs/reference/configuration.mdx +++ b/site/src/content/docs/docs/reference/configuration.mdx @@ -63,6 +63,7 @@ Every secret accepts either its value or its file path: | `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | | `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | | `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_INGEST_TOKEN` | `FANOUT_INGEST_TOKEN_FILE` | | `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | | `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | | `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | diff --git a/site/src/content/docs/docs/reference/configuration/ingest.mdx b/site/src/content/docs/docs/reference/configuration/ingest.mdx index ea76b34a4..022493fd7 100644 --- a/site/src/content/docs/docs/reference/configuration/ingest.mdx +++ b/site/src/content/docs/docs/reference/configuration/ingest.mdx @@ -21,6 +21,7 @@ Vendor credentials and endpoints keep the vendor's documented names. | `FANOUT_DEFAULT_NAMESPACE` | string | `default` | | `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | | `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` | integer | `268435456` | +| `FANOUT_INGEST_TOKEN` or `FANOUT_INGEST_TOKEN_FILE` | string · secret | — | ## Notes @@ -35,3 +36,13 @@ Sets the group-commit row target; valid values are 1–50000. Individual request ### `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` Caps decoded telemetry held across concurrent requests until durable publication. Zero is unbounded. A single request is admitted when no other request holds budget, even if it exceeds the limit. + +### `FANOUT_INGEST_TOKEN` + +Sets the first ingest token when none is stored, so an automated install accepts telemetry as soon as it starts. Use foi_ plus 43 base64url characters. Only its hash is stored. It is ignored once a token exists, so a token rotated in the UI survives restarts. + +:::caution[Secrets] +Settings marked `secret` are redacted from Fanout's startup configuration +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. +::: From e02745abffde9fe184ede81adddb6de96650a24a Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 8 Nov 2026 08:36:24 -0800 Subject: [PATCH 2/2] docs: note that a configured ingest token skips the setup display --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17d33d14e..61f5dd1be 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ docker run --name fanout --restart unless-stopped -p 7520:7520 \ ``` Open the one-time setup URL printed by the container and create the first administrator. Fanout -displays the ingest token exactly once; save it with your collector secrets. For later sign-in +displays the ingest token exactly once, unless `FANOUT_INGEST_TOKEN` already set it; save it with your collector secrets. For later sign-in without SMTP, mint a 15-minute, single-use link against the running container's control database: ```sh