diff --git a/.env.example b/.env.example index cd8a7b0a4..2fbe451d8 100644 --- a/.env.example +++ b/.env.example @@ -179,7 +179,7 @@ # FANOUT_OIDC_EMAIL_VERIFICATION= # FANOUT_OIDC_ISSUER_URL; string -# Selects the identity provider issuer; required in OIDC mode. +# Selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. # Default: (empty) # FANOUT_OIDC_ISSUER_URL= diff --git a/cmd/fanout-docgen/routes.go b/cmd/fanout-docgen/routes.go index b67e36eb8..f3c6552c8 100644 --- a/cmd/fanout-docgen/routes.go +++ b/cmd/fanout-docgen/routes.go @@ -258,7 +258,8 @@ func renderRoutes(routes []api.RouteDoc) []byte { b.WriteString("- `/api/agent/*`: only when `FANOUT_AI_PROVIDER` selects `anthropic` or\n") b.WriteString(" `openai`, with that provider's required key. An empty provider disables chat.\n") b.WriteString("- `/api/auth/oidc/start` and `/api/auth/oidc/callback`: only in\n") - b.WriteString(" `FANOUT_AUTH_MODE=oidc`.\n\n") + b.WriteString(" `FANOUT_AUTH_MODE=oidc`. They return 503 `unavailable` with `Retry-After`\n") + b.WriteString(" until provider discovery succeeds.\n\n") b.WriteString("Everything else is always registered. A path shown with `:name` or `*` is the\n") b.WriteString("pattern Echo matches on, not a literal URL.\n") b.WriteString("\n") diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 5946c821e..41fd3fff0 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -273,9 +273,6 @@ func main() { api.RegisterAuthMiddleware(e, userStore, browserSessions, auditStore, cfg) - // Health checks (liveness + readiness) - api.RegisterHealthRoutes(e, q, cfg) - // Prometheus metrics (internal/ops) e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) @@ -338,10 +335,15 @@ func main() { } codeStore := auth.NewCodeStore(sqlite.DB, cfg.AuthCodeSecret) api.RegisterAuthRoutes(e, userStore, codeStore, setup, settingsStore, browserSessions, auditStore, smtpCfg, cfg) - if err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore); err != nil { + oidcDiscovery, err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore) + if err != nil { slog.Error("OIDC initialization failed", "err", err) os.Exit(1) } + + // Health checks (liveness + readiness) + api.RegisterHealthRoutes(e, q, cfg, oidcDiscovery) + api.RegisterUserRoutes(e, userStore, smtpCfg, cfg) api.RegisterSettingsRoutes(e, cfg, settingsStore, auditStore) slog.Info("auth enabled") diff --git a/docs/authentication.md b/docs/authentication.md index 4f8df6248..3915da7d4 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -195,6 +195,24 @@ session cookies are issued with `Secure`. ## OIDC +### Provider availability + +Fanout runs OIDC discovery once at startup, for at most 5 seconds. If the +provider is unreachable or answers with an error, Fanout starts anyway and +serves ingest, MCP, and existing browser sessions. Discovery is retried in the +background, starting after 1 second and doubling to at most 1 minute between +attempts, until it succeeds or Fanout shuts down. No restart is needed. + +Until then `/api/auth/oidc/start` and `/api/auth/oidc/callback` return 503 with +`Retry-After`, and the `oidc` check in `/readyz` reports `degraded` while the +endpoint still returns 200. The cause is in the logs. + +Invalid configuration still fails startup: a value rejected by configuration +validation, or a provider that answers for an issuer other than +`FANOUT_OIDC_ISSUER_URL`. If a mismatch first appears on a background retry, +because the provider was down at startup, Fanout keeps running and retrying; +correct the issuer and restart. + ### Public and community access OIDC can provision a separate viewer account for every visitor whose provider diff --git a/docs/operations.md b/docs/operations.md index 57783a90d..fa9f89c7e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -204,7 +204,9 @@ proxy and certificate verification behaviour. engine reports `degraded` with 200; a stalled one reports 503. A query still running 25 seconds past its deadline reports `queries` as `degraded`, logs a warning with a hash of the query (never its text), and increments - `fanout_duckdb_statement_overruns_total`. + `fanout_duckdb_statement_overruns_total`. In OIDC mode, the `oidc` check + reports `degraded` with 200 while provider discovery is failing; only new + OIDC sign-ins are affected, and discovery retries without a restart. - `/metrics` exposes Prometheus metrics. Keep it private or configure `FANOUT_METRICS_TOKEN`; do not make it public merely to simplify scraping. diff --git a/internal/api/health.go b/internal/api/health.go index dfacc8feb..d627cb6c4 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -24,6 +24,8 @@ type HealthHandler struct { pool readPool cfg config.Config started time.Time + // oidc is nil outside OIDC mode. + oidc *OIDCDiscovery } type readPool interface { @@ -121,6 +123,9 @@ collect: resp.Checks["data"] = h.checkDataDir() resp.Checks["maintenance"] = h.checkMaintenance() resp.Checks["queries"] = h.checkQueries() + if h.oidc != nil { + resp.Checks["oidc"] = h.oidc.readiness() + } // Determine overall status hasUnhealthy := false @@ -411,8 +416,9 @@ FROM rollup_state`).Scan(&updatedAt, &cacheCount, &ageSeconds) } // RegisterHealthRoutes registers health check endpoints -func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config) { +func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config, oidc *OIDCDiscovery) { h := NewHealthHandler(duck, cfg) + h.oidc = oidc e.GET("/healthz", h.Liveness) e.GET("/readyz", h.Readiness) } diff --git a/internal/api/health_test.go b/internal/api/health_test.go index a7aea401a..26e883891 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -330,7 +330,7 @@ func TestCheckMaintenance_DegradedWhenNeverRanPastGrace(t *testing.T) { func TestRegisterHealthRoutes_RegistersReadiness(t *testing.T) { e := echo.New() - RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()}) + RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()}, nil) req := httptest.NewRequest(http.MethodGet, "/readyz", nil) rec := httptest.NewRecorder() diff --git a/internal/api/oidc.go b/internal/api/oidc.go index 85555bd58..e14751473 100644 --- a/internal/api/oidc.go +++ b/internal/api/oidc.go @@ -29,9 +29,7 @@ const oidcFlowTTL = 10 * time.Minute type OIDCHandler struct { cfg config.Config - provider *oidc.Provider - verifier *oidc.IDTokenVerifier - oauth oauth2.Config + discovery *OIDCDiscovery users *appauth.UserStore identities *appauth.IdentityStore sessions *appauth.BrowserSessions @@ -39,35 +37,67 @@ type OIDCHandler struct { limiter *appauth.KeyedLimiter } -func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) error { +// RegisterOIDCRoutes makes one bounded discovery attempt and registers the +// login routes. It fails only on invalid configuration; if the provider is +// merely unavailable the routes answer 503 while discovery retries until ctx +// ends. The returned discovery is nil outside OIDC mode. +func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) (*OIDCDiscovery, error) { + return registerOIDCRoutes(ctx, e, cfg, users, identities, sessions, audit, defaultOIDCDiscoveryTiming) +} + +func registerOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore, timing oidcDiscoveryTiming) (*OIDCDiscovery, error) { if strings.ToLower(strings.TrimSpace(cfg.AuthMode)) != "oidc" { - return nil + return nil, nil } if users == nil || identities == nil || sessions == nil || audit == nil { - return fmt.Errorf("OIDC dependencies are required") - } - provider, err := oidc.NewProvider(ctx, cfg.OIDCIssuerURL) - if err != nil { - return fmt.Errorf("discover OIDC provider: %w", err) + return nil, fmt.Errorf("OIDC dependencies are required") } callback := strings.TrimRight(cfg.PublicURL, "/") + "/api/auth/oidc/callback" - h := &OIDCHandler{ - cfg: cfg, provider: provider, - verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}), - oauth: oauth2.Config{ - ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret, - Endpoint: provider.Endpoint(), RedirectURL: callback, - Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, + discovery := &OIDCDiscovery{ + issuer: cfg.OIDCIssuerURL, timing: timing, done: make(chan struct{}), + build: func(provider *oidc.Provider) *oidcClient { + return &oidcClient{ + verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}), + oauth: oauth2.Config{ + ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret, + Endpoint: provider.Endpoint(), RedirectURL: callback, + Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, + }, + } }, + } + err := discovery.attempt(ctx) + if err != nil && oidcConfigurationInvalid(err) { + return nil, fmt.Errorf("discover OIDC provider: %w", err) + } + h := &OIDCHandler{ + cfg: cfg, discovery: discovery, users: users, identities: identities, sessions: sessions, audit: audit, limiter: appauth.NewKeyedLimiter(30, 15*time.Minute), } e.GET("/api/auth/oidc/start", h.Start) e.GET("/api/auth/oidc/callback", h.Callback) - return nil + if err != nil { + slog.Warn("OIDC discovery failed at startup; retrying in the background", "issuer", cfg.OIDCIssuerURL, "err", truncatedOIDCDiscoveryError(err)) + go discovery.run(ctx) + } else { + close(discovery.done) + } + return discovery, nil +} + +func oidcUnavailable(c *echo.Context) error { + c.Response().Header().Set("Retry-After", oidcDiscoveryRetryAfter) + return echo.NewHTTPError(http.StatusServiceUnavailable, "OIDC sign-in is temporarily unavailable") } func (h *OIDCHandler) Start(c *echo.Context) error { + // Ahead of the limiter: retries during an outage must not spend the + // sign-in budget that is needed once the provider is back. + client := h.discovery.client() + if client == nil { + return oidcUnavailable(c) + } if !h.limiter.Allow(c.RealIP()) { return rateLimited(c, 15*time.Minute) } @@ -93,7 +123,7 @@ func (h *OIDCHandler) Start(c *echo.Context) error { return echo.NewHTTPError(http.StatusInternalServerError, "failed to start login") } challenge := sha256.Sum256([]byte(verifier)) - redirect := h.oauth.AuthCodeURL(state, + redirect := client.oauth.AuthCodeURL(state, oidc.Nonce(nonce), oauth2.SetAuthURLParam("code_challenge", base64.RawURLEncoding.EncodeToString(challenge[:])), oauth2.SetAuthURLParam("code_challenge_method", "S256"), @@ -132,6 +162,10 @@ func (c oidcClaims) groupsUnreadable() bool { } func (h *OIDCHandler) Callback(c *echo.Context) error { + client := h.discovery.client() + if client == nil { + return oidcUnavailable(c) + } ctx := c.Request().Context() state, expectedNonce, pkceVerifier, returnTo := h.sessions.OIDCFlow(ctx) if state == "" || subtle.ConstantTimeCompare([]byte(state), []byte(c.QueryParam("state"))) != 1 { @@ -143,7 +177,7 @@ func (h *OIDCHandler) Callback(c *echo.Context) error { if providerError := c.QueryParam("error"); providerError != "" { return h.oidcDenied(c, "identity provider denied login") } - token, err := h.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier)) + token, err := client.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier)) if err != nil { return h.oidcDenied(c, "code exchange failed") } @@ -151,7 +185,7 @@ func (h *OIDCHandler) Callback(c *echo.Context) error { if !ok || rawIDToken == "" { return h.oidcDenied(c, "missing ID token") } - idToken, err := h.verifier.Verify(ctx, rawIDToken) + idToken, err := client.verifier.Verify(ctx, rawIDToken) if err != nil { return h.oidcDenied(c, "invalid ID token") } diff --git a/internal/api/oidc_discovery.go b/internal/api/oidc_discovery.go new file mode 100644 index 000000000..5b790bb9a --- /dev/null +++ b/internal/api/oidc_discovery.go @@ -0,0 +1,140 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/url" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "golang.org/x/oauth2" +) + +const ( + oidcDiscoveryAttemptTimeout = 5 * time.Second + oidcDiscoveryInitialBackoff = time.Second + oidcDiscoveryMaxBackoff = time.Minute + oidcDiscoveryLogErrorLimit = 256 +) + +// oidcDiscoveryRetryAfter tells a client when a login is next worth trying: +// no later than the longest wait between discovery attempts. +var oidcDiscoveryRetryAfter = strconv.Itoa(int(oidcDiscoveryMaxBackoff.Seconds())) + +type oidcDiscoveryTiming struct { + attempt, initial, max time.Duration +} + +var defaultOIDCDiscoveryTiming = oidcDiscoveryTiming{ + attempt: oidcDiscoveryAttemptTimeout, + initial: oidcDiscoveryInitialBackoff, + max: oidcDiscoveryMaxBackoff, +} + +// oidcClient is everything built from a discovered provider. +type oidcClient struct { + verifier *oidc.IDTokenVerifier + oauth oauth2.Config +} + +// oidcDiscoveryState is the outcome of the latest discovery attempt; client is +// nil until an attempt succeeds. +type oidcDiscoveryState struct { + client *oidcClient + at time.Time +} + +// OIDCDiscovery resolves the identity provider's metadata. An unreachable +// provider must not take ingest and existing sessions down with it, so a failed +// startup attempt is retried in the background while the login routes answer 503. +type OIDCDiscovery struct { + issuer string + timing oidcDiscoveryTiming + build func(*oidc.Provider) *oidcClient + state atomic.Pointer[oidcDiscoveryState] + done chan struct{} +} + +func (d *OIDCDiscovery) attempt(ctx context.Context) error { + attemptCtx, cancel := context.WithTimeout(ctx, d.timing.attempt) + defer cancel() + provider, err := oidc.NewProvider(attemptCtx, d.issuer) + if err != nil { + d.state.Store(&oidcDiscoveryState{at: time.Now()}) + return err + } + d.state.Store(&oidcDiscoveryState{client: d.build(provider), at: time.Now()}) + return nil +} + +// run retries until discovery succeeds or ctx ends. Only process startup can +// reject a configuration, so an issuer mismatch found here is retried and +// logged like any other failure rather than stopping a running instance. +func (d *OIDCDiscovery) run(ctx context.Context) { + defer close(d.done) + delay := d.timing.initial + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + err := d.attempt(ctx) + if err == nil { + slog.Info("OIDC discovery succeeded", "issuer", d.issuer) + return + } + if ctx.Err() != nil { + return + } + delay = nextOIDCDiscoveryBackoff(delay, d.timing.max) + slog.Warn("OIDC discovery failed; sign-in stays unavailable", "issuer", d.issuer, "retry_in", delay, "err", truncatedOIDCDiscoveryError(err)) + timer.Reset(delay) + } +} + +// truncatedOIDCDiscoveryError bounds what a failure adds to the log: go-oidc +// includes the provider's whole response body in its error. +func truncatedOIDCDiscoveryError(err error) string { + message := err.Error() + if len(message) <= oidcDiscoveryLogErrorLimit { + return message + } + return strings.ToValidUTF8(message[:oidcDiscoveryLogErrorLimit], "") + "..." +} + +func nextOIDCDiscoveryBackoff(delay, limit time.Duration) time.Duration { + return min(2*delay, limit) +} + +func (d *OIDCDiscovery) client() *oidcClient { + return d.state.Load().client +} + +// readiness never reports unhealthy: the instance still serves everything but +// new OIDC logins, and removing it from rotation would make the outage total. +// The endpoint is public and discovery errors can carry the provider's response +// body, so the cause stays in the log. +func (d *OIDCDiscovery) readiness() CheckResult { + state := d.state.Load() + result := CheckResult{Status: "ok", UpdatedAt: state.at.UTC().Format(time.RFC3339)} + if state.client == nil { + result.Status = "degraded" + result.Detail = "discovery failing; OIDC sign-in returns 503 until a retry succeeds" + } + return result +} + +// oidcConfigurationInvalid reports a failure that retrying cannot fix: the +// issuer cannot form a request, or the provider answered for a different issuer. +func oidcConfigurationInvalid(err error) bool { + var mismatch *oidc.IssuerMismatchError + var malformed *url.Error + return errors.As(err, &mismatch) || (errors.As(err, &malformed) && malformed.Op == "parse") +} diff --git a/internal/api/oidc_discovery_test.go b/internal/api/oidc_discovery_test.go new file mode 100644 index 000000000..bf95b4151 --- /dev/null +++ b/internal/api/oidc_discovery_test.go @@ -0,0 +1,521 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + "unicode/utf8" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/echo/v5" + + appauth "github.com/labstack/fanout/internal/auth" + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/engine" + appstore "github.com/labstack/fanout/internal/store" +) + +var fastOIDCDiscovery = oidcDiscoveryTiming{attempt: 5 * time.Second, initial: 5 * time.Millisecond, max: 20 * time.Millisecond} + +// oidcOutageBody stands in for whatever an identity provider or its proxy +// returns during an outage; none of it may reach the public readiness response. +const oidcOutageBody = "upstream-internal-detail" + +// flakyIdP fails discovery until up is set, counting discovery requests. +type flakyIdP struct { + *httptest.Server + up atomic.Bool + requests atomic.Int64 +} + +func newFlakyIdP(t *testing.T) *flakyIdP { + t.Helper() + idp := &flakyIdP{} + idp.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + idp.requests.Add(1) + if !idp.up.Load() { + http.Error(w, oidcOutageBody, http.StatusServiceUnavailable) + return + } + writeOIDCDiscoveryDocument(w, idp.URL) + })) + t.Cleanup(idp.Close) + return idp +} + +func writeOIDCDiscoveryDocument(w http.ResponseWriter, issuer string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "issuer": issuer, "authorization_endpoint": issuer + "/authorize", "token_endpoint": issuer + "/token", + "jwks_uri": issuer + "/keys", "response_types_supported": []string{"code"}, + "subject_types_supported": []string{"public"}, "id_token_signing_alg_values_supported": []string{"RS256"}, + }) +} + +type oidcDiscoveryFixture struct { + echo *echo.Echo + cfg config.Config + users *appauth.UserStore + sessions *appauth.BrowserSessions + audit *appauth.AuditStore + register func(context.Context, oidcDiscoveryTiming) (*OIDCDiscovery, error) +} + +func newOIDCDiscoveryFixture(t *testing.T, issuer string) *oidcDiscoveryFixture { + t.Helper() + db, err := appstore.NewSQLite(":memory:") + if err != nil { + t.Fatalf("NewSQLite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + f := &oidcDiscoveryFixture{ + echo: echo.New(), + cfg: config.Config{ + AuthMode: "oidc", OIDCIssuerURL: issuer, OIDCClientID: "fanout-client", OIDCClientSecret: "client-secret", + PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", + DataDir: os.TempDir(), + }, + users: appauth.NewUserStore(db.DB), + sessions: appauth.NewBrowserSessions(db.DB, 12*time.Hour, 7*24*time.Hour, false), + audit: appauth.NewAuditStore(db.DB), + } + f.echo.HTTPErrorHandler = HTTPErrorHandler + RegisterAuthMiddleware(f.echo, f.users, f.sessions, f.audit, f.cfg) + identities := appauth.NewIdentityStore(db.DB) + f.register = func(ctx context.Context, timing oidcDiscoveryTiming) (*OIDCDiscovery, error) { + return registerOIDCRoutes(ctx, f.echo, f.cfg, f.users, identities, f.sessions, f.audit, timing) + } + return f +} + +func (f *oidcDiscoveryFixture) get(target string) *httptest.ResponseRecorder { + recorder := httptest.NewRecorder() + f.echo.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, target, nil)) + return recorder +} + +func assertOIDCUnavailable(t *testing.T, recorder *httptest.ResponseRecorder) { + t.Helper() + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503: %s", recorder.Code, recorder.Body.String()) + } + if got := recorder.Header().Get("Retry-After"); got != "60" { + t.Fatalf("Retry-After = %q, want 60", got) + } + var body struct { + Code string `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil || body.Code != "unavailable" { + t.Fatalf("body = %s, err %v, want code unavailable", recorder.Body.String(), err) + } +} + +func readinessWithHealthyEngine(t *testing.T, discovery *OIDCDiscovery, cfg config.Config) (*httptest.ResponseRecorder, HealthResponse) { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.MatchExpectationsInOrder(false) + mock.ExpectQuery("SELECT 1$").WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1").WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery("MAX\\(updated_at\\)"). + WillReturnRows(sqlmock.NewRows([]string{"max", "count", "age_seconds"}).AddRow(time.Now().UTC(), 2, int64(30))) + + cfg.RollupInterval = time.Minute + e := echo.New() + RegisterHealthRoutes(e, &engine.Duck{DB: db}, cfg, discovery) + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + var resp HealthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal readiness: %v", err) + } + return recorder, resp +} + +func waitForOIDCDiscoveryExit(t *testing.T, discovery *OIDCDiscovery) { + t.Helper() + select { + case <-discovery.done: + case <-time.After(5 * time.Second): + t.Fatal("discovery retry loop did not stop") + } +} + +func TestOIDCDiscoveryRecoversWithoutRestart(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("startup failed while the provider was down: %v", err) + } + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/callback?code=c&state=s")) + + recorder, resp := readinessWithHealthyEngine(t, discovery, fixture.cfg) + if recorder.Code != http.StatusOK { + t.Fatalf("/readyz status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + if check := resp.Checks["oidc"]; check.Status != "degraded" || check.Detail == "" { + t.Fatalf("oidc check = %+v, want degraded with detail", check) + } + if strings.Contains(recorder.Body.String(), oidcOutageBody) || strings.Contains(recorder.Body.String(), idp.URL) { + t.Fatalf("public readiness exposes discovery failure: %s", recorder.Body.String()) + } + + idp.up.Store(true) + var redirect *httptest.ResponseRecorder + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if redirect = fixture.get("/api/auth/oidc/start"); redirect.Code != http.StatusServiceUnavailable { + break + } + } + if redirect.Code != http.StatusFound { + t.Fatalf("OIDC start after recovery = %d, want 302: %s", redirect.Code, redirect.Body.String()) + } + location, err := url.Parse(redirect.Header().Get("Location")) + if err != nil || !strings.HasPrefix(location.String(), idp.URL+"/authorize") || location.Query().Get("code_challenge") == "" { + t.Fatalf("authorization redirect = %q, err %v", redirect.Header().Get("Location"), err) + } + + waitForOIDCDiscoveryExit(t, discovery) + if _, resp = readinessWithHealthyEngine(t, discovery, fixture.cfg); resp.Checks["oidc"].Status != "ok" { + t.Fatalf("oidc check after recovery = %+v, want ok", resp.Checks["oidc"]) + } +} + +func TestOIDCDiscoveryOutageDoesNotConsumeSignInBudget(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for range 40 { + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + } + idp.up.Store(true) + waitForOIDCDiscoveryExit(t, discovery) + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusFound { + t.Fatalf("OIDC start after outage = %d, want 302: %s", got.Code, got.Body.String()) + } +} + +func TestOIDCDiscoverySuccessAtStartupServesImmediately(t *testing.T) { + idp := newFlakyIdP(t) + idp.up.Store(true) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + + discovery, err := fixture.register(context.Background(), fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusFound { + t.Fatalf("OIDC start = %d, want 302: %s", got.Code, got.Body.String()) + } + waitForOIDCDiscoveryExit(t, discovery) + if got := idp.requests.Load(); got != 1 { + t.Fatalf("discovery requests = %d, want 1", got) + } +} + +func TestOIDCDiscoveryRetriesStopAtShutdown(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for deadline := time.Now().Add(5 * time.Second); idp.requests.Load() < 3; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("discovery requests = %d, want retries", idp.requests.Load()) + } + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) + stopped := idp.requests.Load() + time.Sleep(10 * fastOIDCDiscovery.max) + if got := idp.requests.Load(); got != stopped { + t.Fatalf("discovery requests after shutdown = %d, want %d", got, stopped) + } +} + +func TestOIDCDiscoveryShutdownCancelsInFlightAttempt(t *testing.T) { + release := make(chan struct{}) + var requests atomic.Int64 + hanging := make(chan struct{}) + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + close(hanging) + <-release + })) + defer idp.Close() + defer close(release) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + timing := fastOIDCDiscovery + timing.attempt = time.Minute + discovery, err := fixture.register(ctx, timing) + if err != nil { + t.Fatalf("register: %v", err) + } + select { + case <-hanging: + case <-time.After(5 * time.Second): + t.Fatal("retry never reached the provider") + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) +} + +func TestOIDCDiscoveryStartupAttemptIsBounded(t *testing.T) { + release := make(chan struct{}) + idp := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-release })) + defer idp.Close() + defer close(release) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + timing := fastOIDCDiscovery + timing.attempt = 50 * time.Millisecond + finished := make(chan error, 1) + go func() { + _, err := fixture.register(ctx, timing) + finished <- err + }() + select { + case err := <-finished: + if err != nil { + t.Fatalf("hanging provider failed startup: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("startup blocked on a hanging provider") + } + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) +} + +func TestOIDCExistingSessionServedDuringDiscoveryOutage(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + user, err := fixture.users.CreateWithAudit("admin@example.com", "", "admin", appauth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatalf("create user: %v", err) + } + cookie := authenticatedSessionCookie(t, fixture.sessions, user) + fixture.echo.GET("/api/auth/me", func(c *echo.Context) error { return c.NoContent(http.StatusNoContent) }) + if _, err := fixture.register(ctx, fastOIDCDiscovery); err != nil { + t.Fatalf("register: %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "/api/auth/me", nil) + request.AddCookie(cookie) + recorder := httptest.NewRecorder() + fixture.echo.ServeHTTP(recorder, request) + if recorder.Code != http.StatusNoContent { + t.Fatalf("existing session = %d, want 204: %s", recorder.Code, recorder.Body.String()) + } +} + +func TestOIDCInvalidConfigurationFailsStartup(t *testing.T) { + wrongIssuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeOIDCDiscoveryDocument(w, "https://other.example.com") + })) + defer wrongIssuer.Close() + + for name, issuer := range map[string]string{ + "issuer mismatch": wrongIssuer.URL, + "unparseable issuer": " https://idp.example.com", + } { + t.Run(name, func(t *testing.T) { + fixture := newOIDCDiscoveryFixture(t, issuer) + discovery, err := fixture.register(context.Background(), fastOIDCDiscovery) + if err == nil { + t.Fatal("invalid OIDC configuration started") + } + if discovery != nil { + t.Fatal("failed startup returned a discovery") + } + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusNotFound { + t.Fatalf("OIDC start = %d, want unregistered", got.Code) + } + }) + } + + t.Run("missing dependencies", func(t *testing.T) { + cfg := config.Config{AuthMode: "oidc", OIDCIssuerURL: wrongIssuer.URL} + if _, err := RegisterOIDCRoutes(context.Background(), echo.New(), cfg, nil, nil, nil, nil); err == nil { + t.Fatal("missing dependencies started") + } + }) +} + +func TestOIDCDiscoveryLocalModeRegistersNothing(t *testing.T) { + e := echo.New() + discovery, err := RegisterOIDCRoutes(context.Background(), e, config.Config{AuthMode: "local"}, nil, nil, nil, nil) + if err != nil || discovery != nil { + t.Fatalf("local mode = %v, %v; want nil, nil", discovery, err) + } + _, resp := readinessWithHealthyEngine(t, nil, config.Config{DataDir: os.TempDir()}) + if _, reported := resp.Checks["oidc"]; reported { + t.Fatalf("local mode reports an oidc check: %+v", resp.Checks["oidc"]) + } +} + +func TestOIDCDiscoveryBackoffIsBounded(t *testing.T) { + want := []time.Duration{2, 4, 8, 16, 32, 60, 60, 60} + delay := oidcDiscoveryInitialBackoff + for i, seconds := range want { + delay = nextOIDCDiscoveryBackoff(delay, oidcDiscoveryMaxBackoff) + if delay != seconds*time.Second { + t.Fatalf("backoff %d = %v, want %v", i, delay, seconds*time.Second) + } + } +} + +type lockedLogBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func TestOIDCDiscoveryLogsTruncateProviderBody(t *testing.T) { + logs := &lockedLogBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + body := strings.Repeat("x", 1<<20) + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, body, http.StatusServiceUnavailable) + })) + defer idp.Close() + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for deadline := time.Now().Add(5 * time.Second); !strings.Contains(logs.String(), "sign-in stays unavailable"); time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("retry failure was never logged") + } + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) + + var failures int + for line := range strings.SplitSeq(logs.String(), "\n") { + if !strings.Contains(line, "OIDC discovery failed") { + continue + } + failures++ + if !strings.Contains(line, "503 Service Unavailable") { + t.Fatalf("log lost the failure status: %.400s", line) + } + if len(line) > 1024 { + t.Fatalf("log line is %d bytes, want the provider body truncated", len(line)) + } + } + if failures < 2 { + t.Fatalf("logged failures = %d, want the startup and retry sites", failures) + } +} + +func TestOIDCTruncatedDiscoveryErrorKeepsValidUTF8(t *testing.T) { + got := truncatedOIDCDiscoveryError(errors.New(strings.Repeat("é", 400))) + if !utf8.ValidString(got) || len(got) > oidcDiscoveryLogErrorLimit+len("...") { + t.Fatalf("truncated error is %d bytes, valid %v", len(got), utf8.ValidString(got)) + } + if short := truncatedOIDCDiscoveryError(errors.New("short")); short != "short" { + t.Fatalf("short error = %q", short) + } +} + +func TestOIDCDiscoveryWrongIssuerOnRetryStaysUnavailable(t *testing.T) { + var wrongIssuer atomic.Bool + var wrongAnswers atomic.Int64 + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !wrongIssuer.Load() { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + wrongAnswers.Add(1) + writeOIDCDiscoveryDocument(w, "https://other.example.com") + })) + defer idp.Close() + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("startup failed while the provider was down: %v", err) + } + wrongIssuer.Store(true) + for deadline := time.Now().Add(5 * time.Second); wrongAnswers.Load() < 3; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("wrong-issuer answers = %d, want retries to continue", wrongAnswers.Load()) + } + } + + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/callback?code=c&state=s")) + recorder, resp := readinessWithHealthyEngine(t, discovery, fixture.cfg) + if recorder.Code != http.StatusOK { + t.Fatalf("/readyz status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + if check := resp.Checks["oidc"]; check.Status != "degraded" { + t.Fatalf("oidc check = %+v, want degraded", check) + } + select { + case <-discovery.done: + t.Fatal("retry loop stopped on a wrong issuer") + default: + } +} diff --git a/internal/api/oidc_test.go b/internal/api/oidc_test.go index a0ffecee2..90a2bf311 100644 --- a/internal/api/oidc_test.go +++ b/internal/api/oidc_test.go @@ -204,7 +204,7 @@ func TestOIDCFlowValidatesPKCENonceAndCreatesSession(t *testing.T) { AuthMode: "oidc", OIDCIssuerURL: issuer, OIDCClientID: "fanout-client", OIDCClientSecret: "client-secret", PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", } - if err := RegisterOIDCRoutes(context.Background(), e, cfg, users, identities, sessions, appauth.NewAuditStore(db.DB)); err != nil { + if _, err := RegisterOIDCRoutes(context.Background(), e, cfg, users, identities, sessions, appauth.NewAuditStore(db.DB)); err != nil { t.Fatalf("RegisterOIDCRoutes: %v", err) } @@ -354,7 +354,7 @@ func runRejectedOIDCCallback(t *testing.T, mutate func(jwt.MapClaims), badKey, k PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", OIDCAutoProvision: false, } - if err := RegisterOIDCRoutes(context.Background(), e, cfg, users, appauth.NewIdentityStore(db.DB), sessions, appauth.NewAuditStore(db.DB)); err != nil { + if _, err := RegisterOIDCRoutes(context.Background(), e, cfg, users, appauth.NewIdentityStore(db.DB), sessions, appauth.NewAuditStore(db.DB)); err != nil { t.Fatal(err) } start := httptest.NewRecorder() diff --git a/internal/config/config.go b/internal/config/config.go index e27b6b3d3..bf4b84f5a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -177,7 +177,7 @@ type Config struct { SessionIdleTTL time.Duration `group:"auth" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` // SessionAbsoluteTTL limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. SessionAbsoluteTTL time.Duration `group:"auth" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` - // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. + // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. OIDCIssuerURL string `group:"auth" env:"FANOUT_OIDC_ISSUER_URL"` // OIDCClientID identifies Fanout's client registration; required in OIDC mode. OIDCClientID string `group:"auth" env:"FANOUT_OIDC_CLIENT_ID"` diff --git a/site/src/content/docs/docs/concepts/authentication.mdx b/site/src/content/docs/docs/concepts/authentication.mdx index cdceb77df..c0f12920c 100644 --- a/site/src/content/docs/docs/concepts/authentication.mdx +++ b/site/src/content/docs/docs/concepts/authentication.mdx @@ -37,6 +37,13 @@ on every login, including for linked identities. Removing a user's eligibility blocks their next login; it does not revoke a browser session already issued. Suspending the account in Fanout blocks access and revokes its sessions. +Fanout starts even when the identity provider is unreachable, so ingest, MCP +and existing sessions keep working. Discovery retries in the background with +backoff capped at 1 minute. Until it succeeds, the OIDC sign-in routes return +503 and the `oidc` check in `/readyz` reads `degraded`; no restart is needed. +Invalid configuration, including a provider that answers for a different +issuer at startup, still fails startup. + If neither `FANOUT_OIDC_ALLOWED_GROUPS` nor `FANOUT_OIDC_ALLOWED_DOMAINS` is configured, no eligibility allow policy is enforced; membership is managed locally. diff --git a/site/src/content/docs/docs/reference/api.mdx b/site/src/content/docs/docs/reference/api.mdx index dce248f61..2101702e0 100644 --- a/site/src/content/docs/docs/reference/api.mdx +++ b/site/src/content/docs/docs/reference/api.mdx @@ -28,7 +28,8 @@ configuration: - `/api/agent/*`: only when `FANOUT_AI_PROVIDER` selects `anthropic` or `openai`, with that provider's required key. An empty provider disables chat. - `/api/auth/oidc/start` and `/api/auth/oidc/callback`: only in - `FANOUT_AUTH_MODE=oidc`. + `FANOUT_AUTH_MODE=oidc`. They return 503 `unavailable` with `Retry-After` + until provider discovery succeeds. Everything else is always registered. A path shown with `:name` or `*` is the pattern Echo matches on, not a literal URL. diff --git a/site/src/content/docs/docs/reference/configuration/auth.mdx b/site/src/content/docs/docs/reference/configuration/auth.mdx index 506d6eef3..b8097c70a 100644 --- a/site/src/content/docs/docs/reference/configuration/auth.mdx +++ b/site/src/content/docs/docs/reference/configuration/auth.mdx @@ -83,7 +83,7 @@ Selects required verified-email claims or issuer trust; issuer trust requires an ### `FANOUT_OIDC_ISSUER_URL` -Selects the identity provider issuer; required in OIDC mode. +Selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. ### `FANOUT_OIDC_OPERATOR_GROUPS` diff --git a/site/src/content/docs/docs/reference/endpoints.mdx b/site/src/content/docs/docs/reference/endpoints.mdx index b0ef29b09..bc7348721 100644 --- a/site/src/content/docs/docs/reference/endpoints.mdx +++ b/site/src/content/docs/docs/reference/endpoints.mdx @@ -18,7 +18,7 @@ Everything below is served by the same HTTP listener as the browser client, on | Path | Requires | Reports | |---|---|---| | `/healthz` | nothing | Liveness — the process is up and its own work on the query engine is not stalled | -| `/readyz` | nothing | Readiness — storage is attached, plus resolved runtime sizing; a busy engine, or a query running past its deadline, reads `degraded` | +| `/readyz` | nothing | Readiness — storage is attached, plus resolved runtime sizing; a busy engine, a query running past its deadline, or failing OIDC discovery reads `degraded` | Use `/healthz` for restart decisions and `/readyz` for traffic decisions.