From 8ef5e622f060c0fb94b6940d4119447706e12e01 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 8 Nov 2026 08:28:42 -0800 Subject: [PATCH 1/2] feat(auth): start without the OIDC provider and retry discovery An unreachable identity provider no longer keeps Fanout down. One bounded discovery attempt runs at startup; on failure the instance starts, the OIDC login routes answer 503, and discovery retries with capped backoff until it succeeds or shutdown. Readiness reports an oidc check without failing. Invalid configuration, including an issuer mismatch at startup, still exits. --- .env.example | 2 +- cmd/fanout-docgen/routes.go | 3 +- cmd/fanout/main.go | 10 +- docs/authentication.md | 18 + docs/operations.md | 4 +- internal/api/health.go | 8 +- internal/api/health_test.go | 2 +- internal/api/oidc.go | 76 +++- internal/api/oidc_discovery.go | 128 ++++++ internal/api/oidc_discovery_test.go | 401 ++++++++++++++++++ internal/api/oidc_test.go | 4 +- internal/config/config.go | 2 +- .../docs/docs/concepts/authentication.mdx | 7 + site/src/content/docs/docs/reference/api.mdx | 3 +- .../docs/reference/configuration/auth.mdx | 2 +- .../content/docs/docs/reference/endpoints.mdx | 2 +- 16 files changed, 636 insertions(+), 36 deletions(-) create mode 100644 internal/api/oidc_discovery.go create mode 100644 internal/api/oidc_discovery_test.go diff --git a/.env.example b/.env.example index 0c173671e..660fe9b37 100644 --- a/.env.example +++ b/.env.example @@ -173,7 +173,7 @@ # FANOUT_OIDC_EMAIL_VERIFICATION= # FANOUT_OIDC_ISSUER_URL; string -# Selects the identity provider issuer; required in OIDC mode. +# Selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. # Default: (empty) # FANOUT_OIDC_ISSUER_URL= diff --git a/cmd/fanout-docgen/routes.go b/cmd/fanout-docgen/routes.go index b67e36eb8..f3c6552c8 100644 --- a/cmd/fanout-docgen/routes.go +++ b/cmd/fanout-docgen/routes.go @@ -258,7 +258,8 @@ func renderRoutes(routes []api.RouteDoc) []byte { b.WriteString("- `/api/agent/*`: only when `FANOUT_AI_PROVIDER` selects `anthropic` or\n") b.WriteString(" `openai`, with that provider's required key. An empty provider disables chat.\n") b.WriteString("- `/api/auth/oidc/start` and `/api/auth/oidc/callback`: only in\n") - b.WriteString(" `FANOUT_AUTH_MODE=oidc`.\n\n") + b.WriteString(" `FANOUT_AUTH_MODE=oidc`. They return 503 `unavailable` with `Retry-After`\n") + b.WriteString(" until provider discovery succeeds.\n\n") b.WriteString("Everything else is always registered. A path shown with `:name` or `*` is the\n") b.WriteString("pattern Echo matches on, not a literal URL.\n") b.WriteString("\n") diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 6e8c83453..43350f557 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -262,9 +262,6 @@ func main() { api.RegisterAuthMiddleware(e, userStore, browserSessions, auditStore, cfg) - // Health checks (liveness + readiness) - api.RegisterHealthRoutes(e, q, cfg) - // Prometheus metrics (internal/ops) e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) @@ -327,10 +324,15 @@ func main() { } codeStore := auth.NewCodeStore(sqlite.DB, cfg.AuthCodeSecret) api.RegisterAuthRoutes(e, userStore, codeStore, setup, settingsStore, browserSessions, auditStore, smtpCfg, cfg) - if err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore); err != nil { + oidcDiscovery, err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore) + if err != nil { slog.Error("OIDC initialization failed", "err", err) os.Exit(1) } + + // Health checks (liveness + readiness) + api.RegisterHealthRoutes(e, q, cfg, oidcDiscovery) + api.RegisterUserRoutes(e, userStore, smtpCfg, cfg) api.RegisterSettingsRoutes(e, cfg, settingsStore, auditStore) slog.Info("auth enabled") diff --git a/docs/authentication.md b/docs/authentication.md index 81e58cce3..02e1c9d72 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -162,6 +162,24 @@ session cookies are issued with `Secure`. ## OIDC +### Provider availability + +Fanout runs OIDC discovery once at startup, for at most 5 seconds. If the +provider is unreachable or answers with an error, Fanout starts anyway and +serves ingest, MCP, and existing browser sessions. Discovery is retried in the +background, starting after 1 second and doubling to at most 1 minute between +attempts, until it succeeds or Fanout shuts down. No restart is needed. + +Until then `/api/auth/oidc/start` and `/api/auth/oidc/callback` return 503 with +`Retry-After`, and the `oidc` check in `/readyz` reports `degraded` while the +endpoint still returns 200. The cause is in the logs. + +Invalid configuration still fails startup: a value rejected by configuration +validation, or a provider that answers for an issuer other than +`FANOUT_OIDC_ISSUER_URL`. If a mismatch first appears on a background retry, +because the provider was down at startup, Fanout keeps running and retrying; +correct the issuer and restart. + ### Public and community access OIDC can provision a separate viewer account for every visitor whose provider diff --git a/docs/operations.md b/docs/operations.md index 83a637303..c99407469 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -202,7 +202,9 @@ proxy and certificate verification behaviour. engine reports `degraded` with 200; a stalled one reports 503. A query still running 25 seconds past its deadline reports `queries` as `degraded`, logs a warning with a hash of the query (never its text), and increments - `fanout_duckdb_statement_overruns_total`. + `fanout_duckdb_statement_overruns_total`. In OIDC mode, the `oidc` check + reports `degraded` with 200 while provider discovery is failing; only new + OIDC sign-ins are affected, and discovery retries without a restart. - `/metrics` exposes Prometheus metrics. Keep it private or configure `FANOUT_METRICS_TOKEN`; do not make it public merely to simplify scraping. diff --git a/internal/api/health.go b/internal/api/health.go index dfacc8feb..d627cb6c4 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -24,6 +24,8 @@ type HealthHandler struct { pool readPool cfg config.Config started time.Time + // oidc is nil outside OIDC mode. + oidc *OIDCDiscovery } type readPool interface { @@ -121,6 +123,9 @@ collect: resp.Checks["data"] = h.checkDataDir() resp.Checks["maintenance"] = h.checkMaintenance() resp.Checks["queries"] = h.checkQueries() + if h.oidc != nil { + resp.Checks["oidc"] = h.oidc.readiness() + } // Determine overall status hasUnhealthy := false @@ -411,8 +416,9 @@ FROM rollup_state`).Scan(&updatedAt, &cacheCount, &ageSeconds) } // RegisterHealthRoutes registers health check endpoints -func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config) { +func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config, oidc *OIDCDiscovery) { h := NewHealthHandler(duck, cfg) + h.oidc = oidc e.GET("/healthz", h.Liveness) e.GET("/readyz", h.Readiness) } diff --git a/internal/api/health_test.go b/internal/api/health_test.go index a7aea401a..26e883891 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -330,7 +330,7 @@ func TestCheckMaintenance_DegradedWhenNeverRanPastGrace(t *testing.T) { func TestRegisterHealthRoutes_RegistersReadiness(t *testing.T) { e := echo.New() - RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()}) + RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()}, nil) req := httptest.NewRequest(http.MethodGet, "/readyz", nil) rec := httptest.NewRecorder() diff --git a/internal/api/oidc.go b/internal/api/oidc.go index 85555bd58..9915f4473 100644 --- a/internal/api/oidc.go +++ b/internal/api/oidc.go @@ -29,9 +29,7 @@ const oidcFlowTTL = 10 * time.Minute type OIDCHandler struct { cfg config.Config - provider *oidc.Provider - verifier *oidc.IDTokenVerifier - oauth oauth2.Config + discovery *OIDCDiscovery users *appauth.UserStore identities *appauth.IdentityStore sessions *appauth.BrowserSessions @@ -39,35 +37,67 @@ type OIDCHandler struct { limiter *appauth.KeyedLimiter } -func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) error { +// RegisterOIDCRoutes makes one bounded discovery attempt and registers the +// login routes. It fails only on invalid configuration; if the provider is +// merely unavailable the routes answer 503 while discovery retries until ctx +// ends. The returned discovery is nil outside OIDC mode. +func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) (*OIDCDiscovery, error) { + return registerOIDCRoutes(ctx, e, cfg, users, identities, sessions, audit, defaultOIDCDiscoveryTiming) +} + +func registerOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore, timing oidcDiscoveryTiming) (*OIDCDiscovery, error) { if strings.ToLower(strings.TrimSpace(cfg.AuthMode)) != "oidc" { - return nil + return nil, nil } if users == nil || identities == nil || sessions == nil || audit == nil { - return fmt.Errorf("OIDC dependencies are required") - } - provider, err := oidc.NewProvider(ctx, cfg.OIDCIssuerURL) - if err != nil { - return fmt.Errorf("discover OIDC provider: %w", err) + return nil, fmt.Errorf("OIDC dependencies are required") } callback := strings.TrimRight(cfg.PublicURL, "/") + "/api/auth/oidc/callback" - h := &OIDCHandler{ - cfg: cfg, provider: provider, - verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}), - oauth: oauth2.Config{ - ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret, - Endpoint: provider.Endpoint(), RedirectURL: callback, - Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, + discovery := &OIDCDiscovery{ + issuer: cfg.OIDCIssuerURL, timing: timing, done: make(chan struct{}), + build: func(provider *oidc.Provider) *oidcClient { + return &oidcClient{ + verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}), + oauth: oauth2.Config{ + ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret, + Endpoint: provider.Endpoint(), RedirectURL: callback, + Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, + }, + } }, + } + err := discovery.attempt(ctx) + if err != nil && oidcConfigurationInvalid(err) { + return nil, fmt.Errorf("discover OIDC provider: %w", err) + } + h := &OIDCHandler{ + cfg: cfg, discovery: discovery, users: users, identities: identities, sessions: sessions, audit: audit, limiter: appauth.NewKeyedLimiter(30, 15*time.Minute), } e.GET("/api/auth/oidc/start", h.Start) e.GET("/api/auth/oidc/callback", h.Callback) - return nil + if err != nil { + slog.Warn("OIDC discovery failed at startup; retrying in the background", "issuer", cfg.OIDCIssuerURL, "err", err) + go discovery.run(ctx) + } else { + close(discovery.done) + } + return discovery, nil +} + +func oidcUnavailable(c *echo.Context) error { + c.Response().Header().Set("Retry-After", oidcDiscoveryRetryAfter) + return echo.NewHTTPError(http.StatusServiceUnavailable, "OIDC sign-in is temporarily unavailable") } func (h *OIDCHandler) Start(c *echo.Context) error { + // Ahead of the limiter: retries during an outage must not spend the + // sign-in budget that is needed once the provider is back. + client := h.discovery.client() + if client == nil { + return oidcUnavailable(c) + } if !h.limiter.Allow(c.RealIP()) { return rateLimited(c, 15*time.Minute) } @@ -93,7 +123,7 @@ func (h *OIDCHandler) Start(c *echo.Context) error { return echo.NewHTTPError(http.StatusInternalServerError, "failed to start login") } challenge := sha256.Sum256([]byte(verifier)) - redirect := h.oauth.AuthCodeURL(state, + redirect := client.oauth.AuthCodeURL(state, oidc.Nonce(nonce), oauth2.SetAuthURLParam("code_challenge", base64.RawURLEncoding.EncodeToString(challenge[:])), oauth2.SetAuthURLParam("code_challenge_method", "S256"), @@ -132,6 +162,10 @@ func (c oidcClaims) groupsUnreadable() bool { } func (h *OIDCHandler) Callback(c *echo.Context) error { + client := h.discovery.client() + if client == nil { + return oidcUnavailable(c) + } ctx := c.Request().Context() state, expectedNonce, pkceVerifier, returnTo := h.sessions.OIDCFlow(ctx) if state == "" || subtle.ConstantTimeCompare([]byte(state), []byte(c.QueryParam("state"))) != 1 { @@ -143,7 +177,7 @@ func (h *OIDCHandler) Callback(c *echo.Context) error { if providerError := c.QueryParam("error"); providerError != "" { return h.oidcDenied(c, "identity provider denied login") } - token, err := h.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier)) + token, err := client.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier)) if err != nil { return h.oidcDenied(c, "code exchange failed") } @@ -151,7 +185,7 @@ func (h *OIDCHandler) Callback(c *echo.Context) error { if !ok || rawIDToken == "" { return h.oidcDenied(c, "missing ID token") } - idToken, err := h.verifier.Verify(ctx, rawIDToken) + idToken, err := client.verifier.Verify(ctx, rawIDToken) if err != nil { return h.oidcDenied(c, "invalid ID token") } diff --git a/internal/api/oidc_discovery.go b/internal/api/oidc_discovery.go new file mode 100644 index 000000000..1ef74eb5b --- /dev/null +++ b/internal/api/oidc_discovery.go @@ -0,0 +1,128 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/url" + "strconv" + "sync/atomic" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "golang.org/x/oauth2" +) + +const ( + oidcDiscoveryAttemptTimeout = 5 * time.Second + oidcDiscoveryInitialBackoff = time.Second + oidcDiscoveryMaxBackoff = time.Minute +) + +// oidcDiscoveryRetryAfter tells a client when a login is next worth trying: +// no later than the longest wait between discovery attempts. +var oidcDiscoveryRetryAfter = strconv.Itoa(int(oidcDiscoveryMaxBackoff.Seconds())) + +type oidcDiscoveryTiming struct { + attempt, initial, max time.Duration +} + +var defaultOIDCDiscoveryTiming = oidcDiscoveryTiming{ + attempt: oidcDiscoveryAttemptTimeout, + initial: oidcDiscoveryInitialBackoff, + max: oidcDiscoveryMaxBackoff, +} + +// oidcClient is everything built from a discovered provider. +type oidcClient struct { + verifier *oidc.IDTokenVerifier + oauth oauth2.Config +} + +// oidcDiscoveryState is the outcome of the latest discovery attempt; client is +// nil until an attempt succeeds. +type oidcDiscoveryState struct { + client *oidcClient + at time.Time +} + +// OIDCDiscovery resolves the identity provider's metadata. An unreachable +// provider must not take ingest and existing sessions down with it, so a failed +// startup attempt is retried in the background while the login routes answer 503. +type OIDCDiscovery struct { + issuer string + timing oidcDiscoveryTiming + build func(*oidc.Provider) *oidcClient + state atomic.Pointer[oidcDiscoveryState] + done chan struct{} +} + +func (d *OIDCDiscovery) attempt(ctx context.Context) error { + attemptCtx, cancel := context.WithTimeout(ctx, d.timing.attempt) + defer cancel() + provider, err := oidc.NewProvider(attemptCtx, d.issuer) + if err != nil { + d.state.Store(&oidcDiscoveryState{at: time.Now()}) + return err + } + d.state.Store(&oidcDiscoveryState{client: d.build(provider), at: time.Now()}) + return nil +} + +// run retries until discovery succeeds or ctx ends. Only process startup can +// reject a configuration, so an issuer mismatch found here is retried and +// logged like any other failure rather than stopping a running instance. +func (d *OIDCDiscovery) run(ctx context.Context) { + defer close(d.done) + delay := d.timing.initial + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + err := d.attempt(ctx) + if err == nil { + slog.Info("OIDC discovery succeeded", "issuer", d.issuer) + return + } + if ctx.Err() != nil { + return + } + delay = nextOIDCDiscoveryBackoff(delay, d.timing.max) + slog.Warn("OIDC discovery failed; sign-in stays unavailable", "issuer", d.issuer, "retry_in", delay, "err", err) + timer.Reset(delay) + } +} + +func nextOIDCDiscoveryBackoff(delay, limit time.Duration) time.Duration { + return min(2*delay, limit) +} + +func (d *OIDCDiscovery) client() *oidcClient { + return d.state.Load().client +} + +// readiness never reports unhealthy: the instance still serves everything but +// new OIDC logins, and removing it from rotation would make the outage total. +// The endpoint is public and discovery errors can carry the provider's response +// body, so the cause stays in the log. +func (d *OIDCDiscovery) readiness() CheckResult { + state := d.state.Load() + result := CheckResult{Status: "ok", UpdatedAt: state.at.UTC().Format(time.RFC3339)} + if state.client == nil { + result.Status = "degraded" + result.Detail = "discovery failing; OIDC sign-in returns 503 until a retry succeeds" + } + return result +} + +// oidcConfigurationInvalid reports a failure that retrying cannot fix: the +// issuer cannot form a request, or the provider answered for a different issuer. +func oidcConfigurationInvalid(err error) bool { + var mismatch *oidc.IssuerMismatchError + var malformed *url.Error + return errors.As(err, &mismatch) || (errors.As(err, &malformed) && malformed.Op == "parse") +} diff --git a/internal/api/oidc_discovery_test.go b/internal/api/oidc_discovery_test.go new file mode 100644 index 000000000..458c56fbc --- /dev/null +++ b/internal/api/oidc_discovery_test.go @@ -0,0 +1,401 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/echo/v5" + + appauth "github.com/labstack/fanout/internal/auth" + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/engine" + appstore "github.com/labstack/fanout/internal/store" +) + +var fastOIDCDiscovery = oidcDiscoveryTiming{attempt: 5 * time.Second, initial: 5 * time.Millisecond, max: 20 * time.Millisecond} + +// oidcOutageBody stands in for whatever an identity provider or its proxy +// returns during an outage; none of it may reach the public readiness response. +const oidcOutageBody = "upstream-internal-detail" + +// flakyIdP fails discovery until up is set, counting discovery requests. +type flakyIdP struct { + *httptest.Server + up atomic.Bool + requests atomic.Int64 +} + +func newFlakyIdP(t *testing.T) *flakyIdP { + t.Helper() + idp := &flakyIdP{} + idp.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + idp.requests.Add(1) + if !idp.up.Load() { + http.Error(w, oidcOutageBody, http.StatusServiceUnavailable) + return + } + writeOIDCDiscoveryDocument(w, idp.URL) + })) + t.Cleanup(idp.Close) + return idp +} + +func writeOIDCDiscoveryDocument(w http.ResponseWriter, issuer string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "issuer": issuer, "authorization_endpoint": issuer + "/authorize", "token_endpoint": issuer + "/token", + "jwks_uri": issuer + "/keys", "response_types_supported": []string{"code"}, + "subject_types_supported": []string{"public"}, "id_token_signing_alg_values_supported": []string{"RS256"}, + }) +} + +type oidcDiscoveryFixture struct { + echo *echo.Echo + cfg config.Config + users *appauth.UserStore + sessions *appauth.BrowserSessions + audit *appauth.AuditStore + register func(context.Context, oidcDiscoveryTiming) (*OIDCDiscovery, error) +} + +func newOIDCDiscoveryFixture(t *testing.T, issuer string) *oidcDiscoveryFixture { + t.Helper() + db, err := appstore.NewSQLite(":memory:") + if err != nil { + t.Fatalf("NewSQLite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + f := &oidcDiscoveryFixture{ + echo: echo.New(), + cfg: config.Config{ + AuthMode: "oidc", OIDCIssuerURL: issuer, OIDCClientID: "fanout-client", OIDCClientSecret: "client-secret", + PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", + DataDir: os.TempDir(), + }, + users: appauth.NewUserStore(db.DB), + sessions: appauth.NewBrowserSessions(db.DB, 12*time.Hour, 7*24*time.Hour, false), + audit: appauth.NewAuditStore(db.DB), + } + f.echo.HTTPErrorHandler = HTTPErrorHandler + RegisterAuthMiddleware(f.echo, f.users, f.sessions, f.audit, f.cfg) + identities := appauth.NewIdentityStore(db.DB) + f.register = func(ctx context.Context, timing oidcDiscoveryTiming) (*OIDCDiscovery, error) { + return registerOIDCRoutes(ctx, f.echo, f.cfg, f.users, identities, f.sessions, f.audit, timing) + } + return f +} + +func (f *oidcDiscoveryFixture) get(target string) *httptest.ResponseRecorder { + recorder := httptest.NewRecorder() + f.echo.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, target, nil)) + return recorder +} + +func assertOIDCUnavailable(t *testing.T, recorder *httptest.ResponseRecorder) { + t.Helper() + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503: %s", recorder.Code, recorder.Body.String()) + } + if got := recorder.Header().Get("Retry-After"); got != "60" { + t.Fatalf("Retry-After = %q, want 60", got) + } + var body struct { + Code string `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil || body.Code != "unavailable" { + t.Fatalf("body = %s, err %v, want code unavailable", recorder.Body.String(), err) + } +} + +func readinessWithHealthyEngine(t *testing.T, discovery *OIDCDiscovery, cfg config.Config) (*httptest.ResponseRecorder, HealthResponse) { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.MatchExpectationsInOrder(false) + mock.ExpectQuery("SELECT 1$").WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1").WillReturnRows(sqlmock.NewRows([]string{"1"})) + mock.ExpectQuery("MAX\\(updated_at\\)"). + WillReturnRows(sqlmock.NewRows([]string{"max", "count", "age_seconds"}).AddRow(time.Now().UTC(), 2, int64(30))) + + cfg.RollupInterval = time.Minute + e := echo.New() + RegisterHealthRoutes(e, &engine.Duck{DB: db}, cfg, discovery) + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + var resp HealthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal readiness: %v", err) + } + return recorder, resp +} + +func waitForOIDCDiscoveryExit(t *testing.T, discovery *OIDCDiscovery) { + t.Helper() + select { + case <-discovery.done: + case <-time.After(5 * time.Second): + t.Fatal("discovery retry loop did not stop") + } +} + +func TestOIDCDiscoveryRecoversWithoutRestart(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("startup failed while the provider was down: %v", err) + } + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/callback?code=c&state=s")) + + recorder, resp := readinessWithHealthyEngine(t, discovery, fixture.cfg) + if recorder.Code != http.StatusOK { + t.Fatalf("/readyz status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + if check := resp.Checks["oidc"]; check.Status != "degraded" || check.Detail == "" { + t.Fatalf("oidc check = %+v, want degraded with detail", check) + } + if strings.Contains(recorder.Body.String(), oidcOutageBody) || strings.Contains(recorder.Body.String(), idp.URL) { + t.Fatalf("public readiness exposes discovery failure: %s", recorder.Body.String()) + } + + idp.up.Store(true) + var redirect *httptest.ResponseRecorder + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if redirect = fixture.get("/api/auth/oidc/start"); redirect.Code != http.StatusServiceUnavailable { + break + } + } + if redirect.Code != http.StatusFound { + t.Fatalf("OIDC start after recovery = %d, want 302: %s", redirect.Code, redirect.Body.String()) + } + location, err := url.Parse(redirect.Header().Get("Location")) + if err != nil || !strings.HasPrefix(location.String(), idp.URL+"/authorize") || location.Query().Get("code_challenge") == "" { + t.Fatalf("authorization redirect = %q, err %v", redirect.Header().Get("Location"), err) + } + + waitForOIDCDiscoveryExit(t, discovery) + if _, resp = readinessWithHealthyEngine(t, discovery, fixture.cfg); resp.Checks["oidc"].Status != "ok" { + t.Fatalf("oidc check after recovery = %+v, want ok", resp.Checks["oidc"]) + } +} + +func TestOIDCDiscoveryOutageDoesNotConsumeSignInBudget(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for range 40 { + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + } + idp.up.Store(true) + waitForOIDCDiscoveryExit(t, discovery) + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusFound { + t.Fatalf("OIDC start after outage = %d, want 302: %s", got.Code, got.Body.String()) + } +} + +func TestOIDCDiscoverySuccessAtStartupServesImmediately(t *testing.T) { + idp := newFlakyIdP(t) + idp.up.Store(true) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + + discovery, err := fixture.register(context.Background(), fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusFound { + t.Fatalf("OIDC start = %d, want 302: %s", got.Code, got.Body.String()) + } + waitForOIDCDiscoveryExit(t, discovery) + if got := idp.requests.Load(); got != 1 { + t.Fatalf("discovery requests = %d, want 1", got) + } +} + +func TestOIDCDiscoveryRetriesStopAtShutdown(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for deadline := time.Now().Add(5 * time.Second); idp.requests.Load() < 3; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("discovery requests = %d, want retries", idp.requests.Load()) + } + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) + stopped := idp.requests.Load() + time.Sleep(10 * fastOIDCDiscovery.max) + if got := idp.requests.Load(); got != stopped { + t.Fatalf("discovery requests after shutdown = %d, want %d", got, stopped) + } +} + +func TestOIDCDiscoveryShutdownCancelsInFlightAttempt(t *testing.T) { + release := make(chan struct{}) + var requests atomic.Int64 + hanging := make(chan struct{}) + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + close(hanging) + <-release + })) + defer idp.Close() + defer close(release) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + timing := fastOIDCDiscovery + timing.attempt = time.Minute + discovery, err := fixture.register(ctx, timing) + if err != nil { + t.Fatalf("register: %v", err) + } + select { + case <-hanging: + case <-time.After(5 * time.Second): + t.Fatal("retry never reached the provider") + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) +} + +func TestOIDCDiscoveryStartupAttemptIsBounded(t *testing.T) { + release := make(chan struct{}) + idp := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-release })) + defer idp.Close() + defer close(release) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + timing := fastOIDCDiscovery + timing.attempt = 50 * time.Millisecond + finished := make(chan error, 1) + go func() { + _, err := fixture.register(ctx, timing) + finished <- err + }() + select { + case err := <-finished: + if err != nil { + t.Fatalf("hanging provider failed startup: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("startup blocked on a hanging provider") + } + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) +} + +func TestOIDCExistingSessionServedDuringDiscoveryOutage(t *testing.T) { + idp := newFlakyIdP(t) + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + user, err := fixture.users.CreateWithAudit("admin@example.com", "", "admin", appauth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatalf("create user: %v", err) + } + cookie := authenticatedSessionCookie(t, fixture.sessions, user) + fixture.echo.GET("/api/auth/me", func(c *echo.Context) error { return c.NoContent(http.StatusNoContent) }) + if _, err := fixture.register(ctx, fastOIDCDiscovery); err != nil { + t.Fatalf("register: %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "/api/auth/me", nil) + request.AddCookie(cookie) + recorder := httptest.NewRecorder() + fixture.echo.ServeHTTP(recorder, request) + if recorder.Code != http.StatusNoContent { + t.Fatalf("existing session = %d, want 204: %s", recorder.Code, recorder.Body.String()) + } +} + +func TestOIDCInvalidConfigurationFailsStartup(t *testing.T) { + wrongIssuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeOIDCDiscoveryDocument(w, "https://other.example.com") + })) + defer wrongIssuer.Close() + + for name, issuer := range map[string]string{ + "issuer mismatch": wrongIssuer.URL, + "unparseable issuer": " https://idp.example.com", + } { + t.Run(name, func(t *testing.T) { + fixture := newOIDCDiscoveryFixture(t, issuer) + discovery, err := fixture.register(context.Background(), fastOIDCDiscovery) + if err == nil { + t.Fatal("invalid OIDC configuration started") + } + if discovery != nil { + t.Fatal("failed startup returned a discovery") + } + if got := fixture.get("/api/auth/oidc/start"); got.Code != http.StatusNotFound { + t.Fatalf("OIDC start = %d, want unregistered", got.Code) + } + }) + } + + t.Run("missing dependencies", func(t *testing.T) { + cfg := config.Config{AuthMode: "oidc", OIDCIssuerURL: wrongIssuer.URL} + if _, err := RegisterOIDCRoutes(context.Background(), echo.New(), cfg, nil, nil, nil, nil); err == nil { + t.Fatal("missing dependencies started") + } + }) +} + +func TestOIDCDiscoveryLocalModeRegistersNothing(t *testing.T) { + e := echo.New() + discovery, err := RegisterOIDCRoutes(context.Background(), e, config.Config{AuthMode: "local"}, nil, nil, nil, nil) + if err != nil || discovery != nil { + t.Fatalf("local mode = %v, %v; want nil, nil", discovery, err) + } + _, resp := readinessWithHealthyEngine(t, nil, config.Config{DataDir: os.TempDir()}) + if _, reported := resp.Checks["oidc"]; reported { + t.Fatalf("local mode reports an oidc check: %+v", resp.Checks["oidc"]) + } +} + +func TestOIDCDiscoveryBackoffIsBounded(t *testing.T) { + want := []time.Duration{2, 4, 8, 16, 32, 60, 60, 60} + delay := oidcDiscoveryInitialBackoff + for i, seconds := range want { + delay = nextOIDCDiscoveryBackoff(delay, oidcDiscoveryMaxBackoff) + if delay != seconds*time.Second { + t.Fatalf("backoff %d = %v, want %v", i, delay, seconds*time.Second) + } + } +} diff --git a/internal/api/oidc_test.go b/internal/api/oidc_test.go index a0ffecee2..90a2bf311 100644 --- a/internal/api/oidc_test.go +++ b/internal/api/oidc_test.go @@ -204,7 +204,7 @@ func TestOIDCFlowValidatesPKCENonceAndCreatesSession(t *testing.T) { AuthMode: "oidc", OIDCIssuerURL: issuer, OIDCClientID: "fanout-client", OIDCClientSecret: "client-secret", PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", } - if err := RegisterOIDCRoutes(context.Background(), e, cfg, users, identities, sessions, appauth.NewAuditStore(db.DB)); err != nil { + if _, err := RegisterOIDCRoutes(context.Background(), e, cfg, users, identities, sessions, appauth.NewAuditStore(db.DB)); err != nil { t.Fatalf("RegisterOIDCRoutes: %v", err) } @@ -354,7 +354,7 @@ func runRejectedOIDCCallback(t *testing.T, mutate func(jwt.MapClaims), badKey, k PublicURL: "https://fanout.example.com", OIDCEmailVerification: "required", OIDCDefaultRole: "viewer", OIDCAutoProvision: false, } - if err := RegisterOIDCRoutes(context.Background(), e, cfg, users, appauth.NewIdentityStore(db.DB), sessions, appauth.NewAuditStore(db.DB)); err != nil { + if _, err := RegisterOIDCRoutes(context.Background(), e, cfg, users, appauth.NewIdentityStore(db.DB), sessions, appauth.NewAuditStore(db.DB)); err != nil { t.Fatal(err) } start := httptest.NewRecorder() diff --git a/internal/config/config.go b/internal/config/config.go index 74f2f5e42..62a2d9dba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -171,7 +171,7 @@ type Config struct { SessionIdleTTL time.Duration `group:"auth" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` // SessionAbsoluteTTL limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. SessionAbsoluteTTL time.Duration `group:"auth" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` - // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. + // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. OIDCIssuerURL string `group:"auth" env:"FANOUT_OIDC_ISSUER_URL"` // OIDCClientID identifies Fanout's client registration; required in OIDC mode. OIDCClientID string `group:"auth" env:"FANOUT_OIDC_CLIENT_ID"` diff --git a/site/src/content/docs/docs/concepts/authentication.mdx b/site/src/content/docs/docs/concepts/authentication.mdx index 0d9517660..666270cf3 100644 --- a/site/src/content/docs/docs/concepts/authentication.mdx +++ b/site/src/content/docs/docs/concepts/authentication.mdx @@ -37,6 +37,13 @@ on every login, including for linked identities. Removing a user's eligibility blocks their next login; it does not revoke a browser session already issued. Suspending the account in Fanout blocks access and revokes its sessions. +Fanout starts even when the identity provider is unreachable, so ingest, MCP +and existing sessions keep working. Discovery retries in the background with +backoff capped at 1 minute. Until it succeeds, the OIDC sign-in routes return +503 and the `oidc` check in `/readyz` reads `degraded`; no restart is needed. +Invalid configuration, including a provider that answers for a different +issuer at startup, still fails startup. + If neither `FANOUT_OIDC_ALLOWED_GROUPS` nor `FANOUT_OIDC_ALLOWED_DOMAINS` is configured, no eligibility allow policy is enforced; membership is managed locally. diff --git a/site/src/content/docs/docs/reference/api.mdx b/site/src/content/docs/docs/reference/api.mdx index dce248f61..2101702e0 100644 --- a/site/src/content/docs/docs/reference/api.mdx +++ b/site/src/content/docs/docs/reference/api.mdx @@ -28,7 +28,8 @@ configuration: - `/api/agent/*`: only when `FANOUT_AI_PROVIDER` selects `anthropic` or `openai`, with that provider's required key. An empty provider disables chat. - `/api/auth/oidc/start` and `/api/auth/oidc/callback`: only in - `FANOUT_AUTH_MODE=oidc`. + `FANOUT_AUTH_MODE=oidc`. They return 503 `unavailable` with `Retry-After` + until provider discovery succeeds. Everything else is always registered. A path shown with `:name` or `*` is the pattern Echo matches on, not a literal URL. diff --git a/site/src/content/docs/docs/reference/configuration/auth.mdx b/site/src/content/docs/docs/reference/configuration/auth.mdx index a069e50ef..1da8cd311 100644 --- a/site/src/content/docs/docs/reference/configuration/auth.mdx +++ b/site/src/content/docs/docs/reference/configuration/auth.mdx @@ -83,7 +83,7 @@ Selects required verified-email claims or issuer trust; issuer trust requires an ### `FANOUT_OIDC_ISSUER_URL` -Selects the identity provider issuer; required in OIDC mode. +Selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup. ### `FANOUT_OIDC_OPERATOR_GROUPS` diff --git a/site/src/content/docs/docs/reference/endpoints.mdx b/site/src/content/docs/docs/reference/endpoints.mdx index b0ef29b09..bc7348721 100644 --- a/site/src/content/docs/docs/reference/endpoints.mdx +++ b/site/src/content/docs/docs/reference/endpoints.mdx @@ -18,7 +18,7 @@ Everything below is served by the same HTTP listener as the browser client, on | Path | Requires | Reports | |---|---|---| | `/healthz` | nothing | Liveness — the process is up and its own work on the query engine is not stalled | -| `/readyz` | nothing | Readiness — storage is attached, plus resolved runtime sizing; a busy engine, or a query running past its deadline, reads `degraded` | +| `/readyz` | nothing | Readiness — storage is attached, plus resolved runtime sizing; a busy engine, a query running past its deadline, or failing OIDC discovery reads `degraded` | Use `/healthz` for restart decisions and `/readyz` for traffic decisions. From d02fb34f4ee1638726a9a71ff55e19dc3d847733 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 8 Nov 2026 09:02:29 -0800 Subject: [PATCH 2/2] fix(auth): truncate provider bodies in OIDC discovery logs go-oidc puts the provider's whole response body in its discovery error, so cap what each failure adds to the log at 256 bytes. Cover a provider that comes back with the wrong issuer on a retry: sign-in stays 503 and readiness stays degraded while discovery keeps retrying. --- internal/api/oidc.go | 2 +- internal/api/oidc_discovery.go | 14 +++- internal/api/oidc_discovery_test.go | 120 ++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/internal/api/oidc.go b/internal/api/oidc.go index 9915f4473..e14751473 100644 --- a/internal/api/oidc.go +++ b/internal/api/oidc.go @@ -78,7 +78,7 @@ func registerOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, us e.GET("/api/auth/oidc/start", h.Start) e.GET("/api/auth/oidc/callback", h.Callback) if err != nil { - slog.Warn("OIDC discovery failed at startup; retrying in the background", "issuer", cfg.OIDCIssuerURL, "err", err) + slog.Warn("OIDC discovery failed at startup; retrying in the background", "issuer", cfg.OIDCIssuerURL, "err", truncatedOIDCDiscoveryError(err)) go discovery.run(ctx) } else { close(discovery.done) diff --git a/internal/api/oidc_discovery.go b/internal/api/oidc_discovery.go index 1ef74eb5b..5b790bb9a 100644 --- a/internal/api/oidc_discovery.go +++ b/internal/api/oidc_discovery.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/url" "strconv" + "strings" "sync/atomic" "time" @@ -17,6 +18,7 @@ const ( oidcDiscoveryAttemptTimeout = 5 * time.Second oidcDiscoveryInitialBackoff = time.Second oidcDiscoveryMaxBackoff = time.Minute + oidcDiscoveryLogErrorLimit = 256 ) // oidcDiscoveryRetryAfter tells a client when a login is next worth trying: @@ -92,11 +94,21 @@ func (d *OIDCDiscovery) run(ctx context.Context) { return } delay = nextOIDCDiscoveryBackoff(delay, d.timing.max) - slog.Warn("OIDC discovery failed; sign-in stays unavailable", "issuer", d.issuer, "retry_in", delay, "err", err) + slog.Warn("OIDC discovery failed; sign-in stays unavailable", "issuer", d.issuer, "retry_in", delay, "err", truncatedOIDCDiscoveryError(err)) timer.Reset(delay) } } +// truncatedOIDCDiscoveryError bounds what a failure adds to the log: go-oidc +// includes the provider's whole response body in its error. +func truncatedOIDCDiscoveryError(err error) string { + message := err.Error() + if len(message) <= oidcDiscoveryLogErrorLimit { + return message + } + return strings.ToValidUTF8(message[:oidcDiscoveryLogErrorLimit], "") + "..." +} + func nextOIDCDiscoveryBackoff(delay, limit time.Duration) time.Duration { return min(2*delay, limit) } diff --git a/internal/api/oidc_discovery_test.go b/internal/api/oidc_discovery_test.go index 458c56fbc..bf95b4151 100644 --- a/internal/api/oidc_discovery_test.go +++ b/internal/api/oidc_discovery_test.go @@ -1,16 +1,21 @@ package api import ( + "bytes" "context" "encoding/json" + "errors" + "log/slog" "net/http" "net/http/httptest" "net/url" "os" "strings" + "sync" "sync/atomic" "testing" "time" + "unicode/utf8" "github.com/DATA-DOG/go-sqlmock" "github.com/labstack/echo/v5" @@ -399,3 +404,118 @@ func TestOIDCDiscoveryBackoffIsBounded(t *testing.T) { } } } + +type lockedLogBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func TestOIDCDiscoveryLogsTruncateProviderBody(t *testing.T) { + logs := &lockedLogBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + body := strings.Repeat("x", 1<<20) + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, body, http.StatusServiceUnavailable) + })) + defer idp.Close() + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("register: %v", err) + } + for deadline := time.Now().Add(5 * time.Second); !strings.Contains(logs.String(), "sign-in stays unavailable"); time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("retry failure was never logged") + } + } + cancel() + waitForOIDCDiscoveryExit(t, discovery) + + var failures int + for line := range strings.SplitSeq(logs.String(), "\n") { + if !strings.Contains(line, "OIDC discovery failed") { + continue + } + failures++ + if !strings.Contains(line, "503 Service Unavailable") { + t.Fatalf("log lost the failure status: %.400s", line) + } + if len(line) > 1024 { + t.Fatalf("log line is %d bytes, want the provider body truncated", len(line)) + } + } + if failures < 2 { + t.Fatalf("logged failures = %d, want the startup and retry sites", failures) + } +} + +func TestOIDCTruncatedDiscoveryErrorKeepsValidUTF8(t *testing.T) { + got := truncatedOIDCDiscoveryError(errors.New(strings.Repeat("é", 400))) + if !utf8.ValidString(got) || len(got) > oidcDiscoveryLogErrorLimit+len("...") { + t.Fatalf("truncated error is %d bytes, valid %v", len(got), utf8.ValidString(got)) + } + if short := truncatedOIDCDiscoveryError(errors.New("short")); short != "short" { + t.Fatalf("short error = %q", short) + } +} + +func TestOIDCDiscoveryWrongIssuerOnRetryStaysUnavailable(t *testing.T) { + var wrongIssuer atomic.Bool + var wrongAnswers atomic.Int64 + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !wrongIssuer.Load() { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + wrongAnswers.Add(1) + writeOIDCDiscoveryDocument(w, "https://other.example.com") + })) + defer idp.Close() + fixture := newOIDCDiscoveryFixture(t, idp.URL) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + discovery, err := fixture.register(ctx, fastOIDCDiscovery) + if err != nil { + t.Fatalf("startup failed while the provider was down: %v", err) + } + wrongIssuer.Store(true) + for deadline := time.Now().Add(5 * time.Second); wrongAnswers.Load() < 3; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("wrong-issuer answers = %d, want retries to continue", wrongAnswers.Load()) + } + } + + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/start")) + assertOIDCUnavailable(t, fixture.get("/api/auth/oidc/callback?code=c&state=s")) + recorder, resp := readinessWithHealthyEngine(t, discovery, fixture.cfg) + if recorder.Code != http.StatusOK { + t.Fatalf("/readyz status = %d, want 200: %s", recorder.Code, recorder.Body.String()) + } + if check := resp.Checks["oidc"]; check.Status != "degraded" { + t.Fatalf("oidc check = %+v, want degraded", check) + } + select { + case <-discovery.done: + t.Fatal("retry loop stopped on a wrong issuer") + default: + } +}