Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@
# FANOUT_OIDC_EMAIL_VERIFICATION=

# FANOUT_OIDC_ISSUER_URL; string
# Selects the identity provider issuer; required in OIDC mode.
# Selects the identity provider issuer; required in OIDC mode. Fanout starts while it is unreachable and retries discovery; a mismatched issuer fails startup.
# Default: (empty)
# FANOUT_OIDC_ISSUER_URL=

Expand Down
3 changes: 2 additions & 1 deletion cmd/fanout-docgen/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,8 @@ func renderRoutes(routes []api.RouteDoc) []byte {
b.WriteString("- `/api/agent/*`: only when `FANOUT_AI_PROVIDER` selects `anthropic` or\n")
b.WriteString(" `openai`, with that provider's required key. An empty provider disables chat.\n")
b.WriteString("- `/api/auth/oidc/start` and `/api/auth/oidc/callback`: only in\n")
b.WriteString(" `FANOUT_AUTH_MODE=oidc`.\n\n")
b.WriteString(" `FANOUT_AUTH_MODE=oidc`. They return 503 `unavailable` with `Retry-After`\n")
b.WriteString(" until provider discovery succeeds.\n\n")
b.WriteString("Everything else is always registered. A path shown with `:name` or `*` is the\n")
b.WriteString("pattern Echo matches on, not a literal URL.\n")
b.WriteString("\n")
Expand Down
10 changes: 6 additions & 4 deletions cmd/fanout/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,6 @@ func main() {

api.RegisterAuthMiddleware(e, userStore, browserSessions, auditStore, cfg)

// Health checks (liveness + readiness)
api.RegisterHealthRoutes(e, q, cfg)

// Prometheus metrics (internal/ops)
e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))

Expand Down Expand Up @@ -338,10 +335,15 @@ func main() {
}
codeStore := auth.NewCodeStore(sqlite.DB, cfg.AuthCodeSecret)
api.RegisterAuthRoutes(e, userStore, codeStore, setup, settingsStore, browserSessions, auditStore, smtpCfg, cfg)
if err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore); err != nil {
oidcDiscovery, err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore)
if err != nil {
slog.Error("OIDC initialization failed", "err", err)
os.Exit(1)
}

// Health checks (liveness + readiness)
api.RegisterHealthRoutes(e, q, cfg, oidcDiscovery)

api.RegisterUserRoutes(e, userStore, smtpCfg, cfg)
api.RegisterSettingsRoutes(e, cfg, settingsStore, auditStore)
slog.Info("auth enabled")
Expand Down
18 changes: 18 additions & 0 deletions docs/authentication.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,24 @@ session cookies are issued with `Secure`.

## OIDC

### Provider availability

Fanout runs OIDC discovery once at startup, for at most 5 seconds. If the
provider is unreachable or answers with an error, Fanout starts anyway and
serves ingest, MCP, and existing browser sessions. Discovery is retried in the
background, starting after 1 second and doubling to at most 1 minute between
attempts, until it succeeds or Fanout shuts down. No restart is needed.

Until then `/api/auth/oidc/start` and `/api/auth/oidc/callback` return 503 with
`Retry-After`, and the `oidc` check in `/readyz` reports `degraded` while the
endpoint still returns 200. The cause is in the logs.

Invalid configuration still fails startup: a value rejected by configuration
validation, or a provider that answers for an issuer other than
`FANOUT_OIDC_ISSUER_URL`. If a mismatch first appears on a background retry,
because the provider was down at startup, Fanout keeps running and retrying;
correct the issuer and restart.

### Public and community access

OIDC can provision a separate viewer account for every visitor whose provider
Expand Down
4 changes: 3 additions & 1 deletion docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,9 @@ proxy and certificate verification behaviour.
engine reports `degraded` with 200; a stalled one reports 503. A query still
running 25 seconds past its deadline reports `queries` as `degraded`, logs
a warning with a hash of the query (never its text), and increments
`fanout_duckdb_statement_overruns_total`.
`fanout_duckdb_statement_overruns_total`. In OIDC mode, the `oidc` check
reports `degraded` with 200 while provider discovery is failing; only new
OIDC sign-ins are affected, and discovery retries without a restart.
- `/metrics` exposes Prometheus metrics. Keep it private or configure
`FANOUT_METRICS_TOKEN`; do not make it public merely to simplify scraping.

Expand Down
8 changes: 7 additions & 1 deletion internal/api/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ type HealthHandler struct {
pool readPool
cfg config.Config
started time.Time
// oidc is nil outside OIDC mode.
oidc *OIDCDiscovery
}

type readPool interface {
Expand Down Expand Up @@ -121,6 +123,9 @@ collect:
resp.Checks["data"] = h.checkDataDir()
resp.Checks["maintenance"] = h.checkMaintenance()
resp.Checks["queries"] = h.checkQueries()
if h.oidc != nil {
resp.Checks["oidc"] = h.oidc.readiness()
}

// Determine overall status
hasUnhealthy := false
Expand Down Expand Up @@ -411,8 +416,9 @@ FROM rollup_state`).Scan(&updatedAt, &cacheCount, &ageSeconds)
}

// RegisterHealthRoutes registers health check endpoints
func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config) {
func RegisterHealthRoutes(e *echo.Echo, duck *engine.Duck, cfg config.Config, oidc *OIDCDiscovery) {
h := NewHealthHandler(duck, cfg)
h.oidc = oidc
e.GET("/healthz", h.Liveness)
e.GET("/readyz", h.Readiness)
}
2 changes: 1 addition & 1 deletion internal/api/health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ func TestCheckMaintenance_DegradedWhenNeverRanPastGrace(t *testing.T) {

func TestRegisterHealthRoutes_RegistersReadiness(t *testing.T) {
e := echo.New()
RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()})
RegisterHealthRoutes(e, nil, config.Config{DataDir: os.TempDir()}, nil)

req := httptest.NewRequest(http.MethodGet, "/readyz", nil)
rec := httptest.NewRecorder()
Expand Down
76 changes: 55 additions & 21 deletions internal/api/oidc.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,45 +29,75 @@ const oidcFlowTTL = 10 * time.Minute

type OIDCHandler struct {
cfg config.Config
provider *oidc.Provider
verifier *oidc.IDTokenVerifier
oauth oauth2.Config
discovery *OIDCDiscovery
users *appauth.UserStore
identities *appauth.IdentityStore
sessions *appauth.BrowserSessions
audit *appauth.AuditStore
limiter *appauth.KeyedLimiter
}

func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) error {
// RegisterOIDCRoutes makes one bounded discovery attempt and registers the
// login routes. It fails only on invalid configuration; if the provider is
// merely unavailable the routes answer 503 while discovery retries until ctx
// ends. The returned discovery is nil outside OIDC mode.
func RegisterOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore) (*OIDCDiscovery, error) {
return registerOIDCRoutes(ctx, e, cfg, users, identities, sessions, audit, defaultOIDCDiscoveryTiming)
}

func registerOIDCRoutes(ctx context.Context, e *echo.Echo, cfg config.Config, users *appauth.UserStore, identities *appauth.IdentityStore, sessions *appauth.BrowserSessions, audit *appauth.AuditStore, timing oidcDiscoveryTiming) (*OIDCDiscovery, error) {
if strings.ToLower(strings.TrimSpace(cfg.AuthMode)) != "oidc" {
return nil
return nil, nil
}
if users == nil || identities == nil || sessions == nil || audit == nil {
return fmt.Errorf("OIDC dependencies are required")
}
provider, err := oidc.NewProvider(ctx, cfg.OIDCIssuerURL)
if err != nil {
return fmt.Errorf("discover OIDC provider: %w", err)
return nil, fmt.Errorf("OIDC dependencies are required")
}
callback := strings.TrimRight(cfg.PublicURL, "/") + "/api/auth/oidc/callback"
h := &OIDCHandler{
cfg: cfg, provider: provider,
verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}),
oauth: oauth2.Config{
ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret,
Endpoint: provider.Endpoint(), RedirectURL: callback,
Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"},
discovery := &OIDCDiscovery{
issuer: cfg.OIDCIssuerURL, timing: timing, done: make(chan struct{}),
build: func(provider *oidc.Provider) *oidcClient {
return &oidcClient{
verifier: provider.Verifier(&oidc.Config{ClientID: cfg.OIDCClientID}),
oauth: oauth2.Config{
ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret,
Endpoint: provider.Endpoint(), RedirectURL: callback,
Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"},
},
}
},
}
err := discovery.attempt(ctx)
if err != nil && oidcConfigurationInvalid(err) {
return nil, fmt.Errorf("discover OIDC provider: %w", err)
}
h := &OIDCHandler{
cfg: cfg, discovery: discovery,
users: users, identities: identities, sessions: sessions, audit: audit,
limiter: appauth.NewKeyedLimiter(30, 15*time.Minute),
}
e.GET("/api/auth/oidc/start", h.Start)
e.GET("/api/auth/oidc/callback", h.Callback)
return nil
if err != nil {
slog.Warn("OIDC discovery failed at startup; retrying in the background", "issuer", cfg.OIDCIssuerURL, "err", truncatedOIDCDiscoveryError(err))
go discovery.run(ctx)
} else {
close(discovery.done)
}
return discovery, nil
}

func oidcUnavailable(c *echo.Context) error {
c.Response().Header().Set("Retry-After", oidcDiscoveryRetryAfter)
return echo.NewHTTPError(http.StatusServiceUnavailable, "OIDC sign-in is temporarily unavailable")
}

func (h *OIDCHandler) Start(c *echo.Context) error {
// Ahead of the limiter: retries during an outage must not spend the
// sign-in budget that is needed once the provider is back.
client := h.discovery.client()
if client == nil {
return oidcUnavailable(c)
}
if !h.limiter.Allow(c.RealIP()) {
return rateLimited(c, 15*time.Minute)
}
Expand All @@ -93,7 +123,7 @@ func (h *OIDCHandler) Start(c *echo.Context) error {
return echo.NewHTTPError(http.StatusInternalServerError, "failed to start login")
}
challenge := sha256.Sum256([]byte(verifier))
redirect := h.oauth.AuthCodeURL(state,
redirect := client.oauth.AuthCodeURL(state,
oidc.Nonce(nonce),
oauth2.SetAuthURLParam("code_challenge", base64.RawURLEncoding.EncodeToString(challenge[:])),
oauth2.SetAuthURLParam("code_challenge_method", "S256"),
Expand Down Expand Up @@ -132,6 +162,10 @@ func (c oidcClaims) groupsUnreadable() bool {
}

func (h *OIDCHandler) Callback(c *echo.Context) error {
client := h.discovery.client()
if client == nil {
return oidcUnavailable(c)
}
ctx := c.Request().Context()
state, expectedNonce, pkceVerifier, returnTo := h.sessions.OIDCFlow(ctx)
if state == "" || subtle.ConstantTimeCompare([]byte(state), []byte(c.QueryParam("state"))) != 1 {
Expand All @@ -143,15 +177,15 @@ func (h *OIDCHandler) Callback(c *echo.Context) error {
if providerError := c.QueryParam("error"); providerError != "" {
return h.oidcDenied(c, "identity provider denied login")
}
token, err := h.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier))
token, err := client.oauth.Exchange(ctx, c.QueryParam("code"), oauth2.VerifierOption(pkceVerifier))
if err != nil {
return h.oidcDenied(c, "code exchange failed")
}
rawIDToken, ok := token.Extra("id_token").(string)
if !ok || rawIDToken == "" {
return h.oidcDenied(c, "missing ID token")
}
idToken, err := h.verifier.Verify(ctx, rawIDToken)
idToken, err := client.verifier.Verify(ctx, rawIDToken)
if err != nil {
return h.oidcDenied(c, "invalid ID token")
}
Expand Down
140 changes: 140 additions & 0 deletions internal/api/oidc_discovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package api

import (
"context"
"errors"
"log/slog"
"net/url"
"strconv"
"strings"
"sync/atomic"
"time"

"github.com/coreos/go-oidc/v3/oidc"
"golang.org/x/oauth2"
)

const (
oidcDiscoveryAttemptTimeout = 5 * time.Second
oidcDiscoveryInitialBackoff = time.Second
oidcDiscoveryMaxBackoff = time.Minute
oidcDiscoveryLogErrorLimit = 256
)

// oidcDiscoveryRetryAfter tells a client when a login is next worth trying:
// no later than the longest wait between discovery attempts.
var oidcDiscoveryRetryAfter = strconv.Itoa(int(oidcDiscoveryMaxBackoff.Seconds()))

type oidcDiscoveryTiming struct {
attempt, initial, max time.Duration
}

var defaultOIDCDiscoveryTiming = oidcDiscoveryTiming{
attempt: oidcDiscoveryAttemptTimeout,
initial: oidcDiscoveryInitialBackoff,
max: oidcDiscoveryMaxBackoff,
}

// oidcClient is everything built from a discovered provider.
type oidcClient struct {
verifier *oidc.IDTokenVerifier
oauth oauth2.Config
}

// oidcDiscoveryState is the outcome of the latest discovery attempt; client is
// nil until an attempt succeeds.
type oidcDiscoveryState struct {
client *oidcClient
at time.Time
}

// OIDCDiscovery resolves the identity provider's metadata. An unreachable
// provider must not take ingest and existing sessions down with it, so a failed
// startup attempt is retried in the background while the login routes answer 503.
type OIDCDiscovery struct {
issuer string
timing oidcDiscoveryTiming
build func(*oidc.Provider) *oidcClient
state atomic.Pointer[oidcDiscoveryState]
done chan struct{}
}

func (d *OIDCDiscovery) attempt(ctx context.Context) error {
attemptCtx, cancel := context.WithTimeout(ctx, d.timing.attempt)
defer cancel()
provider, err := oidc.NewProvider(attemptCtx, d.issuer)
if err != nil {
d.state.Store(&oidcDiscoveryState{at: time.Now()})
return err
}
d.state.Store(&oidcDiscoveryState{client: d.build(provider), at: time.Now()})
return nil
}

// run retries until discovery succeeds or ctx ends. Only process startup can
// reject a configuration, so an issuer mismatch found here is retried and
// logged like any other failure rather than stopping a running instance.
func (d *OIDCDiscovery) run(ctx context.Context) {
defer close(d.done)
delay := d.timing.initial
timer := time.NewTimer(delay)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
}
err := d.attempt(ctx)
if err == nil {
slog.Info("OIDC discovery succeeded", "issuer", d.issuer)
return
}
if ctx.Err() != nil {
return
}
delay = nextOIDCDiscoveryBackoff(delay, d.timing.max)
slog.Warn("OIDC discovery failed; sign-in stays unavailable", "issuer", d.issuer, "retry_in", delay, "err", truncatedOIDCDiscoveryError(err))
timer.Reset(delay)
}
}

// truncatedOIDCDiscoveryError bounds what a failure adds to the log: go-oidc
// includes the provider's whole response body in its error.
func truncatedOIDCDiscoveryError(err error) string {
message := err.Error()
if len(message) <= oidcDiscoveryLogErrorLimit {
return message
}
return strings.ToValidUTF8(message[:oidcDiscoveryLogErrorLimit], "") + "..."
}

func nextOIDCDiscoveryBackoff(delay, limit time.Duration) time.Duration {
return min(2*delay, limit)
}

func (d *OIDCDiscovery) client() *oidcClient {
return d.state.Load().client
}

// readiness never reports unhealthy: the instance still serves everything but
// new OIDC logins, and removing it from rotation would make the outage total.
// The endpoint is public and discovery errors can carry the provider's response
// body, so the cause stays in the log.
func (d *OIDCDiscovery) readiness() CheckResult {
state := d.state.Load()
result := CheckResult{Status: "ok", UpdatedAt: state.at.UTC().Format(time.RFC3339)}
if state.client == nil {
result.Status = "degraded"
result.Detail = "discovery failing; OIDC sign-in returns 503 until a retry succeeds"
}
return result
}

// oidcConfigurationInvalid reports a failure that retrying cannot fix: the
// issuer cannot form a request, or the provider answered for a different issuer.
func oidcConfigurationInvalid(err error) bool {
var mismatch *oidc.IssuerMismatchError
var malformed *url.Error
return errors.As(err, &mismatch) || (errors.As(err, &malformed) && malformed.Op == "parse")
}
Loading
Loading