diff --git a/.env.example b/.env.example index 98144bf5a..e45538562 100644 --- a/.env.example +++ b/.env.example @@ -90,6 +90,16 @@ # Default: (empty) # FANOUT_FIRST_ADMIN_EMAIL= +# FANOUT_HEALTHCHECK_CERT_FILE; string +# Selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. +# Default: (empty) +# FANOUT_HEALTHCHECK_CERT_FILE= + +# FANOUT_HEALTHCHECK_KEY_FILE; string +# Selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. +# Default: (empty) +# FANOUT_HEALTHCHECK_KEY_FILE= + # FANOUT_INGEST_BATCH_SIZE; integer # Sets the group-commit row target; valid values are 1–50000. Individual requests remain atomic and can exceed this target. # Default: 50000 @@ -253,6 +263,11 @@ # Default: (empty) # FANOUT_TLS_CERT_FILE= +# FANOUT_TLS_CLIENT_CA_FILE; string +# Selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. +# Default: (empty) +# FANOUT_TLS_CLIENT_CA_FILE= + # FANOUT_TLS_KEY_FILE; string # Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. # Default: (empty) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index d7bab4280..ecabd0569 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -81,7 +81,8 @@ func main() { } } if healthURL != "" || derivedProbe { - if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE")); err != nil { + if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE"), + os.Getenv("FANOUT_HEALTHCHECK_CERT_FILE"), os.Getenv("FANOUT_HEALTHCHECK_KEY_FILE")); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -110,6 +111,15 @@ func main() { return } cfg.LogStartup() + var clientCAs *x509.CertPool + if cfg.ClientCertificatesRequired() { + // Load before opening any store so a bad bundle fails startup at once. + clientCAs, err = server.LoadClientCAs(cfg.TLSClientCAFile) + if err != nil { + slog.Error("invalid configuration", "err", err) + os.Exit(1) + } + } if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil { slog.Error("create data dir failed", "err", err) @@ -413,8 +423,11 @@ func main() { e.GET("/*", spa) httpSrv := server.New(cfg.Addr, e, ingest.NewHTTPHandler(ing, settingsStore), grpcSrv) + if clientCAs != nil { + server.RequireClientCertificates(httpSrv, clientCAs) + } go func() { - slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled()) + slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled(), "client_certificates", clientCAs != nil) var err error if cfg.TLSEnabled() { err = httpSrv.ListenAndServeTLS(cfg.TLSCertFile, cfg.TLSKeyFile) @@ -579,18 +592,31 @@ func healthcheckURL(addr, certFile string) (string, error) { return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil } -func checkHealth(healthURL string, derivedProbe bool, certFile string) error { +func checkHealth(healthURL string, derivedProbe bool, certFile, clientCertFile, clientKeyFile string) error { client := &http.Client{Timeout: 3 * time.Second} if derivedProbe { // Derived probes bypass proxies. HTTPS IP literals trust only the configured // certificate and verify its first DNS or IP SAN instead of the bind IP. // Hostnames verify normally; explicit URLs retain all default behaviour. + // A configured healthcheck client certificate is presented on derived + // HTTPS probes so the check passes a listener that requires one. probe, err := url.Parse(healthURL) if err != nil { return fmt.Errorf("healthcheck: %w", err) } transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil + if probe.Scheme == "https" && (clientCertFile != "" || clientKeyFile != "") { + if clientCertFile == "" || clientKeyFile == "" { + return fmt.Errorf("healthcheck: FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set") + } + pair, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) + if err != nil { + // Key-pair errors describe structure only, never key material. + return fmt.Errorf("healthcheck: load client certificate %q and key %q: %w", clientCertFile, clientKeyFile, err) + } + transport.TLSClientConfig = &tls.Config{Certificates: []tls.Certificate{pair}} + } if _, err := netip.ParseAddr(probe.Hostname()); probe.Scheme == "https" && err == nil { data, err := os.ReadFile(certFile) if err != nil { @@ -620,7 +646,11 @@ func checkHealth(healthURL string, derivedProbe bool, certFile string) error { } roots := x509.NewCertPool() roots.AddCert(leaf) - transport.TLSClientConfig = &tls.Config{RootCAs: roots, ServerName: serverName} + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = roots + transport.TLSClientConfig.ServerName = serverName } defer transport.CloseIdleConnections() client.Transport = transport diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index 1aff13234..04e28dcaf 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "encoding/pem" "errors" "flag" @@ -27,6 +28,7 @@ import ( "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/server" "github.com/labstack/fanout/internal/store" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" @@ -122,7 +124,7 @@ func TestCheckHealth(t *testing.T) { writer.WriteHeader(test.statusCode) })) defer server.Close() - if err := checkHealth(server.URL, false, ""); (err != nil) != test.wantErr { + if err := checkHealth(server.URL, false, "", "", ""); (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } }) @@ -340,11 +342,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) - if err := checkHealth(server.URL+"/healthz", true, certFile); err != nil { + if err := checkHealth(server.URL+"/healthz", true, certFile, "", ""); err != nil { t.Fatal(err) } var verificationError *tls.CertificateVerificationError - if err := checkHealth(server.URL+"/healthz", false, certFile); !errors.As(err, &verificationError) { + if err := checkHealth(server.URL+"/healthz", false, certFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("explicit URL must verify the certificate, got %v", err) } other := *server.Certificate() @@ -359,11 +361,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) { if err := os.WriteFile(otherFile, chain, 0o600); err != nil { t.Fatal(err) } - if err := checkHealth(server.URL+"/healthz", true, otherFile); !errors.As(err, &verificationError) { + if err := checkHealth(server.URL+"/healthz", true, otherFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("derived IP probe must reject a different certificate, got %v", err) } probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz" - if err := checkHealth(probe, true, certFile); !errors.As(err, &verificationError) { + if err := checkHealth(probe, true, certFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("derived hostname probe must verify the certificate, got %v", err) } } @@ -390,6 +392,106 @@ func newHealthcheckCertificate(t *testing.T, certificate *x509.Certificate) tls. return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} } +// The derived probe presents FANOUT_HEALTHCHECK_CERT_FILE to a listener that +// requires client certificates, and fails without it. +func TestCheckHealthWithClientCertificate(t *testing.T) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(10), Subject: pkix.Name{CommonName: "healthcheck CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + clientDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(11), Subject: pkix.Name{CommonName: "fanout healthcheck"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, ca, &clientKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + caFile := filepath.Join(dir, "client-ca.pem") + clientCertFile := filepath.Join(dir, "healthcheck.pem") + clientKeyFile := filepath.Join(dir, "healthcheck-key.pem") + keyDER, err := x509.MarshalPKCS8PrivateKey(clientKey) + if err != nil { + t.Fatal(err) + } + for path, block := range map[string]*pem.Block{ + caFile: {Type: "CERTIFICATE", Bytes: caDER}, + clientCertFile: {Type: "CERTIFICATE", Bytes: clientDER}, + clientKeyFile: {Type: "PRIVATE KEY", Bytes: keyDER}, + } { + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil { + t.Fatal(err) + } + } + + ts := httptest.NewUnstartedServer(nil) + ts.Config = server.New("", http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" || len(request.TLS.PeerCertificates) == 0 { + t.Errorf("probe path %q with %d client certificates", request.URL.Path, len(request.TLS.PeerCertificates)) + } + w.WriteHeader(http.StatusOK) + }), http.NotFoundHandler(), http.NotFoundHandler()) + clientCAs, err := server.LoadClientCAs(caFile) + if err != nil { + t.Fatal(err) + } + server.RequireClientCertificates(ts.Config, clientCAs) + ts.TLS = ts.Config.TLSConfig.Clone() + ts.TLS.Certificates = []tls.Certificate{newHealthcheckCertificate(t, &x509.Certificate{ + SerialNumber: big.NewInt(12), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + })} + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + certFile := writeHealthcheckCertificate(t, ts.Certificate().Raw) + + probe, err := healthcheckURL(ts.Listener.Addr().String(), certFile) + if err != nil { + t.Fatal(err) + } + if err := checkHealth(probe, true, certFile, clientCertFile, clientKeyFile); err != nil { + t.Fatalf("healthcheck with client certificate: %v", err) + } + if err := checkHealth(probe, true, certFile, "", ""); err == nil { + t.Fatal("healthcheck without a client certificate must fail") + } + if err := checkHealth(probe, true, certFile, clientCertFile, ""); err == nil || !strings.Contains(err.Error(), "FANOUT_HEALTHCHECK_KEY_FILE") { + t.Fatalf("partial client pair must name both variables, got %v", err) + } + missing := filepath.Join(dir, "missing-key.pem") + if err := checkHealth(probe, true, certFile, clientCertFile, missing); err == nil || !strings.Contains(err.Error(), missing) { + t.Fatalf("unreadable client key must name its path, got %v", err) + } + keyPEM, err := os.ReadFile(clientKeyFile) + if err != nil { + t.Fatal(err) + } + // A key passed as the certificate must fail without echoing key material. + if err := checkHealth(probe, true, certFile, clientKeyFile, clientKeyFile); err == nil || !strings.Contains(err.Error(), clientKeyFile) || strings.Contains(err.Error(), string(keyPEM[40:80])) { + t.Fatalf("invalid client pair must name its path without contents, got %v", err) + } +} + func TestCheckHealthCertificateFileErrors(t *testing.T) { const contents = "private file contents must not appear in errors" for _, test := range []struct { @@ -408,7 +510,7 @@ func TestCheckHealthCertificateFileErrors(t *testing.T) { t.Fatal(err) } } - err := checkHealth("https://127.0.0.1:7520/healthz", true, path) + err := checkHealth("https://127.0.0.1:7520/healthz", true, path, "", "") if err == nil || !strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { t.Fatalf("certificate file error must name its path without contents, got %v", err) } @@ -453,7 +555,7 @@ func TestCheckHealthTLSCertificate(t *testing.T) { server.StartTLS() defer server.Close() certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) - err := checkHealth(server.URL+"/healthz", true, certFile) + err := checkHealth(server.URL+"/healthz", true, certFile, "", "") if (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } @@ -504,7 +606,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkHealth(probe, true, certFile) + err = checkHealth(probe, true, certFile, "", "") var verificationError *tls.CertificateVerificationError if strings.HasSuffix(addr, "example.com:7520") { if !errors.As(err, &verificationError) { @@ -517,7 +619,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkHealth(explicitProbe, derivedProbe, certFile) + err = checkHealth(explicitProbe, derivedProbe, certFile, "", "") if !errors.As(err, &verificationError) { t.Fatalf("explicit TLS probe must use system verification, got %v", err) } @@ -596,10 +698,10 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { if err != nil { t.Fatal(err) } - if err := checkHealth(probe, true, certFile); err != nil { + if err := checkHealth(probe, true, certFile, "", ""); err != nil { t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err) } - if err := checkHealth(probe, false, certFile); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { + if err := checkHealth(probe, false, certFile, "", ""); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err) } }) diff --git a/docs/operations.md b/docs/operations.md index 8becbb864..83a637303 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -164,6 +164,11 @@ Setting both `FANOUT_TLS_CERT_FILE` and `FANOUT_TLS_KEY_FILE` enables TLS 1.3 on the shared listener. A proxy must forward HTTP/2 to Fanout for gRPC: use h2c for a plaintext backend, or HTTP/2 over TLS when Fanout has a certificate. HTTP/1.1 forwarding supports OTLP/HTTP, but cannot carry gRPC. +`FANOUT_TLS_CLIENT_CA_FILE` additionally requires every client, including a +TLS-terminating proxy such as Cloudflare with Authenticated Origin Pulls, to +present a certificate from that CA bundle. The bundle is read at startup, so +rotation needs a restart; keep the old and new CA in the bundle while clients +move to certificates from the new one. Set `FANOUT_PUBLIC_URL` to the externally reachable HTTPS browser origin. When using forwarded client addresses, set `FANOUT_TRUSTED_PROXY_CIDRS` to only the @@ -182,9 +187,11 @@ default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`. `FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies. HTTPS IP-literal probes trust only the configured certificate and verify its first DNS or IP SAN, which may differ from the bind IP; hostname probes verify -normally. Healthcheck reads only these two settings and does not require valid -secrets. An explicit `fanout healthcheck ` uses default proxy and -certificate verification behaviour. +normally. Derived HTTPS probes present `FANOUT_HEALTHCHECK_CERT_FILE` and +`FANOUT_HEALTHCHECK_KEY_FILE` when set, so they pass a listener with +`FANOUT_TLS_CLIENT_CA_FILE`. Healthcheck reads only these settings and does +not require valid secrets. An explicit `fanout healthcheck ` uses default +proxy and certificate verification behaviour. - `/healthz` is the liveness check. It fails only when Fanout's own work on the query engine stalls: a Parquet publication holds reads for 10 minutes, diff --git a/internal/config/config.go b/internal/config/config.go index df8eac552..b016f203b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -205,8 +205,14 @@ type Config struct { // TLSCertFile selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. TLSCertFile string `group:"server" env:"FANOUT_TLS_CERT_FILE"` // TLSKeyFile selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. - TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` - resolvedSizing sizingSource + TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` + // TLSClientCAFile selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. + TLSClientCAFile string `group:"server" env:"FANOUT_TLS_CLIENT_CA_FILE"` + // HealthcheckCertFile selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. + HealthcheckCertFile string `group:"server" env:"FANOUT_HEALTHCHECK_CERT_FILE"` + // HealthcheckKeyFile selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. + HealthcheckKeyFile string `group:"server" env:"FANOUT_HEALTHCHECK_KEY_FILE"` + resolvedSizing sizingSource } // LogStartup reports the effective non-secret sizing and security-sensitive @@ -262,6 +268,12 @@ func (c Config) TLSEnabled() bool { strings.TrimSpace(c.TLSKeyFile) != "" } +// ClientCertificatesRequired reports whether the TLS listener requires and +// verifies client certificates against FANOUT_TLS_CLIENT_CA_FILE. +func (c Config) ClientCertificatesRequired() bool { + return c.TLSEnabled() && strings.TrimSpace(c.TLSClientCAFile) != "" +} + // Validate checks that config values are sane. func (c Config) Validate() error { if strings.TrimSpace(c.Addr) == "" { @@ -401,6 +413,17 @@ func (c Config) Validate() error { if anySet(c.TLSCertFile, c.TLSKeyFile) && !c.TLSEnabled() { return fmt.Errorf("FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE must both be set") } + if anySet(c.TLSClientCAFile) && !c.TLSEnabled() { + return fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE") + } + if anySet(c.HealthcheckCertFile, c.HealthcheckKeyFile) { + if !anySet(c.HealthcheckCertFile) || !anySet(c.HealthcheckKeyFile) { + return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set") + } + if !anySet(c.TLSClientCAFile) { + return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE require FANOUT_TLS_CLIENT_CA_FILE") + } + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 96981f55e..81c3488f4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -306,6 +306,20 @@ func TestValidate(t *testing.T) { {"trusted proxy invalid CIDR", func(c *Config) { c.TrustedProxyCIDRs = "private-network" }}, {"TLS partial cert", func(c *Config) { c.TLSCertFile = "server.pem" }}, {"TLS partial key", func(c *Config) { c.TLSKeyFile = "server-key.pem" }}, + {"TLS client CA without TLS", func(c *Config) { c.TLSClientCAFile = "client-ca.pem" }}, + {"TLS client CA without key", func(c *Config) { c.TLSCertFile = "server.pem"; c.TLSClientCAFile = "client-ca.pem" }}, + {"healthcheck cert without key", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + c.HealthcheckCertFile = "probe.pem" + }}, + {"healthcheck key without cert", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + c.HealthcheckKeyFile = "probe-key.pem" + }}, + {"healthcheck pair without client CA", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile = "server.pem", "server-key.pem" + c.HealthcheckCertFile, c.HealthcheckKeyFile = "probe.pem", "probe-key.pem" + }}, } for _, tc := range tests { @@ -413,6 +427,24 @@ func TestValidate(t *testing.T) { if !c.TLSEnabled() { t.Error("TLSEnabled = false, want true") } + if c.ClientCertificatesRequired() { + t.Error("ClientCertificatesRequired = true without FANOUT_TLS_CLIENT_CA_FILE") + } + }) + + t.Run("TLS_client_CA_valid", func(t *testing.T) { + c := valid + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + if err := c.Validate(); err != nil { + t.Errorf("client CA config should be valid: %v", err) + } + if !c.ClientCertificatesRequired() { + t.Error("ClientCertificatesRequired = false, want true") + } + c.HealthcheckCertFile, c.HealthcheckKeyFile = "probe.pem", "probe-key.pem" + if err := c.Validate(); err != nil { + t.Errorf("healthcheck client certificate config should be valid: %v", err) + } }) } diff --git a/internal/ingest/listener_test.go b/internal/ingest/listener_test.go index ce5911932..96bc1b8bd 100644 --- a/internal/ingest/listener_test.go +++ b/internal/ingest/listener_test.go @@ -3,10 +3,15 @@ package ingest import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "errors" "io" + "math/big" "net/http" "net/http/httptest" "strings" @@ -29,8 +34,8 @@ import ( ) func TestSharedListenerProtocols(t *testing.T) { - for _, secure := range []bool{false, true} { - t.Run(map[bool]string{false: "plaintext", true: "TLS"}[secure], func(t *testing.T) { + for _, mode := range []string{"plaintext", "TLS", "mutual TLS"} { + t.Run(mode, func(t *testing.T) { f := newHTTPIngestFixture(t, true) g := grpc.NewServer(GRPCServerOptions(f.store)...) RegisterOTLP(g, NewServer(config.Config{}, newTestSubmitter(f.spans, f.logs, f.metrics))) @@ -39,14 +44,23 @@ func TestSharedListenerProtocols(t *testing.T) { ts := httptest.NewUnstartedServer(nil) ts.Config = httpserver.New("", app, f.handler, g) creds := insecure.NewCredentials() - if secure { + if mode == "plaintext" { + ts.Start() + } else { + var clientCertificates []tls.Certificate + if mode == "mutual TLS" { + clientCAs, clientCertificate := newTestClientCA(t) + httpserver.RequireClientCertificates(ts.Config, clientCAs) + // StartTLS replaces the server's TLS config with ts.TLS. + ts.TLS = ts.Config.TLSConfig.Clone() + clientCertificates = []tls.Certificate{clientCertificate} + } ts.EnableHTTP2 = true ts.StartTLS() roots := x509.NewCertPool() roots.AddCert(ts.Certificate()) - creds = credentials.NewTLS(&tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13}) - } else { - ts.Start() + creds = credentials.NewTLS(&tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, Certificates: clientCertificates}) + ts.Client().Transport.(*http.Transport).TLSClientConfig.Certificates = clientCertificates } defer ts.Close() conn, err := grpc.NewClient(ts.Listener.Addr().String(), grpc.WithTransportCredentials(creds)) @@ -128,6 +142,96 @@ func TestSharedListenerProtocols(t *testing.T) { } } +// The listener must refuse gRPC and HTTP handshakes that present no client +// certificate or one from an untrusted CA when client certificates are required. +func TestSharedListenerRejectsMissingClientCertificate(t *testing.T) { + f := newHTTPIngestFixture(t, true) + g := grpc.NewServer(GRPCServerOptions(f.store)...) + RegisterOTLP(g, NewServer(config.Config{}, newTestSubmitter(f.spans, f.logs, f.metrics))) + defer g.Stop() + ts := httptest.NewUnstartedServer(nil) + ts.Config = httpserver.New("", http.NotFoundHandler(), f.handler, g) + clientCAs, _ := newTestClientCA(t) + httpserver.RequireClientCertificates(ts.Config, clientCAs) + ts.TLS = ts.Config.TLSConfig.Clone() + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + if ts.TLS.MinVersion != tls.VersionTLS13 || ts.TLS.ClientAuth != tls.RequireAndVerifyClientCert { + t.Fatalf("TLS config = min %x auth %v", ts.TLS.MinVersion, ts.TLS.ClientAuth) + } + _, untrusted := newTestClientCA(t) + for name, certificates := range map[string][]tls.Certificate{"none": nil, "untrusted": {untrusted}} { + t.Run(name, func(t *testing.T) { + roots := x509.NewCertPool() + roots.AddCert(ts.Certificate()) + clientTLS := &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, Certificates: certificates} + conn, err := grpc.NewClient(ts.Listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(clientTLS.Clone()))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+f.token) + if _, err := collectortrace.NewTraceServiceClient(conn).Export(ctx, testTraceRequest()); status.Code(err) != codes.Unavailable { + t.Fatalf("gRPC without a trusted client certificate: %v", err) + } + transport := &http.Transport{TLSClientConfig: clientTLS.Clone(), ForceAttemptHTTP2: true} + defer transport.CloseIdleConnections() + resp, err := (&http.Client{Transport: transport, Timeout: 5 * time.Second}).Get(ts.URL + "/v1/traces") + if err == nil { + _ = resp.Body.Close() + t.Fatalf("HTTP without a trusted client certificate: %s", resp.Status) + } + }) + } + select { + case <-f.spans: + t.Fatal("rejected handshake delivered telemetry") + default: + } +} + +// newTestClientCA returns a pool holding a fresh CA and a client certificate +// that CA issued. +func newTestClientCA(t *testing.T) (*x509.CertPool, tls.Certificate) { + t.Helper() + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test client CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + clientTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "test client"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, ca, &clientKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(ca) + return pool, tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} +} + type blockedExport struct{ started, release chan struct{} } func (b *blockedExport) Submit(ctx context.Context, _ telemetrystore.Batch) error { diff --git a/internal/server/clientca.go b/internal/server/clientca.go new file mode 100644 index 000000000..a5ecb5056 --- /dev/null +++ b/internal/server/clientca.go @@ -0,0 +1,65 @@ +package server + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "net/http" + "os" +) + +// RequireClientCertificates makes srv's TLS listener require and verify client +// certificates issued by a CA in pool. TLS 1.3 and ALPN (h2 and http/1.1) are +// unchanged, so gRPC and HTTP keep sharing the listener. +func RequireClientCertificates(srv *http.Server, pool *x509.CertPool) { + if srv.TLSConfig == nil { + srv.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13} + } + srv.TLSConfig.ClientCAs = pool + srv.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert +} + +// LoadClientCAs reads the FANOUT_TLS_CLIENT_CA_FILE bundle: one or more PEM +// CERTIFICATE blocks separated only by whitespace. It rejects anything else, +// including text around the blocks and malformed blocks that pem.Decode would +// otherwise skip, so a truncated or wrong file fails startup. Errors name the +// path, never file contents. +func LoadClientCAs(path string) (*x509.CertPool, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: read %q: %w", path, err) + } + pool := x509.NewCertPool() + count := 0 + for { + data = bytes.TrimLeft(data, " \t\r\n") + if len(data) == 0 { + break + } + block, rest := pem.Decode(data) + consumed := data[:len(data)-len(rest)] + // pem.Decode skips leading text and malformed blocks; only accept a + // block that starts the remaining input and is the only one consumed. + if block == nil || !bytes.HasPrefix(data, pemBegin) || bytes.Count(consumed, pemBegin) != 1 { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: %q contains data that is not a PEM block after certificate %d", path, count) + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: %q contains a %q PEM block; only certificates are allowed", path, block.Type) + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: parse certificate %d in %q: %w", count+1, path, err) + } + pool.AddCert(certificate) + count++ + data = rest + } + if count == 0 { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: no PEM certificates in %q", path) + } + return pool, nil +} + +var pemBegin = []byte("-----BEGIN ") diff --git a/internal/server/clientca_test.go b/internal/server/clientca_test.go new file mode 100644 index 000000000..1ca87228a --- /dev/null +++ b/internal/server/clientca_test.go @@ -0,0 +1,215 @@ +package server + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadClientCAs(t *testing.T) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "client CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + const contents = "private file contents must not appear in errors" + for _, test := range []struct { + name string + data []byte + wantErr bool + }{ + {name: "one certificate", data: certificate}, + {name: "bundle", data: append(append([]byte{}, certificate...), certificate...)}, + {name: "unreadable", wantErr: true}, + {name: "empty", data: []byte{}, wantErr: true}, + {name: "not PEM", data: []byte(contents), wantErr: true}, + {name: "private key block", data: append(append([]byte{}, certificate...), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte(contents)})...), wantErr: true}, + {name: "invalid certificate", data: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte(contents)}), wantErr: true}, + {name: "surrounding whitespace", data: append(append([]byte("\n \r\n"), certificate...), "\n\n"...)}, + {name: "trailing text", data: append(append([]byte{}, certificate...), contents...), wantErr: true}, + {name: "leading text", data: append([]byte(contents+"\n"), certificate...), wantErr: true}, + {name: "malformed block before certificate", data: append([]byte("-----BEGIN CERTIFICATE-----\n!!!\n-----END CERTIFICATE-----\n"), certificate...), wantErr: true}, + {name: "truncated certificate", data: certificate[:len(certificate)-30], wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "client-ca.pem") + if test.data != nil { + if err := os.WriteFile(path, test.data, 0o600); err != nil { + t.Fatal(err) + } + } + pool, err := LoadClientCAs(path) + if !test.wantErr { + if err != nil || pool == nil { + t.Fatalf("LoadClientCAs() = %v, %v", pool, err) + } + return + } + if err == nil || !strings.HasPrefix(err.Error(), "FANOUT_TLS_CLIENT_CA_FILE:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { + t.Fatalf("error must name the variable and path without contents, got %v", err) + } + }) + } +} + +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T, name string, parent *testCA) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + issuer, signer := template, key + if parent != nil { + issuer, signer = parent.cert, parent.key + } + der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, signer) + if err != nil { + t.Fatal(err) + } + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return &testCA{cert: certificate, key: key} +} + +// issue returns a leaf certificate followed by any intermediates. +func (ca *testCA) issue(t *testing.T, usage []x509.ExtKeyUsage, ips []net.IP, chain ...*testCA) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usage, IPAddresses: ips, + }, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + certificate := tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} + for _, intermediate := range chain { + certificate.Certificate = append(certificate.Certificate, intermediate.cert.Raw) + } + return certificate +} + +// The stdlib verifier enforces the client-auth EKU and builds chains through +// intermediates the client sends, with only the root in the bundle. +func TestRequireClientCertificatesHandshake(t *testing.T) { + root := newTestCA(t, "client root", nil) + intermediate := newTestCA(t, "client intermediate", root) + serverCA := newTestCA(t, "server CA", nil) + clientAuth := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + for _, test := range []struct { + name string + client []tls.Certificate + wantErr string + }{ + {name: "client auth from root", client: []tls.Certificate{root.issue(t, clientAuth, nil)}}, + {name: "leaf and intermediate", client: []tls.Certificate{intermediate.issue(t, clientAuth, nil, intermediate)}}, + {name: "leaf without its intermediate", client: []tls.Certificate{intermediate.issue(t, clientAuth, nil)}, wantErr: "unknown authority"}, + {name: "server auth only", client: []tls.Certificate{root.issue(t, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, nil)}, wantErr: "incompatible key usage"}, + {name: "no certificate", wantErr: "didn't provide a certificate"}, + } { + t.Run(test.name, func(t *testing.T) { + srv := New("", http.NotFoundHandler(), http.NotFoundHandler(), http.NotFoundHandler()) + pool := x509.NewCertPool() + pool.AddCert(root.cert) + RequireClientCertificates(srv, pool) + serverTLS := srv.TLSConfig.Clone() + serverTLS.Certificates = []tls.Certificate{serverCA.issue(t, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []net.IP{net.ParseIP("127.0.0.1")})} + serverTLS.NextProtos = []string{"h2", "http/1.1"} + roots := x509.NewCertPool() + roots.AddCert(serverCA.cert) + // A buffered loopback socket, unlike net.Pipe, lets the server + // write its alert after the client's handshake has returned. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverErr := make(chan error, 1) + go func() { + serverConn, err := listener.Accept() + if err != nil { + serverErr <- err + return + } + _ = serverConn.SetDeadline(time.Now().Add(10 * time.Second)) + conn := tls.Server(serverConn, serverTLS) + err = conn.Handshake() + if err == nil && conn.ConnectionState().NegotiatedProtocol != "h2" { + err = fmt.Errorf("ALPN = %q, want h2", conn.ConnectionState().NegotiatedProtocol) + } + serverErr <- err + _ = conn.Close() + }() + clientConn, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer clientConn.Close() + _ = clientConn.SetDeadline(time.Now().Add(10 * time.Second)) + clientTLS := &tls.Config{RootCAs: roots, ServerName: "127.0.0.1", MinVersion: tls.VersionTLS13, NextProtos: []string{"h2"}} + if len(test.client) > 0 { + // Always send the certificate, even when its issuer is not among + // the server's acceptable CAs, so the server verifier decides. + clientTLS.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return &test.client[0], nil } + } + client := tls.Client(clientConn, clientTLS) + _ = client.Handshake() + // TLS 1.3 clients finish before the server checks their certificate, + // so the server's result decides. + err = <-serverErr + if test.wantErr == "" && err != nil || test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("server handshake error = %v, want %q", err, test.wantErr) + } + }) + } +} + +func TestRequireClientCertificatesKeepsProtocols(t *testing.T) { + srv := New("", http.NotFoundHandler(), http.NotFoundHandler(), http.NotFoundHandler()) + pool := x509.NewCertPool() + RequireClientCertificates(srv, pool) + if srv.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert || srv.TLSConfig.ClientCAs != pool { + t.Fatalf("client auth = %v, pool set = %v", srv.TLSConfig.ClientAuth, srv.TLSConfig.ClientCAs == pool) + } + if srv.TLSConfig.MinVersion != tls.VersionTLS13 || !srv.Protocols.HTTP2() || !srv.Protocols.HTTP1() { + t.Fatalf("TLS min version %x or protocols changed", srv.TLSConfig.MinVersion) + } +} diff --git a/site/src/content/docs/docs/guides/tls.mdx b/site/src/content/docs/docs/guides/tls.mdx index d50baebeb..dee85d841 100644 --- a/site/src/content/docs/docs/guides/tls.mdx +++ b/site/src/content/docs/docs/guides/tls.mdx @@ -23,6 +23,68 @@ export FANOUT_TLS_KEY_FILE=/etc/fanout/tls/key.pem That enables TLS 1.3 on the shared listener. Browser traffic and OTLP/HTTP use HTTPS; gRPC uses HTTP/2 negotiated through ALPN. +### Require client certificates + +To accept connections only from clients holding a certificate from a CA you +choose, add a PEM bundle of those CA certificates: + +```sh +export FANOUT_TLS_CLIENT_CA_FILE=/etc/fanout/tls/client-ca.pem +``` + +The listener then requires and verifies a client certificate on every +handshake, for the browser, API, MCP, OTLP/HTTP and gRPC alike. TLS 1.3 and +ALPN are unchanged. A handshake without a certificate from a listed CA fails +before any request is read. The setting requires `FANOUT_TLS_CERT_FILE` and +`FANOUT_TLS_KEY_FILE`; an unreadable bundle, or one with no certificates, a +non-certificate block or an invalid certificate, fails startup and names the path. + +The bundle is read once at startup, so restart Fanout after changing it. To +rotate a CA without rejecting clients, list both the old and new CA in the +bundle until every client has a certificate from the new one, then remove the +old CA and restart again. + +A typical use is +[Cloudflare Authenticated Origin Pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/), +where Cloudflare presents a client certificate on every connection to your +origin. Use zone-level or per-hostname pulls: you upload a client certificate +issued by your own CA to Cloudflare, and set `FANOUT_TLS_CLIENT_CA_FILE` to +that CA. The certificate is exclusive to your account, so the origin accepts +only connections that Cloudflare makes on behalf of your zone or hostname. +Connections that bypass Cloudflare are rejected. + +:::caution +Do not trust the Cloudflare-provided certificate used by +[global pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/set-up/global/). +It is shared by all Cloudflare accounts and proves only that a connection came +from Cloudflare's network, so any Cloudflare customer's zone could reach your origin. +::: + +Collectors and agents that connect directly, rather than through that proxy, +need their own client certificate from a CA in the bundle. + +#### Health checks + +`fanout healthcheck` (the container image's `HEALTHCHECK`) connects to the same +listener, so it also needs a client certificate. Give the probe a certificate +and key: + +```sh +export FANOUT_HEALTHCHECK_CERT_FILE=/etc/fanout/tls/healthcheck.pem +export FANOUT_HEALTHCHECK_KEY_FILE=/etc/fanout/tls/healthcheck-key.pem +``` + +If you uploaded your own CA's certificate for Authenticated Origin Pulls, issue +the probe certificate from that same CA; the bundle already trusts it. +Otherwise, issue it from a private CA you control and append that CA to the +bundle. Keep any CA you append dedicated to Fanout clients: the listener +accepts every certificate it issues. + +The probe presents the pair only on its derived HTTPS check of `FANOUT_ADDR`; +an explicit `fanout healthcheck ` does not. Both settings need +`FANOUT_TLS_CLIENT_CA_FILE`, and supplying only one fails. Without them, the +probe's handshake is rejected and the health check fails. + ## Behind a reverse proxy Terminate at the proxy and forward to Fanout's shared port (`7520` by default). Two settings have to diff --git a/site/src/content/docs/docs/reference/cli.mdx b/site/src/content/docs/docs/reference/cli.mdx index 7db7fc092..e5f953481 100644 --- a/site/src/content/docs/docs/reference/cli.mdx +++ b/site/src/content/docs/docs/reference/cli.mdx @@ -42,11 +42,14 @@ works once. It can run while the server is online. See ## Health probes -With no URL, `healthcheck` reads only `FANOUT_ADDR` and -`FANOUT_TLS_CERT_FILE`. Unspecified bind hosts become `127.0.0.1`. +With no URL, `healthcheck` reads only `FANOUT_ADDR`, +`FANOUT_TLS_CERT_FILE`, `FANOUT_HEALTHCHECK_CERT_FILE` and +`FANOUT_HEALTHCHECK_KEY_FILE`. Unspecified bind hosts become `127.0.0.1`. A certificate selects HTTPS. Derived probes bypass environment proxies; HTTPS IP probes trust the configured certificate and verify its first DNS or -IP SAN. Hostname probes use normal verification. +IP SAN. Hostname probes use normal verification. Derived HTTPS probes present +the healthcheck client certificate when one is configured, for a listener that +[requires client certificates](/docs/guides/tls#require-client-certificates). An explicit URL uses normal proxy and certificate verification. Each probe has a three-second timeout and succeeds only on HTTP 200. No auth secret is needed. diff --git a/site/src/content/docs/docs/reference/configuration/server.mdx b/site/src/content/docs/docs/reference/configuration/server.mdx index fc6ad70f2..6ff8e854c 100644 --- a/site/src/content/docs/docs/reference/configuration/server.mdx +++ b/site/src/content/docs/docs/reference/configuration/server.mdx @@ -19,10 +19,13 @@ Vendor credentials and endpoints keep the vendor's documented names. | Environment variable | Type | Default | |---|---|---| | `FANOUT_ADDR` | string | `127.0.0.1:7520` | +| `FANOUT_HEALTHCHECK_CERT_FILE` | string | — | +| `FANOUT_HEALTHCHECK_KEY_FILE` | string | — | | `FANOUT_PPROF_ENABLED` | boolean | `false` | | `FANOUT_PUBLIC_URL` | string | — | | `FANOUT_QUERY_CACHE_TTL` | duration | `10s` | | `FANOUT_TLS_CERT_FILE` | string | — | +| `FANOUT_TLS_CLIENT_CA_FILE` | string | — | | `FANOUT_TLS_KEY_FILE` | string | — | | `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | @@ -32,6 +35,14 @@ Vendor credentials and endpoints keep the vendor's documented names. Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +### `FANOUT_HEALTHCHECK_CERT_FILE` + +Selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. + +### `FANOUT_HEALTHCHECK_KEY_FILE` + +Selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. + ### `FANOUT_PPROF_ENABLED` Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. @@ -48,6 +59,10 @@ The oldest a dashboard panel, exemplar or trace detail result served from memory Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. +### `FANOUT_TLS_CLIENT_CA_FILE` + +Selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. + ### `FANOUT_TLS_KEY_FILE` Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE.