From d7044b2801292974552cf7dd8c0a598a3e52e716 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 7 Nov 2026 10:48:05 -0800 Subject: [PATCH 1/2] feat(server): verify client certificates with FANOUT_TLS_CLIENT_CA_FILE Deployments behind a TLS-terminating proxy such as Cloudflare with Authenticated Origin Pulls need the origin to refuse connections that do not come through the proxy. A server certificate alone cannot do that. FANOUT_TLS_CLIENT_CA_FILE makes the shared listener require and verify a client certificate against the given PEM bundle (tls.RequireAndVerifyClientCert). TLS 1.3 and ALPN are unchanged, so gRPC and HTTP keep sharing the listener. The setting requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. The bundle is loaded strictly before any store opens: an unreadable file, no certificates, a non-certificate block or an invalid certificate fails startup and names the path. fanout healthcheck connects to the same listener, so it would fail once client certificates are required. FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE give the derived probe a client certificate to present. No listener or path skips verification. Both must be set together and require FANOUT_TLS_CLIENT_CA_FILE. Closes #428 --- .env.example | 15 +++ cmd/fanout/main.go | 38 +++++- cmd/fanout/main_test.go | 124 ++++++++++++++++-- docs/operations.md | 9 +- internal/config/config.go | 27 +++- internal/config/config_test.go | 32 +++++ internal/ingest/listener_test.go | 116 +++++++++++++++- internal/server/clientca.go | 53 ++++++++ internal/server/clientca_test.go | 80 +++++++++++ site/src/content/docs/docs/guides/tls.mdx | 46 +++++++ site/src/content/docs/docs/reference/cli.mdx | 9 +- .../docs/reference/configuration/server.mdx | 15 +++ 12 files changed, 536 insertions(+), 28 deletions(-) create mode 100644 internal/server/clientca.go create mode 100644 internal/server/clientca_test.go diff --git a/.env.example b/.env.example index 98144bf5a..e45538562 100644 --- a/.env.example +++ b/.env.example @@ -90,6 +90,16 @@ # Default: (empty) # FANOUT_FIRST_ADMIN_EMAIL= +# FANOUT_HEALTHCHECK_CERT_FILE; string +# Selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. +# Default: (empty) +# FANOUT_HEALTHCHECK_CERT_FILE= + +# FANOUT_HEALTHCHECK_KEY_FILE; string +# Selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. +# Default: (empty) +# FANOUT_HEALTHCHECK_KEY_FILE= + # FANOUT_INGEST_BATCH_SIZE; integer # Sets the group-commit row target; valid values are 1–50000. Individual requests remain atomic and can exceed this target. # Default: 50000 @@ -253,6 +263,11 @@ # Default: (empty) # FANOUT_TLS_CERT_FILE= +# FANOUT_TLS_CLIENT_CA_FILE; string +# Selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. +# Default: (empty) +# FANOUT_TLS_CLIENT_CA_FILE= + # FANOUT_TLS_KEY_FILE; string # Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. # Default: (empty) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index d7bab4280..ecabd0569 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -81,7 +81,8 @@ func main() { } } if healthURL != "" || derivedProbe { - if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE")); err != nil { + if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE"), + os.Getenv("FANOUT_HEALTHCHECK_CERT_FILE"), os.Getenv("FANOUT_HEALTHCHECK_KEY_FILE")); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -110,6 +111,15 @@ func main() { return } cfg.LogStartup() + var clientCAs *x509.CertPool + if cfg.ClientCertificatesRequired() { + // Load before opening any store so a bad bundle fails startup at once. + clientCAs, err = server.LoadClientCAs(cfg.TLSClientCAFile) + if err != nil { + slog.Error("invalid configuration", "err", err) + os.Exit(1) + } + } if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil { slog.Error("create data dir failed", "err", err) @@ -413,8 +423,11 @@ func main() { e.GET("/*", spa) httpSrv := server.New(cfg.Addr, e, ingest.NewHTTPHandler(ing, settingsStore), grpcSrv) + if clientCAs != nil { + server.RequireClientCertificates(httpSrv, clientCAs) + } go func() { - slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled()) + slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled(), "client_certificates", clientCAs != nil) var err error if cfg.TLSEnabled() { err = httpSrv.ListenAndServeTLS(cfg.TLSCertFile, cfg.TLSKeyFile) @@ -579,18 +592,31 @@ func healthcheckURL(addr, certFile string) (string, error) { return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil } -func checkHealth(healthURL string, derivedProbe bool, certFile string) error { +func checkHealth(healthURL string, derivedProbe bool, certFile, clientCertFile, clientKeyFile string) error { client := &http.Client{Timeout: 3 * time.Second} if derivedProbe { // Derived probes bypass proxies. HTTPS IP literals trust only the configured // certificate and verify its first DNS or IP SAN instead of the bind IP. // Hostnames verify normally; explicit URLs retain all default behaviour. + // A configured healthcheck client certificate is presented on derived + // HTTPS probes so the check passes a listener that requires one. probe, err := url.Parse(healthURL) if err != nil { return fmt.Errorf("healthcheck: %w", err) } transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil + if probe.Scheme == "https" && (clientCertFile != "" || clientKeyFile != "") { + if clientCertFile == "" || clientKeyFile == "" { + return fmt.Errorf("healthcheck: FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set") + } + pair, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) + if err != nil { + // Key-pair errors describe structure only, never key material. + return fmt.Errorf("healthcheck: load client certificate %q and key %q: %w", clientCertFile, clientKeyFile, err) + } + transport.TLSClientConfig = &tls.Config{Certificates: []tls.Certificate{pair}} + } if _, err := netip.ParseAddr(probe.Hostname()); probe.Scheme == "https" && err == nil { data, err := os.ReadFile(certFile) if err != nil { @@ -620,7 +646,11 @@ func checkHealth(healthURL string, derivedProbe bool, certFile string) error { } roots := x509.NewCertPool() roots.AddCert(leaf) - transport.TLSClientConfig = &tls.Config{RootCAs: roots, ServerName: serverName} + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = roots + transport.TLSClientConfig.ServerName = serverName } defer transport.CloseIdleConnections() client.Transport = transport diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index 1aff13234..04e28dcaf 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "encoding/pem" "errors" "flag" @@ -27,6 +28,7 @@ import ( "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/server" "github.com/labstack/fanout/internal/store" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" @@ -122,7 +124,7 @@ func TestCheckHealth(t *testing.T) { writer.WriteHeader(test.statusCode) })) defer server.Close() - if err := checkHealth(server.URL, false, ""); (err != nil) != test.wantErr { + if err := checkHealth(server.URL, false, "", "", ""); (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } }) @@ -340,11 +342,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) - if err := checkHealth(server.URL+"/healthz", true, certFile); err != nil { + if err := checkHealth(server.URL+"/healthz", true, certFile, "", ""); err != nil { t.Fatal(err) } var verificationError *tls.CertificateVerificationError - if err := checkHealth(server.URL+"/healthz", false, certFile); !errors.As(err, &verificationError) { + if err := checkHealth(server.URL+"/healthz", false, certFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("explicit URL must verify the certificate, got %v", err) } other := *server.Certificate() @@ -359,11 +361,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) { if err := os.WriteFile(otherFile, chain, 0o600); err != nil { t.Fatal(err) } - if err := checkHealth(server.URL+"/healthz", true, otherFile); !errors.As(err, &verificationError) { + if err := checkHealth(server.URL+"/healthz", true, otherFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("derived IP probe must reject a different certificate, got %v", err) } probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz" - if err := checkHealth(probe, true, certFile); !errors.As(err, &verificationError) { + if err := checkHealth(probe, true, certFile, "", ""); !errors.As(err, &verificationError) { t.Fatalf("derived hostname probe must verify the certificate, got %v", err) } } @@ -390,6 +392,106 @@ func newHealthcheckCertificate(t *testing.T, certificate *x509.Certificate) tls. return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} } +// The derived probe presents FANOUT_HEALTHCHECK_CERT_FILE to a listener that +// requires client certificates, and fails without it. +func TestCheckHealthWithClientCertificate(t *testing.T) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(10), Subject: pkix.Name{CommonName: "healthcheck CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + clientDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(11), Subject: pkix.Name{CommonName: "fanout healthcheck"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, ca, &clientKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + caFile := filepath.Join(dir, "client-ca.pem") + clientCertFile := filepath.Join(dir, "healthcheck.pem") + clientKeyFile := filepath.Join(dir, "healthcheck-key.pem") + keyDER, err := x509.MarshalPKCS8PrivateKey(clientKey) + if err != nil { + t.Fatal(err) + } + for path, block := range map[string]*pem.Block{ + caFile: {Type: "CERTIFICATE", Bytes: caDER}, + clientCertFile: {Type: "CERTIFICATE", Bytes: clientDER}, + clientKeyFile: {Type: "PRIVATE KEY", Bytes: keyDER}, + } { + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil { + t.Fatal(err) + } + } + + ts := httptest.NewUnstartedServer(nil) + ts.Config = server.New("", http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" || len(request.TLS.PeerCertificates) == 0 { + t.Errorf("probe path %q with %d client certificates", request.URL.Path, len(request.TLS.PeerCertificates)) + } + w.WriteHeader(http.StatusOK) + }), http.NotFoundHandler(), http.NotFoundHandler()) + clientCAs, err := server.LoadClientCAs(caFile) + if err != nil { + t.Fatal(err) + } + server.RequireClientCertificates(ts.Config, clientCAs) + ts.TLS = ts.Config.TLSConfig.Clone() + ts.TLS.Certificates = []tls.Certificate{newHealthcheckCertificate(t, &x509.Certificate{ + SerialNumber: big.NewInt(12), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + })} + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + certFile := writeHealthcheckCertificate(t, ts.Certificate().Raw) + + probe, err := healthcheckURL(ts.Listener.Addr().String(), certFile) + if err != nil { + t.Fatal(err) + } + if err := checkHealth(probe, true, certFile, clientCertFile, clientKeyFile); err != nil { + t.Fatalf("healthcheck with client certificate: %v", err) + } + if err := checkHealth(probe, true, certFile, "", ""); err == nil { + t.Fatal("healthcheck without a client certificate must fail") + } + if err := checkHealth(probe, true, certFile, clientCertFile, ""); err == nil || !strings.Contains(err.Error(), "FANOUT_HEALTHCHECK_KEY_FILE") { + t.Fatalf("partial client pair must name both variables, got %v", err) + } + missing := filepath.Join(dir, "missing-key.pem") + if err := checkHealth(probe, true, certFile, clientCertFile, missing); err == nil || !strings.Contains(err.Error(), missing) { + t.Fatalf("unreadable client key must name its path, got %v", err) + } + keyPEM, err := os.ReadFile(clientKeyFile) + if err != nil { + t.Fatal(err) + } + // A key passed as the certificate must fail without echoing key material. + if err := checkHealth(probe, true, certFile, clientKeyFile, clientKeyFile); err == nil || !strings.Contains(err.Error(), clientKeyFile) || strings.Contains(err.Error(), string(keyPEM[40:80])) { + t.Fatalf("invalid client pair must name its path without contents, got %v", err) + } +} + func TestCheckHealthCertificateFileErrors(t *testing.T) { const contents = "private file contents must not appear in errors" for _, test := range []struct { @@ -408,7 +510,7 @@ func TestCheckHealthCertificateFileErrors(t *testing.T) { t.Fatal(err) } } - err := checkHealth("https://127.0.0.1:7520/healthz", true, path) + err := checkHealth("https://127.0.0.1:7520/healthz", true, path, "", "") if err == nil || !strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { t.Fatalf("certificate file error must name its path without contents, got %v", err) } @@ -453,7 +555,7 @@ func TestCheckHealthTLSCertificate(t *testing.T) { server.StartTLS() defer server.Close() certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) - err := checkHealth(server.URL+"/healthz", true, certFile) + err := checkHealth(server.URL+"/healthz", true, certFile, "", "") if (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } @@ -504,7 +606,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkHealth(probe, true, certFile) + err = checkHealth(probe, true, certFile, "", "") var verificationError *tls.CertificateVerificationError if strings.HasSuffix(addr, "example.com:7520") { if !errors.As(err, &verificationError) { @@ -517,7 +619,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkHealth(explicitProbe, derivedProbe, certFile) + err = checkHealth(explicitProbe, derivedProbe, certFile, "", "") if !errors.As(err, &verificationError) { t.Fatalf("explicit TLS probe must use system verification, got %v", err) } @@ -596,10 +698,10 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { if err != nil { t.Fatal(err) } - if err := checkHealth(probe, true, certFile); err != nil { + if err := checkHealth(probe, true, certFile, "", ""); err != nil { t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err) } - if err := checkHealth(probe, false, certFile); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { + if err := checkHealth(probe, false, certFile, "", ""); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err) } }) diff --git a/docs/operations.md b/docs/operations.md index 8becbb864..6cc0931f0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -164,6 +164,9 @@ Setting both `FANOUT_TLS_CERT_FILE` and `FANOUT_TLS_KEY_FILE` enables TLS 1.3 on the shared listener. A proxy must forward HTTP/2 to Fanout for gRPC: use h2c for a plaintext backend, or HTTP/2 over TLS when Fanout has a certificate. HTTP/1.1 forwarding supports OTLP/HTTP, but cannot carry gRPC. +`FANOUT_TLS_CLIENT_CA_FILE` additionally requires every client, including a +TLS-terminating proxy such as Cloudflare with Authenticated Origin Pulls, to +present a certificate from that CA bundle. Set `FANOUT_PUBLIC_URL` to the externally reachable HTTPS browser origin. When using forwarded client addresses, set `FANOUT_TRUSTED_PROXY_CIDRS` to only the @@ -182,8 +185,10 @@ default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`. `FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies. HTTPS IP-literal probes trust only the configured certificate and verify its first DNS or IP SAN, which may differ from the bind IP; hostname probes verify -normally. Healthcheck reads only these two settings and does not require valid -secrets. An explicit `fanout healthcheck ` uses default proxy and +normally. Derived HTTPS probes present `FANOUT_HEALTHCHECK_CERT_FILE` and +`FANOUT_HEALTHCHECK_KEY_FILE` when set, so they pass a listener with +`FANOUT_TLS_CLIENT_CA_FILE`. Healthcheck +reads only these settings and does not require valid secrets. An explicit `fanout healthcheck ` uses default proxy and certificate verification behaviour. - `/healthz` is the liveness check. It fails only when Fanout's own work on diff --git a/internal/config/config.go b/internal/config/config.go index df8eac552..b016f203b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -205,8 +205,14 @@ type Config struct { // TLSCertFile selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. TLSCertFile string `group:"server" env:"FANOUT_TLS_CERT_FILE"` // TLSKeyFile selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. - TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` - resolvedSizing sizingSource + TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` + // TLSClientCAFile selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. + TLSClientCAFile string `group:"server" env:"FANOUT_TLS_CLIENT_CA_FILE"` + // HealthcheckCertFile selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. + HealthcheckCertFile string `group:"server" env:"FANOUT_HEALTHCHECK_CERT_FILE"` + // HealthcheckKeyFile selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. + HealthcheckKeyFile string `group:"server" env:"FANOUT_HEALTHCHECK_KEY_FILE"` + resolvedSizing sizingSource } // LogStartup reports the effective non-secret sizing and security-sensitive @@ -262,6 +268,12 @@ func (c Config) TLSEnabled() bool { strings.TrimSpace(c.TLSKeyFile) != "" } +// ClientCertificatesRequired reports whether the TLS listener requires and +// verifies client certificates against FANOUT_TLS_CLIENT_CA_FILE. +func (c Config) ClientCertificatesRequired() bool { + return c.TLSEnabled() && strings.TrimSpace(c.TLSClientCAFile) != "" +} + // Validate checks that config values are sane. func (c Config) Validate() error { if strings.TrimSpace(c.Addr) == "" { @@ -401,6 +413,17 @@ func (c Config) Validate() error { if anySet(c.TLSCertFile, c.TLSKeyFile) && !c.TLSEnabled() { return fmt.Errorf("FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE must both be set") } + if anySet(c.TLSClientCAFile) && !c.TLSEnabled() { + return fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE") + } + if anySet(c.HealthcheckCertFile, c.HealthcheckKeyFile) { + if !anySet(c.HealthcheckCertFile) || !anySet(c.HealthcheckKeyFile) { + return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set") + } + if !anySet(c.TLSClientCAFile) { + return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE require FANOUT_TLS_CLIENT_CA_FILE") + } + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 96981f55e..81c3488f4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -306,6 +306,20 @@ func TestValidate(t *testing.T) { {"trusted proxy invalid CIDR", func(c *Config) { c.TrustedProxyCIDRs = "private-network" }}, {"TLS partial cert", func(c *Config) { c.TLSCertFile = "server.pem" }}, {"TLS partial key", func(c *Config) { c.TLSKeyFile = "server-key.pem" }}, + {"TLS client CA without TLS", func(c *Config) { c.TLSClientCAFile = "client-ca.pem" }}, + {"TLS client CA without key", func(c *Config) { c.TLSCertFile = "server.pem"; c.TLSClientCAFile = "client-ca.pem" }}, + {"healthcheck cert without key", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + c.HealthcheckCertFile = "probe.pem" + }}, + {"healthcheck key without cert", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + c.HealthcheckKeyFile = "probe-key.pem" + }}, + {"healthcheck pair without client CA", func(c *Config) { + c.TLSCertFile, c.TLSKeyFile = "server.pem", "server-key.pem" + c.HealthcheckCertFile, c.HealthcheckKeyFile = "probe.pem", "probe-key.pem" + }}, } for _, tc := range tests { @@ -413,6 +427,24 @@ func TestValidate(t *testing.T) { if !c.TLSEnabled() { t.Error("TLSEnabled = false, want true") } + if c.ClientCertificatesRequired() { + t.Error("ClientCertificatesRequired = true without FANOUT_TLS_CLIENT_CA_FILE") + } + }) + + t.Run("TLS_client_CA_valid", func(t *testing.T) { + c := valid + c.TLSCertFile, c.TLSKeyFile, c.TLSClientCAFile = "server.pem", "server-key.pem", "client-ca.pem" + if err := c.Validate(); err != nil { + t.Errorf("client CA config should be valid: %v", err) + } + if !c.ClientCertificatesRequired() { + t.Error("ClientCertificatesRequired = false, want true") + } + c.HealthcheckCertFile, c.HealthcheckKeyFile = "probe.pem", "probe-key.pem" + if err := c.Validate(); err != nil { + t.Errorf("healthcheck client certificate config should be valid: %v", err) + } }) } diff --git a/internal/ingest/listener_test.go b/internal/ingest/listener_test.go index ce5911932..96bc1b8bd 100644 --- a/internal/ingest/listener_test.go +++ b/internal/ingest/listener_test.go @@ -3,10 +3,15 @@ package ingest import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "errors" "io" + "math/big" "net/http" "net/http/httptest" "strings" @@ -29,8 +34,8 @@ import ( ) func TestSharedListenerProtocols(t *testing.T) { - for _, secure := range []bool{false, true} { - t.Run(map[bool]string{false: "plaintext", true: "TLS"}[secure], func(t *testing.T) { + for _, mode := range []string{"plaintext", "TLS", "mutual TLS"} { + t.Run(mode, func(t *testing.T) { f := newHTTPIngestFixture(t, true) g := grpc.NewServer(GRPCServerOptions(f.store)...) RegisterOTLP(g, NewServer(config.Config{}, newTestSubmitter(f.spans, f.logs, f.metrics))) @@ -39,14 +44,23 @@ func TestSharedListenerProtocols(t *testing.T) { ts := httptest.NewUnstartedServer(nil) ts.Config = httpserver.New("", app, f.handler, g) creds := insecure.NewCredentials() - if secure { + if mode == "plaintext" { + ts.Start() + } else { + var clientCertificates []tls.Certificate + if mode == "mutual TLS" { + clientCAs, clientCertificate := newTestClientCA(t) + httpserver.RequireClientCertificates(ts.Config, clientCAs) + // StartTLS replaces the server's TLS config with ts.TLS. + ts.TLS = ts.Config.TLSConfig.Clone() + clientCertificates = []tls.Certificate{clientCertificate} + } ts.EnableHTTP2 = true ts.StartTLS() roots := x509.NewCertPool() roots.AddCert(ts.Certificate()) - creds = credentials.NewTLS(&tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13}) - } else { - ts.Start() + creds = credentials.NewTLS(&tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, Certificates: clientCertificates}) + ts.Client().Transport.(*http.Transport).TLSClientConfig.Certificates = clientCertificates } defer ts.Close() conn, err := grpc.NewClient(ts.Listener.Addr().String(), grpc.WithTransportCredentials(creds)) @@ -128,6 +142,96 @@ func TestSharedListenerProtocols(t *testing.T) { } } +// The listener must refuse gRPC and HTTP handshakes that present no client +// certificate or one from an untrusted CA when client certificates are required. +func TestSharedListenerRejectsMissingClientCertificate(t *testing.T) { + f := newHTTPIngestFixture(t, true) + g := grpc.NewServer(GRPCServerOptions(f.store)...) + RegisterOTLP(g, NewServer(config.Config{}, newTestSubmitter(f.spans, f.logs, f.metrics))) + defer g.Stop() + ts := httptest.NewUnstartedServer(nil) + ts.Config = httpserver.New("", http.NotFoundHandler(), f.handler, g) + clientCAs, _ := newTestClientCA(t) + httpserver.RequireClientCertificates(ts.Config, clientCAs) + ts.TLS = ts.Config.TLSConfig.Clone() + ts.EnableHTTP2 = true + ts.StartTLS() + defer ts.Close() + if ts.TLS.MinVersion != tls.VersionTLS13 || ts.TLS.ClientAuth != tls.RequireAndVerifyClientCert { + t.Fatalf("TLS config = min %x auth %v", ts.TLS.MinVersion, ts.TLS.ClientAuth) + } + _, untrusted := newTestClientCA(t) + for name, certificates := range map[string][]tls.Certificate{"none": nil, "untrusted": {untrusted}} { + t.Run(name, func(t *testing.T) { + roots := x509.NewCertPool() + roots.AddCert(ts.Certificate()) + clientTLS := &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, Certificates: certificates} + conn, err := grpc.NewClient(ts.Listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(clientTLS.Clone()))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+f.token) + if _, err := collectortrace.NewTraceServiceClient(conn).Export(ctx, testTraceRequest()); status.Code(err) != codes.Unavailable { + t.Fatalf("gRPC without a trusted client certificate: %v", err) + } + transport := &http.Transport{TLSClientConfig: clientTLS.Clone(), ForceAttemptHTTP2: true} + defer transport.CloseIdleConnections() + resp, err := (&http.Client{Transport: transport, Timeout: 5 * time.Second}).Get(ts.URL + "/v1/traces") + if err == nil { + _ = resp.Body.Close() + t.Fatalf("HTTP without a trusted client certificate: %s", resp.Status) + } + }) + } + select { + case <-f.spans: + t.Fatal("rejected handshake delivered telemetry") + default: + } +} + +// newTestClientCA returns a pool holding a fresh CA and a client certificate +// that CA issued. +func newTestClientCA(t *testing.T) (*x509.CertPool, tls.Certificate) { + t.Helper() + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test client CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + clientTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "test client"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, ca, &clientKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(ca) + return pool, tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} +} + type blockedExport struct{ started, release chan struct{} } func (b *blockedExport) Submit(ctx context.Context, _ telemetrystore.Batch) error { diff --git a/internal/server/clientca.go b/internal/server/clientca.go new file mode 100644 index 000000000..8c8e5c46a --- /dev/null +++ b/internal/server/clientca.go @@ -0,0 +1,53 @@ +package server + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "net/http" + "os" +) + +// RequireClientCertificates makes srv's TLS listener require and verify client +// certificates issued by a CA in pool. TLS 1.3 and ALPN (h2 and http/1.1) are +// unchanged, so gRPC and HTTP keep sharing the listener. +func RequireClientCertificates(srv *http.Server, pool *x509.CertPool) { + if srv.TLSConfig == nil { + srv.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13} + } + srv.TLSConfig.ClientCAs = pool + srv.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert +} + +// LoadClientCAs reads the FANOUT_TLS_CLIENT_CA_FILE bundle. It rejects every +// block it cannot use instead of silently skipping it, so a truncated or wrong +// file fails startup. Errors name the path, never file contents. +func LoadClientCAs(path string) (*x509.CertPool, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: read %q: %w", path, err) + } + pool := x509.NewCertPool() + count := 0 + for { + var block *pem.Block + block, data = pem.Decode(data) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: %q contains a %q PEM block; only certificates are allowed", path, block.Type) + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: parse certificate %d in %q: %w", count+1, path, err) + } + pool.AddCert(certificate) + count++ + } + if count == 0 { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: no PEM certificates in %q", path) + } + return pool, nil +} diff --git a/internal/server/clientca_test.go b/internal/server/clientca_test.go new file mode 100644 index 000000000..64789608a --- /dev/null +++ b/internal/server/clientca_test.go @@ -0,0 +1,80 @@ +package server + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadClientCAs(t *testing.T) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "client CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + const contents = "private file contents must not appear in errors" + for _, test := range []struct { + name string + data []byte + wantErr bool + }{ + {name: "one certificate", data: certificate}, + {name: "bundle", data: append(append([]byte{}, certificate...), certificate...)}, + {name: "unreadable", wantErr: true}, + {name: "empty", data: []byte{}, wantErr: true}, + {name: "not PEM", data: []byte(contents), wantErr: true}, + {name: "private key block", data: append(append([]byte{}, certificate...), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte(contents)})...), wantErr: true}, + {name: "invalid certificate", data: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte(contents)}), wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "client-ca.pem") + if test.data != nil { + if err := os.WriteFile(path, test.data, 0o600); err != nil { + t.Fatal(err) + } + } + pool, err := LoadClientCAs(path) + if !test.wantErr { + if err != nil || pool == nil { + t.Fatalf("LoadClientCAs() = %v, %v", pool, err) + } + return + } + if err == nil || !strings.HasPrefix(err.Error(), "FANOUT_TLS_CLIENT_CA_FILE:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { + t.Fatalf("error must name the variable and path without contents, got %v", err) + } + }) + } +} + +func TestRequireClientCertificatesKeepsProtocols(t *testing.T) { + srv := New("", http.NotFoundHandler(), http.NotFoundHandler(), http.NotFoundHandler()) + pool := x509.NewCertPool() + RequireClientCertificates(srv, pool) + if srv.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert || srv.TLSConfig.ClientCAs != pool { + t.Fatalf("client auth = %v, pool set = %v", srv.TLSConfig.ClientAuth, srv.TLSConfig.ClientCAs == pool) + } + if srv.TLSConfig.MinVersion != tls.VersionTLS13 || !srv.Protocols.HTTP2() || !srv.Protocols.HTTP1() { + t.Fatalf("TLS min version %x or protocols changed", srv.TLSConfig.MinVersion) + } +} diff --git a/site/src/content/docs/docs/guides/tls.mdx b/site/src/content/docs/docs/guides/tls.mdx index d50baebeb..7e9d50130 100644 --- a/site/src/content/docs/docs/guides/tls.mdx +++ b/site/src/content/docs/docs/guides/tls.mdx @@ -23,6 +23,52 @@ export FANOUT_TLS_KEY_FILE=/etc/fanout/tls/key.pem That enables TLS 1.3 on the shared listener. Browser traffic and OTLP/HTTP use HTTPS; gRPC uses HTTP/2 negotiated through ALPN. +### Require client certificates + +To accept connections only from clients holding a certificate from a CA you +choose, add a PEM bundle of those CA certificates: + +```sh +export FANOUT_TLS_CLIENT_CA_FILE=/etc/fanout/tls/client-ca.pem +``` + +The listener then requires and verifies a client certificate on every +handshake, for the browser, API, MCP, OTLP/HTTP and gRPC alike. TLS 1.3 and +ALPN are unchanged. A handshake without a certificate from a listed CA fails +before any request is read. The setting requires `FANOUT_TLS_CERT_FILE` and +`FANOUT_TLS_KEY_FILE`; an unreadable bundle, or one with no certificates, a +non-certificate block or an invalid certificate, fails startup and names the path. + +A typical use is +[Cloudflare Authenticated Origin Pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/): +Cloudflare presents its client certificate when it connects to your origin, so +setting `FANOUT_TLS_CLIENT_CA_FILE` to Cloudflare's origin pull CA rejects +connections that bypass Cloudflare. With +[zone-level pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/set-up/zone-level/), +every Cloudflare zone presents that certificate, so the CA proves that the +connection came through Cloudflare, not through your zone. Use a +per-hostname certificate to restrict connections to your own zone. + +Collectors and agents that connect directly, rather than through that proxy, +need their own client certificate from a CA in the bundle. + +#### Health checks + +`fanout healthcheck` (the container image's `HEALTHCHECK`) connects to the same +listener, so it also needs a client certificate. Issue one from a private CA +you control, append that CA to the bundle, and give the probe the pair: + +```sh +export FANOUT_HEALTHCHECK_CERT_FILE=/etc/fanout/tls/healthcheck.pem +export FANOUT_HEALTHCHECK_KEY_FILE=/etc/fanout/tls/healthcheck-key.pem +``` + +The probe presents it only on its derived HTTPS check of `FANOUT_ADDR`; an +explicit `fanout healthcheck ` does not. Keep that CA dedicated to Fanout clients: +the listener accepts any certificate it issues. Both settings need +`FANOUT_TLS_CLIENT_CA_FILE`, and supplying only one fails. Without them, the +probe's handshake is rejected and the health check fails. + ## Behind a reverse proxy Terminate at the proxy and forward to Fanout's shared port (`7520` by default). Two settings have to diff --git a/site/src/content/docs/docs/reference/cli.mdx b/site/src/content/docs/docs/reference/cli.mdx index 7db7fc092..e5f953481 100644 --- a/site/src/content/docs/docs/reference/cli.mdx +++ b/site/src/content/docs/docs/reference/cli.mdx @@ -42,11 +42,14 @@ works once. It can run while the server is online. See ## Health probes -With no URL, `healthcheck` reads only `FANOUT_ADDR` and -`FANOUT_TLS_CERT_FILE`. Unspecified bind hosts become `127.0.0.1`. +With no URL, `healthcheck` reads only `FANOUT_ADDR`, +`FANOUT_TLS_CERT_FILE`, `FANOUT_HEALTHCHECK_CERT_FILE` and +`FANOUT_HEALTHCHECK_KEY_FILE`. Unspecified bind hosts become `127.0.0.1`. A certificate selects HTTPS. Derived probes bypass environment proxies; HTTPS IP probes trust the configured certificate and verify its first DNS or -IP SAN. Hostname probes use normal verification. +IP SAN. Hostname probes use normal verification. Derived HTTPS probes present +the healthcheck client certificate when one is configured, for a listener that +[requires client certificates](/docs/guides/tls#require-client-certificates). An explicit URL uses normal proxy and certificate verification. Each probe has a three-second timeout and succeeds only on HTTP 200. No auth secret is needed. diff --git a/site/src/content/docs/docs/reference/configuration/server.mdx b/site/src/content/docs/docs/reference/configuration/server.mdx index fc6ad70f2..6ff8e854c 100644 --- a/site/src/content/docs/docs/reference/configuration/server.mdx +++ b/site/src/content/docs/docs/reference/configuration/server.mdx @@ -19,10 +19,13 @@ Vendor credentials and endpoints keep the vendor's documented names. | Environment variable | Type | Default | |---|---|---| | `FANOUT_ADDR` | string | `127.0.0.1:7520` | +| `FANOUT_HEALTHCHECK_CERT_FILE` | string | — | +| `FANOUT_HEALTHCHECK_KEY_FILE` | string | — | | `FANOUT_PPROF_ENABLED` | boolean | `false` | | `FANOUT_PUBLIC_URL` | string | — | | `FANOUT_QUERY_CACHE_TTL` | duration | `10s` | | `FANOUT_TLS_CERT_FILE` | string | — | +| `FANOUT_TLS_CLIENT_CA_FILE` | string | — | | `FANOUT_TLS_KEY_FILE` | string | — | | `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | @@ -32,6 +35,14 @@ Vendor credentials and endpoints keep the vendor's documented names. Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +### `FANOUT_HEALTHCHECK_CERT_FILE` + +Selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE. + +### `FANOUT_HEALTHCHECK_KEY_FILE` + +Selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE. + ### `FANOUT_PPROF_ENABLED` Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. @@ -48,6 +59,10 @@ The oldest a dashboard panel, exemplar or trace detail result served from memory Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. +### `FANOUT_TLS_CLIENT_CA_FILE` + +Selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE. + ### `FANOUT_TLS_KEY_FILE` Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. From 5e8e25b91d7b9b7ed47d915f2c0e19ead6f91ad7 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 7 Nov 2026 11:29:16 -0800 Subject: [PATCH 2/2] fix(server): reject stray PEM data and correct the AOP guidance Review follow-up for client certificate verification. The TLS guide described zone-level Authenticated Origin Pulls as using a shared Cloudflare certificate. That is global pulls; zone-level and per-hostname pulls use a certificate you upload, exclusive to your account. Recommend those with your own CA, warn that trusting the global Cloudflare CA admits any Cloudflare customer's zone, issue the healthcheck certificate from the same CA when you uploaded your own, and document that the bundle is read at startup so rotation needs a restart with old and new CA listed during the overlap. LoadClientCAs documented that it rejects every unusable block, but pem.Decode silently skips text around blocks and malformed blocks. Accept only whitespace-separated CERTIFICATE blocks and fail on anything else. Add handshake tests showing the stdlib verifier rejects a trusted certificate with only the serverAuth usage, accepts a leaf sent with its intermediate when only the root is in the bundle, and rejects the leaf without that intermediate. Refs #428 --- docs/operations.md | 10 +- internal/server/clientca.go | 24 +++- internal/server/clientca_test.go | 135 ++++++++++++++++++++++ site/src/content/docs/docs/guides/tls.mdx | 42 ++++--- 4 files changed, 188 insertions(+), 23 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 6cc0931f0..83a637303 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -166,7 +166,9 @@ h2c for a plaintext backend, or HTTP/2 over TLS when Fanout has a certificate. HTTP/1.1 forwarding supports OTLP/HTTP, but cannot carry gRPC. `FANOUT_TLS_CLIENT_CA_FILE` additionally requires every client, including a TLS-terminating proxy such as Cloudflare with Authenticated Origin Pulls, to -present a certificate from that CA bundle. +present a certificate from that CA bundle. The bundle is read at startup, so +rotation needs a restart; keep the old and new CA in the bundle while clients +move to certificates from the new one. Set `FANOUT_PUBLIC_URL` to the externally reachable HTTPS browser origin. When using forwarded client addresses, set `FANOUT_TRUSTED_PROXY_CIDRS` to only the @@ -187,9 +189,9 @@ HTTPS IP-literal probes trust only the configured certificate and verify its first DNS or IP SAN, which may differ from the bind IP; hostname probes verify normally. Derived HTTPS probes present `FANOUT_HEALTHCHECK_CERT_FILE` and `FANOUT_HEALTHCHECK_KEY_FILE` when set, so they pass a listener with -`FANOUT_TLS_CLIENT_CA_FILE`. Healthcheck -reads only these settings and does not require valid secrets. An explicit `fanout healthcheck ` uses default proxy and -certificate verification behaviour. +`FANOUT_TLS_CLIENT_CA_FILE`. Healthcheck reads only these settings and does +not require valid secrets. An explicit `fanout healthcheck ` uses default +proxy and certificate verification behaviour. - `/healthz` is the liveness check. It fails only when Fanout's own work on the query engine stalls: a Parquet publication holds reads for 10 minutes, diff --git a/internal/server/clientca.go b/internal/server/clientca.go index 8c8e5c46a..a5ecb5056 100644 --- a/internal/server/clientca.go +++ b/internal/server/clientca.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "crypto/tls" "crypto/x509" "encoding/pem" @@ -20,9 +21,11 @@ func RequireClientCertificates(srv *http.Server, pool *x509.CertPool) { srv.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert } -// LoadClientCAs reads the FANOUT_TLS_CLIENT_CA_FILE bundle. It rejects every -// block it cannot use instead of silently skipping it, so a truncated or wrong -// file fails startup. Errors name the path, never file contents. +// LoadClientCAs reads the FANOUT_TLS_CLIENT_CA_FILE bundle: one or more PEM +// CERTIFICATE blocks separated only by whitespace. It rejects anything else, +// including text around the blocks and malformed blocks that pem.Decode would +// otherwise skip, so a truncated or wrong file fails startup. Errors name the +// path, never file contents. func LoadClientCAs(path string) (*x509.CertPool, error) { data, err := os.ReadFile(path) if err != nil { @@ -31,11 +34,17 @@ func LoadClientCAs(path string) (*x509.CertPool, error) { pool := x509.NewCertPool() count := 0 for { - var block *pem.Block - block, data = pem.Decode(data) - if block == nil { + data = bytes.TrimLeft(data, " \t\r\n") + if len(data) == 0 { break } + block, rest := pem.Decode(data) + consumed := data[:len(data)-len(rest)] + // pem.Decode skips leading text and malformed blocks; only accept a + // block that starts the remaining input and is the only one consumed. + if block == nil || !bytes.HasPrefix(data, pemBegin) || bytes.Count(consumed, pemBegin) != 1 { + return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: %q contains data that is not a PEM block after certificate %d", path, count) + } if block.Type != "CERTIFICATE" { return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: %q contains a %q PEM block; only certificates are allowed", path, block.Type) } @@ -45,9 +54,12 @@ func LoadClientCAs(path string) (*x509.CertPool, error) { } pool.AddCert(certificate) count++ + data = rest } if count == 0 { return nil, fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE: no PEM certificates in %q", path) } return pool, nil } + +var pemBegin = []byte("-----BEGIN ") diff --git a/internal/server/clientca_test.go b/internal/server/clientca_test.go index 64789608a..1ca87228a 100644 --- a/internal/server/clientca_test.go +++ b/internal/server/clientca_test.go @@ -8,7 +8,9 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "math/big" + "net" "net/http" "os" "path/filepath" @@ -45,6 +47,11 @@ func TestLoadClientCAs(t *testing.T) { {name: "not PEM", data: []byte(contents), wantErr: true}, {name: "private key block", data: append(append([]byte{}, certificate...), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte(contents)})...), wantErr: true}, {name: "invalid certificate", data: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte(contents)}), wantErr: true}, + {name: "surrounding whitespace", data: append(append([]byte("\n \r\n"), certificate...), "\n\n"...)}, + {name: "trailing text", data: append(append([]byte{}, certificate...), contents...), wantErr: true}, + {name: "leading text", data: append([]byte(contents+"\n"), certificate...), wantErr: true}, + {name: "malformed block before certificate", data: append([]byte("-----BEGIN CERTIFICATE-----\n!!!\n-----END CERTIFICATE-----\n"), certificate...), wantErr: true}, + {name: "truncated certificate", data: certificate[:len(certificate)-30], wantErr: true}, } { t.Run(test.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "client-ca.pem") @@ -67,6 +74,134 @@ func TestLoadClientCAs(t *testing.T) { } } +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T, name string, parent *testCA) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + issuer, signer := template, key + if parent != nil { + issuer, signer = parent.cert, parent.key + } + der, err := x509.CreateCertificate(rand.Reader, template, issuer, &key.PublicKey, signer) + if err != nil { + t.Fatal(err) + } + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return &testCA{cert: certificate, key: key} +} + +// issue returns a leaf certificate followed by any intermediates. +func (ca *testCA) issue(t *testing.T, usage []x509.ExtKeyUsage, ips []net.IP, chain ...*testCA) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usage, IPAddresses: ips, + }, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + certificate := tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} + for _, intermediate := range chain { + certificate.Certificate = append(certificate.Certificate, intermediate.cert.Raw) + } + return certificate +} + +// The stdlib verifier enforces the client-auth EKU and builds chains through +// intermediates the client sends, with only the root in the bundle. +func TestRequireClientCertificatesHandshake(t *testing.T) { + root := newTestCA(t, "client root", nil) + intermediate := newTestCA(t, "client intermediate", root) + serverCA := newTestCA(t, "server CA", nil) + clientAuth := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + for _, test := range []struct { + name string + client []tls.Certificate + wantErr string + }{ + {name: "client auth from root", client: []tls.Certificate{root.issue(t, clientAuth, nil)}}, + {name: "leaf and intermediate", client: []tls.Certificate{intermediate.issue(t, clientAuth, nil, intermediate)}}, + {name: "leaf without its intermediate", client: []tls.Certificate{intermediate.issue(t, clientAuth, nil)}, wantErr: "unknown authority"}, + {name: "server auth only", client: []tls.Certificate{root.issue(t, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, nil)}, wantErr: "incompatible key usage"}, + {name: "no certificate", wantErr: "didn't provide a certificate"}, + } { + t.Run(test.name, func(t *testing.T) { + srv := New("", http.NotFoundHandler(), http.NotFoundHandler(), http.NotFoundHandler()) + pool := x509.NewCertPool() + pool.AddCert(root.cert) + RequireClientCertificates(srv, pool) + serverTLS := srv.TLSConfig.Clone() + serverTLS.Certificates = []tls.Certificate{serverCA.issue(t, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []net.IP{net.ParseIP("127.0.0.1")})} + serverTLS.NextProtos = []string{"h2", "http/1.1"} + roots := x509.NewCertPool() + roots.AddCert(serverCA.cert) + // A buffered loopback socket, unlike net.Pipe, lets the server + // write its alert after the client's handshake has returned. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverErr := make(chan error, 1) + go func() { + serverConn, err := listener.Accept() + if err != nil { + serverErr <- err + return + } + _ = serverConn.SetDeadline(time.Now().Add(10 * time.Second)) + conn := tls.Server(serverConn, serverTLS) + err = conn.Handshake() + if err == nil && conn.ConnectionState().NegotiatedProtocol != "h2" { + err = fmt.Errorf("ALPN = %q, want h2", conn.ConnectionState().NegotiatedProtocol) + } + serverErr <- err + _ = conn.Close() + }() + clientConn, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer clientConn.Close() + _ = clientConn.SetDeadline(time.Now().Add(10 * time.Second)) + clientTLS := &tls.Config{RootCAs: roots, ServerName: "127.0.0.1", MinVersion: tls.VersionTLS13, NextProtos: []string{"h2"}} + if len(test.client) > 0 { + // Always send the certificate, even when its issuer is not among + // the server's acceptable CAs, so the server verifier decides. + clientTLS.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return &test.client[0], nil } + } + client := tls.Client(clientConn, clientTLS) + _ = client.Handshake() + // TLS 1.3 clients finish before the server checks their certificate, + // so the server's result decides. + err = <-serverErr + if test.wantErr == "" && err != nil || test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("server handshake error = %v, want %q", err, test.wantErr) + } + }) + } +} + func TestRequireClientCertificatesKeepsProtocols(t *testing.T) { srv := New("", http.NotFoundHandler(), http.NotFoundHandler(), http.NotFoundHandler()) pool := x509.NewCertPool() diff --git a/site/src/content/docs/docs/guides/tls.mdx b/site/src/content/docs/docs/guides/tls.mdx index 7e9d50130..dee85d841 100644 --- a/site/src/content/docs/docs/guides/tls.mdx +++ b/site/src/content/docs/docs/guides/tls.mdx @@ -39,15 +39,26 @@ before any request is read. The setting requires `FANOUT_TLS_CERT_FILE` and `FANOUT_TLS_KEY_FILE`; an unreadable bundle, or one with no certificates, a non-certificate block or an invalid certificate, fails startup and names the path. +The bundle is read once at startup, so restart Fanout after changing it. To +rotate a CA without rejecting clients, list both the old and new CA in the +bundle until every client has a certificate from the new one, then remove the +old CA and restart again. + A typical use is -[Cloudflare Authenticated Origin Pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/): -Cloudflare presents its client certificate when it connects to your origin, so -setting `FANOUT_TLS_CLIENT_CA_FILE` to Cloudflare's origin pull CA rejects -connections that bypass Cloudflare. With -[zone-level pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/set-up/zone-level/), -every Cloudflare zone presents that certificate, so the CA proves that the -connection came through Cloudflare, not through your zone. Use a -per-hostname certificate to restrict connections to your own zone. +[Cloudflare Authenticated Origin Pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/), +where Cloudflare presents a client certificate on every connection to your +origin. Use zone-level or per-hostname pulls: you upload a client certificate +issued by your own CA to Cloudflare, and set `FANOUT_TLS_CLIENT_CA_FILE` to +that CA. The certificate is exclusive to your account, so the origin accepts +only connections that Cloudflare makes on behalf of your zone or hostname. +Connections that bypass Cloudflare are rejected. + +:::caution +Do not trust the Cloudflare-provided certificate used by +[global pulls](https://developers.cloudflare.com/ssl/origin-configuration/authenticated-origin-pull/set-up/global/). +It is shared by all Cloudflare accounts and proves only that a connection came +from Cloudflare's network, so any Cloudflare customer's zone could reach your origin. +::: Collectors and agents that connect directly, rather than through that proxy, need their own client certificate from a CA in the bundle. @@ -55,17 +66,22 @@ need their own client certificate from a CA in the bundle. #### Health checks `fanout healthcheck` (the container image's `HEALTHCHECK`) connects to the same -listener, so it also needs a client certificate. Issue one from a private CA -you control, append that CA to the bundle, and give the probe the pair: +listener, so it also needs a client certificate. Give the probe a certificate +and key: ```sh export FANOUT_HEALTHCHECK_CERT_FILE=/etc/fanout/tls/healthcheck.pem export FANOUT_HEALTHCHECK_KEY_FILE=/etc/fanout/tls/healthcheck-key.pem ``` -The probe presents it only on its derived HTTPS check of `FANOUT_ADDR`; an -explicit `fanout healthcheck ` does not. Keep that CA dedicated to Fanout clients: -the listener accepts any certificate it issues. Both settings need +If you uploaded your own CA's certificate for Authenticated Origin Pulls, issue +the probe certificate from that same CA; the bundle already trusts it. +Otherwise, issue it from a private CA you control and append that CA to the +bundle. Keep any CA you append dedicated to Fanout clients: the listener +accepts every certificate it issues. + +The probe presents the pair only on its derived HTTPS check of `FANOUT_ADDR`; +an explicit `fanout healthcheck ` does not. Both settings need `FANOUT_TLS_CLIENT_CA_FILE`, and supplying only one fails. Without them, the probe's handshake is rejected and the health check fails.