Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@
# Default: (empty)
# FANOUT_FIRST_ADMIN_EMAIL=

# FANOUT_HEALTHCHECK_CERT_FILE; string
# Selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE.
# Default: (empty)
# FANOUT_HEALTHCHECK_CERT_FILE=

# FANOUT_HEALTHCHECK_KEY_FILE; string
# Selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE.
# Default: (empty)
# FANOUT_HEALTHCHECK_KEY_FILE=

# FANOUT_INGEST_BATCH_SIZE; integer
# Sets the group-commit row target; valid values are 1–50000. Individual requests remain atomic and can exceed this target.
# Default: 50000
Expand Down Expand Up @@ -253,6 +263,11 @@
# Default: (empty)
# FANOUT_TLS_CERT_FILE=

# FANOUT_TLS_CLIENT_CA_FILE; string
# Selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE.
# Default: (empty)
# FANOUT_TLS_CLIENT_CA_FILE=

# FANOUT_TLS_KEY_FILE; string
# Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE.
# Default: (empty)
Expand Down
38 changes: 34 additions & 4 deletions cmd/fanout/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ func main() {
}
}
if healthURL != "" || derivedProbe {
if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE")); err != nil {
if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE"),
os.Getenv("FANOUT_HEALTHCHECK_CERT_FILE"), os.Getenv("FANOUT_HEALTHCHECK_KEY_FILE")); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
Expand Down Expand Up @@ -110,6 +111,15 @@ func main() {
return
}
cfg.LogStartup()
var clientCAs *x509.CertPool
if cfg.ClientCertificatesRequired() {
// Load before opening any store so a bad bundle fails startup at once.
clientCAs, err = server.LoadClientCAs(cfg.TLSClientCAFile)
if err != nil {
slog.Error("invalid configuration", "err", err)
os.Exit(1)
}
}

if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil {
slog.Error("create data dir failed", "err", err)
Expand Down Expand Up @@ -413,8 +423,11 @@ func main() {
e.GET("/*", spa)

httpSrv := server.New(cfg.Addr, e, ingest.NewHTTPHandler(ing, settingsStore), grpcSrv)
if clientCAs != nil {
server.RequireClientCertificates(httpSrv, clientCAs)
}
go func() {
slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled())
slog.Info("HTTP and OTLP listening", "addr", cfg.Addr, "tls", cfg.TLSEnabled(), "client_certificates", clientCAs != nil)
var err error
if cfg.TLSEnabled() {
err = httpSrv.ListenAndServeTLS(cfg.TLSCertFile, cfg.TLSKeyFile)
Expand Down Expand Up @@ -579,18 +592,31 @@ func healthcheckURL(addr, certFile string) (string, error) {
return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil
}

func checkHealth(healthURL string, derivedProbe bool, certFile string) error {
func checkHealth(healthURL string, derivedProbe bool, certFile, clientCertFile, clientKeyFile string) error {
client := &http.Client{Timeout: 3 * time.Second}
if derivedProbe {
// Derived probes bypass proxies. HTTPS IP literals trust only the configured
// certificate and verify its first DNS or IP SAN instead of the bind IP.
// Hostnames verify normally; explicit URLs retain all default behaviour.
// A configured healthcheck client certificate is presented on derived
// HTTPS probes so the check passes a listener that requires one.
probe, err := url.Parse(healthURL)
if err != nil {
return fmt.Errorf("healthcheck: %w", err)
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.Proxy = nil
if probe.Scheme == "https" && (clientCertFile != "" || clientKeyFile != "") {
if clientCertFile == "" || clientKeyFile == "" {
return fmt.Errorf("healthcheck: FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set")
}
pair, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
if err != nil {
// Key-pair errors describe structure only, never key material.
return fmt.Errorf("healthcheck: load client certificate %q and key %q: %w", clientCertFile, clientKeyFile, err)
}
transport.TLSClientConfig = &tls.Config{Certificates: []tls.Certificate{pair}}
}
if _, err := netip.ParseAddr(probe.Hostname()); probe.Scheme == "https" && err == nil {
data, err := os.ReadFile(certFile)
if err != nil {
Expand Down Expand Up @@ -620,7 +646,11 @@ func checkHealth(healthURL string, derivedProbe bool, certFile string) error {
}
roots := x509.NewCertPool()
roots.AddCert(leaf)
transport.TLSClientConfig = &tls.Config{RootCAs: roots, ServerName: serverName}
if transport.TLSClientConfig == nil {
transport.TLSClientConfig = &tls.Config{}
}
transport.TLSClientConfig.RootCAs = roots
transport.TLSClientConfig.ServerName = serverName
}
defer transport.CloseIdleConnections()
client.Transport = transport
Expand Down
124 changes: 113 additions & 11 deletions cmd/fanout/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"flag"
Expand All @@ -27,6 +28,7 @@ import (

"github.com/labstack/fanout/internal/auth"
"github.com/labstack/fanout/internal/config"
"github.com/labstack/fanout/internal/server"
"github.com/labstack/fanout/internal/store"
"github.com/labstack/fanout/internal/telemetry"
telemetrystore "github.com/labstack/fanout/internal/telemetry/store"
Expand Down Expand Up @@ -122,7 +124,7 @@ func TestCheckHealth(t *testing.T) {
writer.WriteHeader(test.statusCode)
}))
defer server.Close()
if err := checkHealth(server.URL, false, ""); (err != nil) != test.wantErr {
if err := checkHealth(server.URL, false, "", "", ""); (err != nil) != test.wantErr {
t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr)
}
})
Expand Down Expand Up @@ -340,11 +342,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
certFile := writeHealthcheckCertificate(t, server.Certificate().Raw)
if err := checkHealth(server.URL+"/healthz", true, certFile); err != nil {
if err := checkHealth(server.URL+"/healthz", true, certFile, "", ""); err != nil {
t.Fatal(err)
}
var verificationError *tls.CertificateVerificationError
if err := checkHealth(server.URL+"/healthz", false, certFile); !errors.As(err, &verificationError) {
if err := checkHealth(server.URL+"/healthz", false, certFile, "", ""); !errors.As(err, &verificationError) {
t.Fatalf("explicit URL must verify the certificate, got %v", err)
}
other := *server.Certificate()
Expand All @@ -359,11 +361,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) {
if err := os.WriteFile(otherFile, chain, 0o600); err != nil {
t.Fatal(err)
}
if err := checkHealth(server.URL+"/healthz", true, otherFile); !errors.As(err, &verificationError) {
if err := checkHealth(server.URL+"/healthz", true, otherFile, "", ""); !errors.As(err, &verificationError) {
t.Fatalf("derived IP probe must reject a different certificate, got %v", err)
}
probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz"
if err := checkHealth(probe, true, certFile); !errors.As(err, &verificationError) {
if err := checkHealth(probe, true, certFile, "", ""); !errors.As(err, &verificationError) {
t.Fatalf("derived hostname probe must verify the certificate, got %v", err)
}
}
Expand All @@ -390,6 +392,106 @@ func newHealthcheckCertificate(t *testing.T, certificate *x509.Certificate) tls.
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
}

// The derived probe presents FANOUT_HEALTHCHECK_CERT_FILE to a listener that
// requires client certificates, and fails without it.
func TestCheckHealthWithClientCertificate(t *testing.T) {
caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
caTemplate := &x509.Certificate{
SerialNumber: big.NewInt(10), Subject: pkix.Name{CommonName: "healthcheck CA"},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
ca, err := x509.ParseCertificate(caDER)
if err != nil {
t.Fatal(err)
}
clientKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
clientDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{
SerialNumber: big.NewInt(11), Subject: pkix.Name{CommonName: "fanout healthcheck"},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}, ca, &clientKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
caFile := filepath.Join(dir, "client-ca.pem")
clientCertFile := filepath.Join(dir, "healthcheck.pem")
clientKeyFile := filepath.Join(dir, "healthcheck-key.pem")
keyDER, err := x509.MarshalPKCS8PrivateKey(clientKey)
if err != nil {
t.Fatal(err)
}
for path, block := range map[string]*pem.Block{
caFile: {Type: "CERTIFICATE", Bytes: caDER},
clientCertFile: {Type: "CERTIFICATE", Bytes: clientDER},
clientKeyFile: {Type: "PRIVATE KEY", Bytes: keyDER},
} {
if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil {
t.Fatal(err)
}
}

ts := httptest.NewUnstartedServer(nil)
ts.Config = server.New("", http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/healthz" || len(request.TLS.PeerCertificates) == 0 {
t.Errorf("probe path %q with %d client certificates", request.URL.Path, len(request.TLS.PeerCertificates))
}
w.WriteHeader(http.StatusOK)
}), http.NotFoundHandler(), http.NotFoundHandler())
clientCAs, err := server.LoadClientCAs(caFile)
if err != nil {
t.Fatal(err)
}
server.RequireClientCertificates(ts.Config, clientCAs)
ts.TLS = ts.Config.TLSConfig.Clone()
ts.TLS.Certificates = []tls.Certificate{newHealthcheckCertificate(t, &x509.Certificate{
SerialNumber: big.NewInt(12), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
})}
ts.EnableHTTP2 = true
ts.StartTLS()
defer ts.Close()
certFile := writeHealthcheckCertificate(t, ts.Certificate().Raw)

probe, err := healthcheckURL(ts.Listener.Addr().String(), certFile)
if err != nil {
t.Fatal(err)
}
if err := checkHealth(probe, true, certFile, clientCertFile, clientKeyFile); err != nil {
t.Fatalf("healthcheck with client certificate: %v", err)
}
if err := checkHealth(probe, true, certFile, "", ""); err == nil {
t.Fatal("healthcheck without a client certificate must fail")
}
if err := checkHealth(probe, true, certFile, clientCertFile, ""); err == nil || !strings.Contains(err.Error(), "FANOUT_HEALTHCHECK_KEY_FILE") {
t.Fatalf("partial client pair must name both variables, got %v", err)
}
missing := filepath.Join(dir, "missing-key.pem")
if err := checkHealth(probe, true, certFile, clientCertFile, missing); err == nil || !strings.Contains(err.Error(), missing) {
t.Fatalf("unreadable client key must name its path, got %v", err)
}
keyPEM, err := os.ReadFile(clientKeyFile)
if err != nil {
t.Fatal(err)
}
// A key passed as the certificate must fail without echoing key material.
if err := checkHealth(probe, true, certFile, clientKeyFile, clientKeyFile); err == nil || !strings.Contains(err.Error(), clientKeyFile) || strings.Contains(err.Error(), string(keyPEM[40:80])) {
t.Fatalf("invalid client pair must name its path without contents, got %v", err)
}
}

func TestCheckHealthCertificateFileErrors(t *testing.T) {
const contents = "private file contents must not appear in errors"
for _, test := range []struct {
Expand All @@ -408,7 +510,7 @@ func TestCheckHealthCertificateFileErrors(t *testing.T) {
t.Fatal(err)
}
}
err := checkHealth("https://127.0.0.1:7520/healthz", true, path)
err := checkHealth("https://127.0.0.1:7520/healthz", true, path, "", "")
if err == nil || !strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) {
t.Fatalf("certificate file error must name its path without contents, got %v", err)
}
Expand Down Expand Up @@ -453,7 +555,7 @@ func TestCheckHealthTLSCertificate(t *testing.T) {
server.StartTLS()
defer server.Close()
certFile := writeHealthcheckCertificate(t, server.Certificate().Raw)
err := checkHealth(server.URL+"/healthz", true, certFile)
err := checkHealth(server.URL+"/healthz", true, certFile, "", "")
if (err != nil) != test.wantErr {
t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr)
}
Expand Down Expand Up @@ -504,7 +606,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = checkHealth(probe, true, certFile)
err = checkHealth(probe, true, certFile, "", "")
var verificationError *tls.CertificateVerificationError
if strings.HasSuffix(addr, "example.com:7520") {
if !errors.As(err, &verificationError) {
Expand All @@ -517,7 +619,7 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = checkHealth(explicitProbe, derivedProbe, certFile)
err = checkHealth(explicitProbe, derivedProbe, certFile, "", "")
if !errors.As(err, &verificationError) {
t.Fatalf("explicit TLS probe must use system verification, got %v", err)
}
Expand Down Expand Up @@ -596,10 +698,10 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := checkHealth(probe, true, certFile); err != nil {
if err := checkHealth(probe, true, certFile, "", ""); err != nil {
t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err)
}
if err := checkHealth(probe, false, certFile); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) {
if err := checkHealth(probe, false, certFile, "", ""); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) {
t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err)
}
})
Expand Down
13 changes: 10 additions & 3 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,11 @@ Setting both `FANOUT_TLS_CERT_FILE` and `FANOUT_TLS_KEY_FILE` enables TLS 1.3 on
the shared listener. A proxy must forward HTTP/2 to Fanout for gRPC: use
h2c for a plaintext backend, or HTTP/2 over TLS when Fanout has a certificate.
HTTP/1.1 forwarding supports OTLP/HTTP, but cannot carry gRPC.
`FANOUT_TLS_CLIENT_CA_FILE` additionally requires every client, including a
TLS-terminating proxy such as Cloudflare with Authenticated Origin Pulls, to
present a certificate from that CA bundle. The bundle is read at startup, so
rotation needs a restart; keep the old and new CA in the bundle while clients
move to certificates from the new one.

Set `FANOUT_PUBLIC_URL` to the externally reachable HTTPS browser origin. When
using forwarded client addresses, set `FANOUT_TRUSTED_PROXY_CIDRS` to only the
Expand All @@ -182,9 +187,11 @@ default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`.
`FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies.
HTTPS IP-literal probes trust only the configured certificate and verify its
first DNS or IP SAN, which may differ from the bind IP; hostname probes verify
normally. Healthcheck reads only these two settings and does not require valid
secrets. An explicit `fanout healthcheck <url>` uses default proxy and
certificate verification behaviour.
normally. Derived HTTPS probes present `FANOUT_HEALTHCHECK_CERT_FILE` and
`FANOUT_HEALTHCHECK_KEY_FILE` when set, so they pass a listener with
`FANOUT_TLS_CLIENT_CA_FILE`. Healthcheck reads only these settings and does
not require valid secrets. An explicit `fanout healthcheck <url>` uses default
proxy and certificate verification behaviour.

- `/healthz` is the liveness check. It fails only when Fanout's own work on
the query engine stalls: a Parquet publication holds reads for 10 minutes,
Expand Down
27 changes: 25 additions & 2 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,14 @@ type Config struct {
// TLSCertFile selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE.
TLSCertFile string `group:"server" env:"FANOUT_TLS_CERT_FILE"`
// TLSKeyFile selects the TLS private key file; requires FANOUT_TLS_CERT_FILE.
TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"`
resolvedSizing sizingSource
TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"`
// TLSClientCAFile selects a PEM CA bundle; when set, the listener requires client certificates signed by it. Requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE.
TLSClientCAFile string `group:"server" env:"FANOUT_TLS_CLIENT_CA_FILE"`
// HealthcheckCertFile selects the client certificate fanout healthcheck presents; requires FANOUT_HEALTHCHECK_KEY_FILE and FANOUT_TLS_CLIENT_CA_FILE.
HealthcheckCertFile string `group:"server" env:"FANOUT_HEALTHCHECK_CERT_FILE"`
// HealthcheckKeyFile selects the private key for FANOUT_HEALTHCHECK_CERT_FILE; requires FANOUT_HEALTHCHECK_CERT_FILE.
HealthcheckKeyFile string `group:"server" env:"FANOUT_HEALTHCHECK_KEY_FILE"`
resolvedSizing sizingSource
}

// LogStartup reports the effective non-secret sizing and security-sensitive
Expand Down Expand Up @@ -262,6 +268,12 @@ func (c Config) TLSEnabled() bool {
strings.TrimSpace(c.TLSKeyFile) != ""
}

// ClientCertificatesRequired reports whether the TLS listener requires and
// verifies client certificates against FANOUT_TLS_CLIENT_CA_FILE.
func (c Config) ClientCertificatesRequired() bool {
return c.TLSEnabled() && strings.TrimSpace(c.TLSClientCAFile) != ""
}

// Validate checks that config values are sane.
func (c Config) Validate() error {
if strings.TrimSpace(c.Addr) == "" {
Expand Down Expand Up @@ -401,6 +413,17 @@ func (c Config) Validate() error {
if anySet(c.TLSCertFile, c.TLSKeyFile) && !c.TLSEnabled() {
return fmt.Errorf("FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE must both be set")
}
if anySet(c.TLSClientCAFile) && !c.TLSEnabled() {
return fmt.Errorf("FANOUT_TLS_CLIENT_CA_FILE requires FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE")
}
if anySet(c.HealthcheckCertFile, c.HealthcheckKeyFile) {
if !anySet(c.HealthcheckCertFile) || !anySet(c.HealthcheckKeyFile) {
return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE must both be set")
}
if !anySet(c.TLSClientCAFile) {
return fmt.Errorf("FANOUT_HEALTHCHECK_CERT_FILE and FANOUT_HEALTHCHECK_KEY_FILE require FANOUT_TLS_CLIENT_CA_FILE")
}
}
return nil
}

Expand Down
Loading
Loading