Add mTLS support for remote write

README.md

@@ -20,6 +20,26 @@ go install github.com/prometheus-community/avalanche/cmd/avalanche@latest
 ${GOPATH}/bin/avalanche --help
 ```
 
+### TLS flags for remote write
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--tls-client-insecure` | `false` | Skip server certificate verification |
+| `--tls-client-cert-file` | `""` | Path to PEM-encoded client certificate (mTLS) |
+| `--tls-client-key-file` | `""` | Path to PEM-encoded client private key (mTLS) |
+| `--tls-ca-cert-file` | `""` | Path to PEM-encoded CA certificate to verify the server |
+
+`--tls-client-cert-file` and `--tls-client-key-file` must be set together or not at all.
+
+Example with mTLS:
+
+```bash
+avalanche --remote-url=https://example.com/api/v1/push \
+  --tls-client-cert-file=client.crt \
+  --tls-client-key-file=client.key \
+  --tls-ca-cert-file=ca.crt
+```
+
 ### Docker 
 
 ```bash

metricsgen/tls_test.go

@@ -0,0 +1,195 @@
+// Copyright 2022 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metricsgen
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"testing"
+	"time"
+)
+
+// generateSelfSignedCert writes a self-signed PEM certificate and key to the
+// provided file paths.
+func generateSelfSignedCert(t *testing.T, certFile, keyFile string) {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate key: %v", err)
+	}
+
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "test"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("create certificate: %v", err)
+	}
+
+	cf, err := os.Create(certFile)
+	if err != nil {
+		t.Fatalf("create cert file: %v", err)
+	}
+	defer cf.Close()
+	if err := pem.Encode(cf, &pem.Block{Type: "CERTIFICATE", Bytes: certDER}); err != nil {
+		t.Fatalf("encode cert: %v", err)
+	}
+
+	kf, err := os.Create(keyFile)
+	if err != nil {
+		t.Fatalf("create key file: %v", err)
+	}
+	defer kf.Close()
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		t.Fatalf("marshal key: %v", err)
+	}
+	if err := pem.Encode(kf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}); err != nil {
+		t.Fatalf("encode key: %v", err)
+	}
+}
+
+func TestBuildTLSConfig(t *testing.T) {
+	dir := t.TempDir()
+	certFile := dir + "/client.crt"
+	keyFile := dir + "/client.key"
+	caFile := dir + "/ca.crt"
+	caKeyFile := dir + "/ca.key"
+
+	generateSelfSignedCert(t, certFile, keyFile)
+	generateSelfSignedCert(t, caFile, caKeyFile)
+
+	tests := []struct {
+		name     string
+		insecure bool
+		certFile string
+		keyFile  string
+		caFile   string
+		wantErr  bool
+	}{
+		{
+			name:     "defaults: no TLS material",
+			insecure: false,
+			wantErr:  false,
+		},
+		{
+			name:     "insecure skip verify",
+			insecure: true,
+			wantErr:  false,
+		},
+		{
+			name:     "client cert and key",
+			certFile: certFile,
+			keyFile:  keyFile,
+			wantErr:  false,
+		},
+		{
+			name:    "CA cert only",
+			caFile:  caFile,
+			wantErr: false,
+		},
+		{
+			name:     "all three flags",
+			insecure: false,
+			certFile: certFile,
+			keyFile:  keyFile,
+			caFile:   caFile,
+			wantErr:  false,
+		},
+		{
+			name:     "insecure plus CA file (CA ignored, no error)",
+			insecure: true,
+			caFile:   caFile,
+			wantErr:  false,
+		},
+		{
+			name:     "cert without key",
+			certFile: certFile,
+			wantErr:  true,
+		},
+		{
+			name:    "key without cert",
+			keyFile: keyFile,
+			wantErr: true,
+		},
+		{
+			name:     "nonexistent cert file",
+			certFile: dir + "/no.crt",
+			keyFile:  keyFile,
+			wantErr:  true,
+		},
+		{
+			name:     "nonexistent key file",
+			certFile: certFile,
+			keyFile:  dir + "/no.key",
+			wantErr:  true,
+		},
+		{
+			name:    "nonexistent CA file",
+			caFile:  dir + "/no-ca.crt",
+			wantErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg, err := buildTLSConfig(tc.insecure, tc.certFile, tc.keyFile, tc.caFile)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("expected error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if cfg == nil {
+				t.Fatal("expected non-nil tls.Config, got nil")
+			}
+
+			if cfg.InsecureSkipVerify != tc.insecure {
+				t.Errorf("InsecureSkipVerify: got %v, want %v", cfg.InsecureSkipVerify, tc.insecure)
+			}
+
+			wantCerts := tc.certFile != ""
+			if wantCerts && len(cfg.Certificates) != 1 {
+				t.Errorf("expected exactly 1 client certificate, got %d", len(cfg.Certificates))
+			}
+			if !wantCerts && len(cfg.Certificates) != 0 {
+				t.Errorf("expected no client certificates, got %d", len(cfg.Certificates))
+			}
+
+			wantCA := tc.caFile != ""
+			if wantCA && cfg.RootCAs == nil {
+				t.Error("expected RootCAs to be set, got nil")
+			}
+			if !wantCA && cfg.RootCAs != nil {
+				t.Error("expected RootCAs to be nil")
+			}
+		})
+	}
+}

metricsgen/write.go

@@ -16,12 +16,14 @@ package metricsgen
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"log"
 	"log/slog"
 	"net/http"
 	"net/url"
+	"os"
 	"sort"
 	"sync"
 	"time"
@@ -40,14 +42,17 @@ type ConfigWrite struct {
 	RequestInterval time.Duration
 	BatchSize,
 	RequestCount int
-	UpdateNotify    chan struct{}
-	PprofURLs       []*url.URL
-	Tenant          string
-	TLSClientConfig tls.Config
-	TenantHeader    string
-	OutOfOrder      bool
-	Concurrency     int
-	WriteV2         bool
+	UpdateNotify      chan struct{}
+	PprofURLs         []*url.URL
+	Tenant            string
+	TLSClientInsecure bool
+	TLSClientCertFile string
+	TLSClientKeyFile  string
+	TLSCACertFile     string
+	TenantHeader      string
+	OutOfOrder        bool
+	Concurrency       int
+	WriteV2           bool
 }
 
 func NewWriteConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause) *ConfigWrite {
@@ -65,7 +70,13 @@ func NewWriteConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause
 	flagReg("remote-tenant", "Tenant ID to include in remote_write send").Default("0").
 		StringVar(&cfg.Tenant)
 	flagReg("tls-client-insecure", "Skip certificate check on tls connection").Default("false").
-		BoolVar(&cfg.TLSClientConfig.InsecureSkipVerify)
+		BoolVar(&cfg.TLSClientInsecure)
+	flagReg("tls-client-cert-file", "Client certificate file for TLS connections.").Default("").
+		StringVar(&cfg.TLSClientCertFile)
+	flagReg("tls-client-key-file", "Client key file for TLS connections.").Default("").
+		StringVar(&cfg.TLSClientKeyFile)
+	flagReg("tls-ca-cert-file", "CA certificate file to verify server certificate.").Default("").
+		StringVar(&cfg.TLSCACertFile)
 	flagReg("remote-tenant-header", "Tenant ID to include in remote_write send. The default, is the default tenant header expected by Cortex.").Default("X-Scope-OrgID").
 		StringVar(&cfg.TenantHeader)
 	// TODO(bwplotka): Make this a non-bool flag (e.g. out-of-order-min-time).
@@ -91,9 +102,46 @@ func (c *ConfigWrite) Validate() error {
 			return fmt.Errorf("--remote-write-interval must be greater than 0, got %v", c.RequestInterval)
 		}
 	}
+	if (c.TLSClientCertFile == "") != (c.TLSClientKeyFile == "") {
+		return fmt.Errorf("--tls-client-cert-file and --tls-client-key-file must both be set or both be unset")
+	}
 	return nil
 }
 
+func buildTLSConfig(insecure bool, certFile, keyFile, caFile string) (*tls.Config, error) {
+	if (certFile == "") != (keyFile == "") {
+		return nil, fmt.Errorf("--tls-client-cert-file and --tls-client-key-file must both be set or both be unset")
+	}
+
+	if insecure && caFile != "" {
+		log.Printf("warning: --tls-ca-cert-file is ignored when --tls-client-insecure is set")
+	}
+
+	cfg := &tls.Config{InsecureSkipVerify: insecure} //nolint:gosec
+
+	if certFile != "" {
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load TLS client cert/key: %w", err)
+		}
+		cfg.Certificates = []tls.Certificate{cert}
+	}
+
+	if caFile != "" {
+		caCert, err := os.ReadFile(caFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA cert file: %w", err)
+		}
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to parse CA cert file %q: no valid PEM certificates found", caFile)
+		}
+		cfg.RootCAs = caCertPool
+	}
+
+	return cfg, nil
+}
+
 // Writer for remote write requests.
 type Writer struct {
 	logger    *slog.Logger
@@ -105,8 +153,13 @@ type Writer struct {
 
 // RunRemoteWriting initializes a http client and starts a Writer for remote writing metrics to a prometheus compatible remote endpoint.
 func RunRemoteWriting(ctx context.Context, logger *slog.Logger, cfg *ConfigWrite, gatherer prometheus.Gatherer) error {
+	tlsCfg, err := buildTLSConfig(cfg.TLSClientInsecure, cfg.TLSClientCertFile, cfg.TLSClientKeyFile, cfg.TLSCACertFile)
+	if err != nil {
+		return err
+	}
+
 	var rt http.RoundTripper = &http.Transport{
-		TLSClientConfig: &cfg.TLSClientConfig,
+		TLSClientConfig: tlsCfg,
 	}
 	rt = &tenantRoundTripper{tenant: cfg.Tenant, tenantHeader: cfg.TenantHeader, rt: rt}
 	rt = &userAgentRoundTripper{userAgent: "avalanche", rt: rt}