diff --git a/README.md b/README.md index 7955d6e..f8a9817 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,26 @@ go install github.com/prometheus-community/avalanche/cmd/avalanche@latest ${GOPATH}/bin/avalanche --help ``` +### TLS flags for remote write + +| Flag | Default | Description | +|------|---------|-------------| +| `--tls-client-insecure` | `false` | Skip server certificate verification | +| `--tls-client-cert-file` | `""` | Path to PEM-encoded client certificate (mTLS) | +| `--tls-client-key-file` | `""` | Path to PEM-encoded client private key (mTLS) | +| `--tls-ca-cert-file` | `""` | Path to PEM-encoded CA certificate to verify the server | + +`--tls-client-cert-file` and `--tls-client-key-file` must be set together or not at all. + +Example with mTLS: + +```bash +avalanche --remote-url=https://example.com/api/v1/push \ + --tls-client-cert-file=client.crt \ + --tls-client-key-file=client.key \ + --tls-ca-cert-file=ca.crt +``` + ### Docker ```bash diff --git a/metricsgen/tls_test.go b/metricsgen/tls_test.go new file mode 100644 index 0000000..25ae099 --- /dev/null +++ b/metricsgen/tls_test.go @@ -0,0 +1,195 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricsgen + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "testing" + "time" +) + +// generateSelfSignedCert writes a self-signed PEM certificate and key to the +// provided file paths. +func generateSelfSignedCert(t *testing.T, certFile, keyFile string) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + + cf, err := os.Create(certFile) + if err != nil { + t.Fatalf("create cert file: %v", err) + } + defer cf.Close() + if err := pem.Encode(cf, &pem.Block{Type: "CERTIFICATE", Bytes: certDER}); err != nil { + t.Fatalf("encode cert: %v", err) + } + + kf, err := os.Create(keyFile) + if err != nil { + t.Fatalf("create key file: %v", err) + } + defer kf.Close() + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + if err := pem.Encode(kf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}); err != nil { + t.Fatalf("encode key: %v", err) + } +} + +func TestBuildTLSConfig(t *testing.T) { + dir := t.TempDir() + certFile := dir + "/client.crt" + keyFile := dir + "/client.key" + caFile := dir + "/ca.crt" + caKeyFile := dir + "/ca.key" + + generateSelfSignedCert(t, certFile, keyFile) + generateSelfSignedCert(t, caFile, caKeyFile) + + tests := []struct { + name string + insecure bool + certFile string + keyFile string + caFile string + wantErr bool + }{ + { + name: "defaults: no TLS material", + insecure: false, + wantErr: false, + }, + { + name: "insecure skip verify", + insecure: true, + wantErr: false, + }, + { + name: "client cert and key", + certFile: certFile, + keyFile: keyFile, + wantErr: false, + }, + { + name: "CA cert only", + caFile: caFile, + wantErr: false, + }, + { + name: "all three flags", + insecure: false, + certFile: certFile, + keyFile: keyFile, + caFile: caFile, + wantErr: false, + }, + { + name: "insecure plus CA file (CA ignored, no error)", + insecure: true, + caFile: caFile, + wantErr: false, + }, + { + name: "cert without key", + certFile: certFile, + wantErr: true, + }, + { + name: "key without cert", + keyFile: keyFile, + wantErr: true, + }, + { + name: "nonexistent cert file", + certFile: dir + "/no.crt", + keyFile: keyFile, + wantErr: true, + }, + { + name: "nonexistent key file", + certFile: certFile, + keyFile: dir + "/no.key", + wantErr: true, + }, + { + name: "nonexistent CA file", + caFile: dir + "/no-ca.crt", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg, err := buildTLSConfig(tc.insecure, tc.certFile, tc.keyFile, tc.caFile) + if tc.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg == nil { + t.Fatal("expected non-nil tls.Config, got nil") + } + + if cfg.InsecureSkipVerify != tc.insecure { + t.Errorf("InsecureSkipVerify: got %v, want %v", cfg.InsecureSkipVerify, tc.insecure) + } + + wantCerts := tc.certFile != "" + if wantCerts && len(cfg.Certificates) != 1 { + t.Errorf("expected exactly 1 client certificate, got %d", len(cfg.Certificates)) + } + if !wantCerts && len(cfg.Certificates) != 0 { + t.Errorf("expected no client certificates, got %d", len(cfg.Certificates)) + } + + wantCA := tc.caFile != "" + if wantCA && cfg.RootCAs == nil { + t.Error("expected RootCAs to be set, got nil") + } + if !wantCA && cfg.RootCAs != nil { + t.Error("expected RootCAs to be nil") + } + }) + } +} diff --git a/metricsgen/write.go b/metricsgen/write.go index 55792fa..cd8d43e 100644 --- a/metricsgen/write.go +++ b/metricsgen/write.go @@ -16,12 +16,14 @@ package metricsgen import ( "context" "crypto/tls" + "crypto/x509" "errors" "fmt" "log" "log/slog" "net/http" "net/url" + "os" "sort" "sync" "time" @@ -40,14 +42,17 @@ type ConfigWrite struct { RequestInterval time.Duration BatchSize, RequestCount int - UpdateNotify chan struct{} - PprofURLs []*url.URL - Tenant string - TLSClientConfig tls.Config - TenantHeader string - OutOfOrder bool - Concurrency int - WriteV2 bool + UpdateNotify chan struct{} + PprofURLs []*url.URL + Tenant string + TLSClientInsecure bool + TLSClientCertFile string + TLSClientKeyFile string + TLSCACertFile string + TenantHeader string + OutOfOrder bool + Concurrency int + WriteV2 bool } func NewWriteConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause) *ConfigWrite { @@ -65,7 +70,13 @@ func NewWriteConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause flagReg("remote-tenant", "Tenant ID to include in remote_write send").Default("0"). StringVar(&cfg.Tenant) flagReg("tls-client-insecure", "Skip certificate check on tls connection").Default("false"). - BoolVar(&cfg.TLSClientConfig.InsecureSkipVerify) + BoolVar(&cfg.TLSClientInsecure) + flagReg("tls-client-cert-file", "Client certificate file for TLS connections.").Default(""). + StringVar(&cfg.TLSClientCertFile) + flagReg("tls-client-key-file", "Client key file for TLS connections.").Default(""). + StringVar(&cfg.TLSClientKeyFile) + flagReg("tls-ca-cert-file", "CA certificate file to verify server certificate.").Default(""). + StringVar(&cfg.TLSCACertFile) flagReg("remote-tenant-header", "Tenant ID to include in remote_write send. The default, is the default tenant header expected by Cortex.").Default("X-Scope-OrgID"). StringVar(&cfg.TenantHeader) // TODO(bwplotka): Make this a non-bool flag (e.g. out-of-order-min-time). @@ -91,9 +102,46 @@ func (c *ConfigWrite) Validate() error { return fmt.Errorf("--remote-write-interval must be greater than 0, got %v", c.RequestInterval) } } + if (c.TLSClientCertFile == "") != (c.TLSClientKeyFile == "") { + return fmt.Errorf("--tls-client-cert-file and --tls-client-key-file must both be set or both be unset") + } return nil } +func buildTLSConfig(insecure bool, certFile, keyFile, caFile string) (*tls.Config, error) { + if (certFile == "") != (keyFile == "") { + return nil, fmt.Errorf("--tls-client-cert-file and --tls-client-key-file must both be set or both be unset") + } + + if insecure && caFile != "" { + log.Printf("warning: --tls-ca-cert-file is ignored when --tls-client-insecure is set") + } + + cfg := &tls.Config{InsecureSkipVerify: insecure} //nolint:gosec + + if certFile != "" { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("failed to load TLS client cert/key: %w", err) + } + cfg.Certificates = []tls.Certificate{cert} + } + + if caFile != "" { + caCert, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("failed to read CA cert file: %w", err) + } + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse CA cert file %q: no valid PEM certificates found", caFile) + } + cfg.RootCAs = caCertPool + } + + return cfg, nil +} + // Writer for remote write requests. type Writer struct { logger *slog.Logger @@ -105,8 +153,13 @@ type Writer struct { // RunRemoteWriting initializes a http client and starts a Writer for remote writing metrics to a prometheus compatible remote endpoint. func RunRemoteWriting(ctx context.Context, logger *slog.Logger, cfg *ConfigWrite, gatherer prometheus.Gatherer) error { + tlsCfg, err := buildTLSConfig(cfg.TLSClientInsecure, cfg.TLSClientCertFile, cfg.TLSClientKeyFile, cfg.TLSCACertFile) + if err != nil { + return err + } + var rt http.RoundTripper = &http.Transport{ - TLSClientConfig: &cfg.TLSClientConfig, + TLSClientConfig: tlsCfg, } rt = &tenantRoundTripper{tenant: cfg.Tenant, tenantHeader: cfg.TenantHeader, rt: rt} rt = &userAgentRoundTripper{userAgent: "avalanche", rt: rt}