diff --git a/controllers/configuration/ca_bundle_watcher.go b/controllers/configuration/ca_bundle_watcher.go new file mode 100644 index 000000000..2c88d39b6 --- /dev/null +++ b/controllers/configuration/ca_bundle_watcher.go @@ -0,0 +1,128 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration + +import ( + "context" + "crypto/x509" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const ( + CABundleConfigMapName = "threescale-ca-bundle" + CABundleConfigMapKey = "ca-bundle.crt" +) + +// CABundleWatcher watches the threescale-ca-bundle ConfigMap and updates the +// package-level CA pool via SetRootCAs on every successful reconcile. +type CABundleWatcher struct { + client.Client + Recorder record.EventRecorder + Namespace string +} + +// SetupWithManager registers the controller with the manager. +func (r *CABundleWatcher) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("cabundlewatcher"). + // NeedLeaderElection is disabled so that every operator replica runs + // this controller, not just the leader. Each replica must load the CA + // bundle into its own in-process TLS config. + // + // controller-runtime replays a synthetic Add event for every object + // already in the informer cache when a controller starts, so a newly + // promoted leader would pick up the bundle automatically. Running on + // all replicas avoids the small window between manager start and leader + // promotion during which capability reconcilers could attempt outbound + // TLS calls before the bundle has been loaded. Those calls would fail + // and requeue, but disabling leader election here eliminates the window + // entirely. + WithOptions(controller.Options{NeedLeaderElection: ptr.To(false)}). + For(&corev1.ConfigMap{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == r.Namespace && object.GetName() == CABundleConfigMapName + }))). + Complete(r) +} + +// Reconcile fetches the CA bundle ConfigMap and updates the package-level +// TLS config. On an invalid bundle, the error is logged and recorded as a Warning event on +// the ConfigMap; the existing TLS config is left unchanged so capability +// controllers continue to operate with the last known good CA. +func (r *CABundleWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := ctrl.LoggerFrom(ctx) + + cm := &corev1.ConfigMap{} + err := r.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: CABundleConfigMapName}, cm) + if err != nil { + if apierrors.IsNotFound(err) { + SetRootCAs(nil) + logger.Info("CA bundle ConfigMap not found; using system default CAs", "namespace", r.Namespace, "configmap", CABundleConfigMapName) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + pool, parseErr := parseBundleFromConfigMap(cm) + if parseErr != nil { + logger.Error(parseErr, "CA bundle ConfigMap contains an invalid certificate bundle; keeping previous TLS config") + if r.Recorder != nil { + r.Recorder.Eventf(cm, corev1.EventTypeWarning, "InvalidCABundle", "%v", parseErr) + } + return ctrl.Result{}, nil + } + + if pool == nil { + SetRootCAs(nil) + logger.Info("CA bundle ConfigMap key absent; using system default CAs", "namespace", r.Namespace, "configmap", CABundleConfigMapName, "key", CABundleConfigMapKey) + return ctrl.Result{}, nil + } + + SetRootCAs(pool) + logger.Info("CA bundle updated and applied", "namespace", r.Namespace, "configmap", CABundleConfigMapName) + return ctrl.Result{}, nil +} + +// parseBundleFromConfigMap parses the CA bundle from a ConfigMap and returns a +// *x509.CertPool. Returns nil, nil if the key is absent (no custom bundle +// configured). Returns an error if the PEM data is present but invalid. +func parseBundleFromConfigMap(cm *corev1.ConfigMap) (*x509.CertPool, error) { + val, exists := cm.Data[CABundleConfigMapKey] + if !exists { + return nil, nil // no bundle configured — use system roots + } + + if len(val) == 0 { + return nil, fmt.Errorf("InvalidCAFormat: key %q in ConfigMap %s/%s is empty", CABundleConfigMapKey, cm.Namespace, cm.Name) + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM([]byte(val)) { + return nil, fmt.Errorf("InvalidCAFormat: no valid PEM-encoded certificates found in CA bundle") + } + + return certPool, nil +} diff --git a/controllers/configuration/ca_bundle_watcher_test.go b/controllers/configuration/ca_bundle_watcher_test.go new file mode 100644 index 000000000..5fe83907a --- /dev/null +++ b/controllers/configuration/ca_bundle_watcher_test.go @@ -0,0 +1,316 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + fakectrlclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/3scale/3scale-operator/controllers/configuration" + "github.com/3scale/3scale-operator/pkg/testhelper" +) + +// errorInjectingClient makes r.Get return a non-NotFound error so the +// reconciler propagates it to the workqueue. +type errorInjectingClient struct { + client.Client + getErr error +} + +func (e *errorInjectingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return e.getErr +} + +// generateSelfSignedCert creates a minimal self-signed RSA certificate and +// returns its PEM-encoded DER bytes. It is not a real secret and carries no +// trust outside the test suite. +func generateSelfSignedCert(t *testing.T, cn string) []byte { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate RSA key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + + return der +} + +func encodePEM(der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func TestCABundleWatcher_Reconcile(t *testing.T) { + certDER := generateSelfSignedCert(t, "test-ca") + certPEM := encodePEM(certDER) + + seedDER := generateSelfSignedCert(t, "test-ca-at-start") + seedPool := x509.NewCertPool() + seedCert, err := x509.ParseCertificate(seedDER) + if err != nil { + t.Fatalf("parse seed cert: %v", err) + } + seedPool.AddCert(seedCert) + + validCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: configuration.CABundleConfigMapName}, + Data: map[string]string{configuration.CABundleConfigMapKey: string(certPEM)}, + } + emptyCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: configuration.CABundleConfigMapName}, + Data: map[string]string{configuration.CABundleConfigMapKey: ""}, + } + invalidCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: configuration.CABundleConfigMapName}, + Data: map[string]string{configuration.CABundleConfigMapKey: "not-a-pem"}, + } + keyAbsentCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: configuration.CABundleConfigMapName}, + Data: map[string]string{}, + } + + tests := []struct { + name string + objects []runtime.Object + seedPool *x509.CertPool + injectGetErr bool + wantErr bool + wantErrMsg string + check func(t *testing.T, rec *record.FakeRecorder) + }{ + { + name: "ConfigMap not found — SetRootCAs(nil), no error, no event", + objects: []runtime.Object{}, + seedPool: seedPool, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + if configuration.GetRootCAs() != nil { + t.Error("expected GetRootCAs() == nil after not-found") + } + assertNoEventW(t, rec) + }, + }, + { + name: "generic Get error — error returned, CAs unchanged, no event", + objects: []runtime.Object{}, + seedPool: seedPool, + injectGetErr: true, + wantErr: true, + wantErrMsg: "synthetic server error", + check: func(t *testing.T, rec *record.FakeRecorder) { + pool := configuration.GetRootCAs() + if pool == nil { + t.Error("expected GetRootCAs() to remain non-nil (seeded value)") + } else if !pool.Equal(seedPool) { + t.Error("loaded CA pool does not contain the expected certificate") + } + assertNoEventW(t, rec) + }, + }, + { + name: "ConfigMap present, key present but empty — Warning event, no error, CAs unchanged", + objects: []runtime.Object{emptyCM}, + seedPool: seedPool, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + pool := configuration.GetRootCAs() + if pool == nil { + t.Error("expected GetRootCAs() to remain non-nil (kept previous config)") + } else if !pool.Equal(seedPool) { + t.Error("loaded CA pool does not contain the expected certificate") + } + + msg := drainOneEventW(t, rec) + for _, sub := range []string{"Warning", "InvalidCABundle", "InvalidCAFormat: key \"ca-bundle.crt\" in ConfigMap test-ns/threescale-ca-bundle is empty"} { + if !strings.Contains(msg, sub) { + t.Errorf("event %q missing expected substring %q", msg, sub) + } + } + assertNoEventW(t, rec) + }, + }, + { + name: "ConfigMap present, PEM invalid — Warning event, no error, CAs unchanged", + objects: []runtime.Object{invalidCM}, + seedPool: seedPool, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + pool := configuration.GetRootCAs() + if pool == nil { + t.Error("expected GetRootCAs() to remain non-nil (kept previous config)") + } else if !pool.Equal(seedPool) { + t.Error("loaded CA pool does not contain the expected certificate") + } + msg := drainOneEventW(t, rec) + for _, sub := range []string{"Warning", "InvalidCABundle", "InvalidCAFormat: no valid PEM-encoded certificates found in CA bundle"} { + if !strings.Contains(msg, sub) { + t.Errorf("event %q missing expected substring %q", msg, sub) + } + } + assertNoEventW(t, rec) + }, + }, + { + name: "ConfigMap present, key absent — SetRootCAs(nil), no error, no event", + objects: []runtime.Object{keyAbsentCM}, + seedPool: seedPool, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + if configuration.GetRootCAs() != nil { + t.Error("expected GetRootCAs() == nil after key-absent") + } + assertNoEventW(t, rec) + }, + }, + { + name: "ConfigMap present, valid PEM — SetRootCAs(pool), no error, no event", + objects: []runtime.Object{validCM}, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + pool := configuration.GetRootCAs() + if pool == nil { + t.Fatal("expected GetRootCAs() != nil after valid PEM") + } + cert, err := x509.ParseCertificate(certDER) + if err != nil { + t.Fatalf("parse test certificate: %v", err) + } + expected := x509.NewCertPool() + expected.AddCert(cert) + if !pool.Equal(expected) { + t.Error("loaded CA pool does not contain the expected certificate") + } + assertNoEventW(t, rec) + }, + }, + { + name: "ConfigMap present, valid PEM, replace seed case — SetRootCAs(pool), no error, no event", + objects: []runtime.Object{validCM}, + seedPool: seedPool, + wantErr: false, + check: func(t *testing.T, rec *record.FakeRecorder) { + pool := configuration.GetRootCAs() + if pool == nil { + t.Fatal("expected GetRootCAs() != nil after valid PEM") + } + cert, err := x509.ParseCertificate(certDER) + if err != nil { + t.Fatalf("parse test certificate: %v", err) + } + expected := x509.NewCertPool() + expected.AddCert(cert) + if !pool.Equal(expected) { + t.Error("loaded CA pool does not contain the expected certificate") + } + assertNoEventW(t, rec) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Seed a known non-nil pool so "nil after Reconcile" assertions are unambiguous. + testhelper.SeedRootCAsForTest(t, tc.seedPool) + + s := runtime.NewScheme() + if err := corev1.AddToScheme(s); err != nil { + t.Fatalf("AddToScheme corev1: %v", err) + } + + cl := fakectrlclient.NewClientBuilder(). + WithScheme(s). + WithRuntimeObjects(tc.objects...). + Build() + + var cl2 client.Client = cl + if tc.injectGetErr { + cl2 = &errorInjectingClient{Client: cl, getErr: errors.New("synthetic server error")} + } + + rec := record.NewFakeRecorder(10) + watcher := &configuration.CABundleWatcher{ + Client: cl2, + Recorder: rec, + Namespace: "test-ns", + } + + _, err := watcher.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: "test-ns", Name: configuration.CABundleConfigMapName}, + }) + + if (err != nil) != tc.wantErr { + t.Errorf("wantErr=%v, got err=%v", tc.wantErr, err) + } + if tc.wantErrMsg != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErrMsg)) { + t.Errorf("expected error containing %q, got %v", tc.wantErrMsg, err) + } + tc.check(t, rec) + }) + } +} + +// drainOneEventW reads one event from the recorder and returns it. +// Calls t.Fatal immediately if the channel is empty. +func drainOneEventW(t *testing.T, rec *record.FakeRecorder) string { + t.Helper() + select { + case msg := <-rec.Events: + return msg + default: + t.Fatal("expected one event on recorder, but channel was empty") + return "" + } +} + +// assertNoEventW fails the test if any event remains on the recorder. +func assertNoEventW(t *testing.T, rec *record.FakeRecorder) { + t.Helper() + select { + case msg := <-rec.Events: + t.Errorf("unexpected event: %q", msg) + default: + } +} diff --git a/controllers/configuration/tls_config.go b/controllers/configuration/tls_config.go new file mode 100644 index 000000000..30a0b56fe --- /dev/null +++ b/controllers/configuration/tls_config.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration + +import ( + "crypto/x509" + "sync" +) + +// currentRootCAs is the package-level CA pool derived from the most recently +// reconciled threescale-ca-bundle ConfigMap. +// nil means "no custom CA bundle" — callers should use system roots. +var currentRootCAs *x509.CertPool + +var tlsConfigMu sync.RWMutex + +// SetRootCAs replaces the current package-level CA pool. +// Pass nil to revert to system roots (e.g. when the ConfigMap is deleted). +func SetRootCAs(pool *x509.CertPool) { + tlsConfigMu.Lock() + currentRootCAs = pool + tlsConfigMu.Unlock() +} + +// GetRootCAs returns the current package-level CA pool, or nil if no bundle +// has been configured. Callers construct their own tls.Config and set RootCAs +// from the returned value. The pool must not be modified. +func GetRootCAs() *x509.CertPool { + tlsConfigMu.RLock() + defer tlsConfigMu.RUnlock() + return currentRootCAs +} diff --git a/doc/operator-ca-bundle.md b/doc/operator-ca-bundle.md new file mode 100644 index 000000000..107e2b10c --- /dev/null +++ b/doc/operator-ca-bundle.md @@ -0,0 +1,247 @@ +# Configuring a custom CA bundle + +The 3scale operator supports injecting a custom CA certificate bundle into the TLS configuration used when communicating with external HTTPS services. This is done via a well-known `ConfigMap` that the operator watches in the same namespace. + +When the operator finds the `threescale-ca-bundle` ConfigMap, it reads CA certificates from the `ca-bundle.crt` key and uses them as trusted root CAs for all outbound TLS connections. If the ConfigMap is absent or the key is missing, the operator falls back to the system default CA pool. + +## Creating the ConfigMap + +### Single CA certificate + +```bash +kubectl create configmap threescale-ca-bundle \ + --from-file=ca-bundle.crt=my-ca.crt \ + -n +``` + +### Multiple CA certificates (bundle) + +Concatenate all PEM root certificates into a single file before creating the ConfigMap: + +```bash +cat root-ca-1.crt root-ca-2.crt > ca-bundle.crt + +kubectl create configmap threescale-ca-bundle \ + --from-file=ca-bundle.crt=ca-bundle.crt \ + -n +``` + +### OpenShift proxy CA injection (optional) + +If your cluster uses a custom proxy CA and you need the operator to trust it, label an empty ConfigMap to have the cluster CA injection mechanism populate it with the merged proxy trust bundle: + +```bash +kubectl create configmap threescale-ca-bundle -n +kubectl label configmap threescale-ca-bundle \ + config.openshift.io/inject-trusted-cabundle=true \ + -n +``` + +> **Note:** The injected bundle contains the proxy CA and system trusted CAs, but does **not** include the ingress router CA. If you need both, use [trust-manager](#combining-ca-sources-with-trust-manager) to combine the ingress CA with the proxy trust bundle. + +## Combining CA sources with trust-manager + +Use this section when you need to merge multiple CA sources into the `threescale-ca-bundle` ConfigMap — for example, the OpenShift cluster proxy CA together with a private CA used by a 3scale instance. + +When the `config.openshift.io/inject-trusted-cabundle=true` label is set on the ConfigMap, the cluster CA injection mechanism takes ownership and replaces its contents on every update, preventing manual additions. [trust-manager](https://cert-manager.io/docs/trust/trust-manager/) solves this by merging multiple CA sources and writing the result to the target ConfigMap automatically. + +### Prerequisites + +Install cert-manager and trust-manager by following the [trust-manager installation guide](https://cert-manager.io/docs/trust/trust-manager/installation/). + +### Step 1 — Create a ConfigMap for the cluster CA source + +trust-manager reads Bundle sources from its trust namespace (`cert-manager` by default), so the injected ConfigMap must live there. Label a ConfigMap in the `cert-manager` namespace for OpenShift CA injection: + +```bash +kubectl create configmap openshift-ca -n cert-manager +kubectl label configmap openshift-ca \ + config.openshift.io/inject-trusted-cabundle=true \ + -n cert-manager +``` + +### Step 2 — Create a ConfigMap or Secret for your custom CA + +Place your custom CA in a separate ConfigMap (or Secret) also in the `cert-manager` namespace: + +```bash +kubectl create configmap my-custom-ca \ + --from-file=ca.crt=my-custom-ca.crt \ + -n cert-manager +``` + +If your custom CA is already in a cert-manager-managed Secret, you can reference it directly as a `secret` source in the next step — no copy needed. + +### Step 3 — Create a Bundle + +Create a `Bundle` that merges both sources and writes the result to `threescale-ca-bundle` in the operator namespace: + +```yaml +apiVersion: trust.cert-manager.io/v1alpha1 +kind: Bundle +metadata: + name: threescale-ca-bundle +spec: + sources: + - configMap: + name: openshift-ca # cluster CA injected in cert-manager namespace + key: ca-bundle.crt + - configMap: + name: my-custom-ca # your custom CA + key: ca.crt + target: + configMap: + key: ca-bundle.crt # must match the key the operator reads + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: +``` + +For a cert-manager-managed Secret source, copy the CA certificate into a dedicated ConfigMap and reference that instead. See the trust-manager docs on [intentionally copying CA certificates](https://cert-manager.io/docs/trust/trust-manager/#cert-manager-integration-intentionally-copying-ca-certificates) for the recommended approach. + +Apply it: + +```bash +kubectl apply -f threescale-bundle.yaml +``` + +Verify it was written to the operator namespace: + +```bash +kubectl get configmap threescale-ca-bundle -n \ + -o jsonpath='{.data.ca-bundle\.crt}' | openssl x509 -noout -subject -issuer +``` + +### Verifying the operator picked up the bundle + +Check that the operator emitted no validation errors on the ConfigMap: + +```bash +kubectl describe configmap threescale-ca-bundle -n +``` + +A successful load produces no `Warning` events. An invalid bundle shows an event similar to: + +``` +Warning InvalidCABundle CABundleWatcher InvalidCAFormat: no valid PEM-encoded certificates found in CA bundle +``` + +If you see this, verify the ConfigMap key contains well-formed PEM data (see [Verifying certificates with openssl](#verifying-certificates-with-openssl)). + +### Example ConfigMap manifest + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: threescale-ca-bundle + namespace: +data: + ca-bundle.crt: | + -----BEGIN CERTIFICATE----- + + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + + -----END CERTIFICATE----- +``` + +## Validation + +The operator validates the contents of the `ca-bundle.crt` key when the ConfigMap is created or updated. + +| Condition | Operator behaviour | +|---|---| +| ConfigMap absent | TLS uses system default CA pool; no error | +| `ca-bundle.crt` key absent | TLS uses system default CA pool; no error | +| Key present but empty | Error — `InvalidCAFormat`; `Warning` event emitted on the ConfigMap | +| Key contains no valid PEM CERTIFICATE blocks | Error — `InvalidCAFormat`; `Warning` event emitted on the ConfigMap | +| Key contains valid CERTIFICATE blocks plus other PEM block types (e.g. `PRIVATE KEY`) | Non-certificate blocks are silently skipped; certificates are loaded normally | +| Key contains one or more valid CERTIFICATE blocks | Success; TLS config updated immediately | + +When validation fails, the operator emits a Kubernetes `Warning` event on the ConfigMap (reason: `InvalidCABundle`) and keeps the last known-good TLS configuration active. Inspect these events with: + +```bash +kubectl describe configmap threescale-ca-bundle -n +``` + +## Reloading + +The operator watches the `threescale-ca-bundle` ConfigMap for create, update, and delete events. Any change is picked up automatically — **no operator restart or annotation change is required**. + +When the ConfigMap is deleted, the TLS configuration reverts to the system default CA pool immediately. + +## Verifying certificates with openssl + +### Inspect a single certificate + +```bash +openssl x509 -in my-ca.crt -text -noout +``` + +This prints the subject, issuer, validity dates, and extensions of the certificate. + +### Inspect all certificates in a bundle + +```bash +awk '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/' ca-bundle.crt \ + | awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++; f="cert-"n".pem"} {print > f}' + +for f in cert-*.pem; do + echo "=== $f ===" + openssl x509 -in "$f" -subject -issuer -dates -noout +done + +rm -f cert-*.pem +``` + +### Verify the chain of trust + +Check that a server certificate is signed by one of the CAs in the bundle: + +```bash +openssl verify -CAfile ca-bundle.crt server.crt +``` + +Expected output: + +``` +server.crt: OK +``` + +### Test a live TLS connection using the bundle + +```bash +openssl s_client -connect : -CAfile ca-bundle.crt -verify_return_error +``` + +A successful handshake shows `Verify return code: 0 (ok)` near the end of the output. + +### Check certificate expiry dates across a bundle + +```bash +openssl crl2pkcs7 -nocrl -certfile ca-bundle.crt \ + | openssl pkcs7 -print_certs -noout \ + | grep -A2 "subject" +``` + +Or using a loop: + +```bash +while openssl x509 -noout -subject -dates 2>/dev/null; do :; done < ca-bundle.crt +``` + +### Generate a self-signed CA certificate for testing + +```bash +openssl genrsa -out ca.key 2048 + +openssl req -x509 -new -nodes \ + -key ca.key \ + -sha256 \ + -days 365 \ + -out ca.crt \ + -subj "/CN=My Test CA" +``` + +The resulting `ca.crt` can be placed in the `ca-bundle.crt` key of the ConfigMap. diff --git a/doc/operator-user-guide.md b/doc/operator-user-guide.md index 0c16bdf9e..d2633ee4e 100644 --- a/doc/operator-user-guide.md +++ b/doc/operator-user-guide.md @@ -1014,6 +1014,8 @@ Whenever a controller reconciles an object it creates a new porta client to make * ProxyConfigPromote * Tenant +For production environments where the 3scale API is served by a private or self-signed CA that is not in the operator's system trust store, configure a custom CA bundle instead of disabling certificate verification. See [Configuring a Custom CA Bundle](operator-ca-bundle.md). + #### Disabling zync route generation or zync entirely If you need to disable zync entirely, this includes removing zync deployment and associated with deployment resources, you can do so by adding a new flag to APIManager: diff --git a/main.go b/main.go index 3fde8e04f..9d25629d9 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( capabilitiesv1beta1 "github.com/3scale/3scale-operator/apis/capabilities/v1beta1" appscontroller "github.com/3scale/3scale-operator/controllers/apps" capabilitiescontroller "github.com/3scale/3scale-operator/controllers/capabilities" + configurationcontroller "github.com/3scale/3scale-operator/controllers/configuration" "github.com/3scale/3scale-operator/pkg/reconcilers" "github.com/3scale/3scale-operator/version" grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" @@ -160,6 +161,16 @@ func main() { os.Exit(1) } + watcher := &configurationcontroller.CABundleWatcher{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor("CABundleWatcher"), + Namespace: operatorInstallationNamespace, + } + if err = watcher.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CABundleWatcher") + os.Exit(1) + } + secretLabelSelector, err := apimachinerymetav1.ParseToLabelSelector("apimanager.apps.3scale.net/watched-by=apimanager") if err != nil { setupLog.Error(err, "unable parse apimanager secrets label") diff --git a/pkg/controller/helper/threescale_api.go b/pkg/controller/helper/threescale_api.go index 515aeeb01..04f2b3e7f 100644 --- a/pkg/controller/helper/threescale_api.go +++ b/pkg/controller/helper/threescale_api.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" + configuration "github.com/3scale/3scale-operator/controllers/configuration" "github.com/3scale/3scale-operator/pkg/helper" threescaleapi "github.com/3scale/3scale-porta-go-client/client" @@ -19,12 +20,15 @@ type ProviderAccount struct { Token string } -// PortaClient instantiates porta_client.ThreeScaleClient from ProviderAccount object +// PortaClient instantiates a ThreeScaleClient from a ProviderAccount. +// When insecureSkipVerify is true, the CA bundle is ignored and an insecure +// client is used instead. func PortaClient(providerAccount *ProviderAccount, insecureSkipVerify bool) (*threescaleapi.ThreeScaleClient, error) { return PortaClientFromURLString(providerAccount.AdminURLStr, providerAccount.Token, insecureSkipVerify) } -// PortaClientFromURLString instantiates porta_client.ThreeScaleClient from url string +// PortaClientFromURLString instantiates a ThreeScaleClient from an admin URL string +// and access token. When insecureSkipVerify is true, the CA bundle is ignored. func PortaClientFromURLString(adminURLStr, token string, insecureSkipVerify bool) (*threescaleapi.ThreeScaleClient, error) { adminURL, err := url.Parse(adminURLStr) if err != nil { @@ -33,19 +37,22 @@ func PortaClientFromURLString(adminURLStr, token string, insecureSkipVerify bool return PortaClientFromURL(adminURL, token, insecureSkipVerify) } -// PortaClientFromURL instantiates porta_client.ThreeScaleClient from admin url object +// PortaClientFromURL instantiates a ThreeScaleClient from an admin URL +// and access token. When insecureSkipVerify is true, certificate verification +// is disabled regardless of the configured CA bundle. +// Note: Callers must re-create the client on each reconcile cycle to pick up CA rotations. func PortaClientFromURL(url *url.URL, token string, insecureSkipVerify bool) (*threescaleapi.ThreeScaleClient, error) { adminPortal, err := threescaleapi.NewAdminPortal(url.Scheme, url.Hostname(), helper.PortFromURL(url)) if err != nil { return nil, err } - - // Activated by some env var or Spec param var transport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ //nolint:gosec // G402: InsecureSkipVerify is intentional and user-controlled + RootCAs: configuration.GetRootCAs(), + InsecureSkipVerify: insecureSkipVerify, + }, } - if helper.GetEnvVar(HTTP_VERBOSE_ENVVAR, "0") == "1" { transport = &helper.Transport{Transport: transport} } diff --git a/pkg/controller/helper/threescale_api_test.go b/pkg/controller/helper/threescale_api_test.go index bad6ded7a..dd9344767 100644 --- a/pkg/controller/helper/threescale_api_test.go +++ b/pkg/controller/helper/threescale_api_test.go @@ -1,34 +1,178 @@ package helper import ( + "crypto/x509" + "encoding/json" + "net/http" + "net/http/httptest" "net/url" + "strings" "testing" + + threescaleapi "github.com/3scale/3scale-porta-go-client/client" + + "github.com/3scale/3scale-operator/pkg/testhelper" ) -func TestPortaClientInvalidURL(t *testing.T) { - providerAccount := &ProviderAccount{AdminURLStr: ":foo", Token: "some token"} - _, err := PortaClient(providerAccount, false) - assert(t, err != nil, "error should not be nil") +func backendListHandler(t *testing.T) http.Handler { + t.Helper() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(threescaleapi.BackendApiList{}); err != nil { + t.Errorf("failed to encode response: %v", err) + } + }) } func TestPortaClient(t *testing.T) { - providerAccount := &ProviderAccount{AdminURLStr: "http://somedomain.example.com", Token: "some token"} - _, err := PortaClient(providerAccount, false) - ok(t, err) -} - -func TestPortaClientFromURLStringInvalidURL(t *testing.T) { - _, err := PortaClientFromURLString(":foo", "some token", false) - assert(t, err != nil, "error should not be nil") + tests := []struct { + name string + account *ProviderAccount + wantErrContains string + }{ + { + name: "invalid URL is rejected", + account: &ProviderAccount{AdminURLStr: ":foo", Token: "some token"}, + wantErrContains: "missing protocol scheme", + }, + { + name: "valid URL produces a usable client", + account: &ProviderAccount{AdminURLStr: "http://somedomain.example.com", Token: "some token"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := PortaClient(tc.account, false) + if tc.wantErrContains != "" { + assert(t, err != nil, "error should not be nil") + assert(t, strings.Contains(err.Error(), tc.wantErrContains), + "expected %q in error, got: %v", tc.wantErrContains, err) + } else { + ok(t, err) + } + }) + } } func TestPortaClientFromURLString(t *testing.T) { - _, err := PortaClientFromURLString("http://somedomain.example.com", "some token", false) - ok(t, err) + tests := []struct { + name string + rawURL string + wantErrContains string + }{ + { + name: "invalid URL is rejected", + rawURL: ":foo", + wantErrContains: "missing protocol scheme", + }, + { + name: "valid URL produces a usable client", + rawURL: "http://somedomain.example.com", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := PortaClientFromURLString(tc.rawURL, "some token", false) + if tc.wantErrContains != "" { + assert(t, err != nil, "error should not be nil") + assert(t, strings.Contains(err.Error(), tc.wantErrContains), + "expected %q in error, got: %v", tc.wantErrContains, err) + } else { + ok(t, err) + } + }) + } } func TestPortaClientFromURL(t *testing.T) { - url := &url.URL{} - _, err := PortaClientFromURL(url, "some token", false) - assert(t, err != nil, "error should not be nil") + tests := []struct { + name string + buildURL func(t *testing.T, srv *httptest.Server) *url.URL + insecureSkipVerify bool + setupCA func(t *testing.T, srv *httptest.Server) + wantClientErrContains string + requestErrContains string + }{ + { + name: "empty URL is rejected", + buildURL: func(t *testing.T, _ *httptest.Server) *url.URL { + return &url.URL{} + }, + wantClientErrContains: "missing protocol scheme", + }, + { + name: "untrusted server certificate is rejected", + buildURL: func(t *testing.T, srv *httptest.Server) *url.URL { + u, err := url.Parse(srv.URL) + ok(t, err) + return u + }, + setupCA: func(t *testing.T, _ *httptest.Server) { + testhelper.SeedRootCAsForTest(t, nil) + }, + requestErrContains: "x509: certificate signed by unknown authority", + }, + { + name: "matching CA allows a successful request", + buildURL: func(t *testing.T, srv *httptest.Server) *url.URL { + u, err := url.Parse(srv.URL) + ok(t, err) + return u + }, + setupCA: func(t *testing.T, srv *httptest.Server) { + pool := x509.NewCertPool() + for _, cert := range srv.TLS.Certificates { + for _, c := range cert.Certificate { + parsed, err := x509.ParseCertificate(c) + ok(t, err) + pool.AddCert(parsed) + } + } + testhelper.SeedRootCAsForTest(t, pool) + }, + }, + { + name: "insecureSkipVerify accepts untrusted certificate", + buildURL: func(t *testing.T, srv *httptest.Server) *url.URL { + u, err := url.Parse(srv.URL) + ok(t, err) + return u + }, + insecureSkipVerify: true, + setupCA: func(t *testing.T, _ *httptest.Server) { + testhelper.SeedRootCAsForTest(t, nil) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewTLSServer(backendListHandler(t)) + defer srv.Close() + + if tc.setupCA != nil { + tc.setupCA(t, srv) + } + + u := tc.buildURL(t, srv) + + c, err := PortaClientFromURL(u, "token", tc.insecureSkipVerify) + if tc.wantClientErrContains != "" { + assert(t, err != nil, "expected client construction error, got nil") + assert(t, strings.Contains(err.Error(), tc.wantClientErrContains), + "expected %q in error, got: %v", tc.wantClientErrContains, err) + return + } + ok(t, err) + + _, reqErr := c.ListBackendApis() + if tc.requestErrContains != "" { + assert(t, reqErr != nil, "expected request error, got nil") + assert(t, strings.Contains(reqErr.Error(), tc.requestErrContains), + "expected %q in error, got: %v", tc.requestErrContains, reqErr) + } else { + ok(t, reqErr) + } + }) + } } diff --git a/pkg/testhelper/tls.go b/pkg/testhelper/tls.go new file mode 100644 index 000000000..a9bfa012a --- /dev/null +++ b/pkg/testhelper/tls.go @@ -0,0 +1,41 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testhelper provides shared test utilities that must be importable +// from multiple test packages across the repository. +package testhelper + +import ( + "crypto/x509" + "testing" + + configuration "github.com/3scale/3scale-operator/controllers/configuration" +) + +// SeedRootCAsForTest seeds the package-level CA pool to pool and restores the +// previous value via t.Cleanup. +// +// SetRootCAs/GetRootCAs mutate process-level global state, so any test that +// calls this helper MUST NOT call t.Parallel(). t.Setenv is the mechanism +// that enforces this: it panics when called from inside a parallel test, +// making the constraint mechanical rather than advisory. +func SeedRootCAsForTest(t testing.TB, pool *x509.CertPool) { + t.Helper() + t.Setenv("_TEST_TLS_GUARD", "1") + prev := configuration.GetRootCAs() + configuration.SetRootCAs(pool) + t.Cleanup(func() { configuration.SetRootCAs(prev) }) +}