Probe 0.9 proxies on their admin listener (v0.3.1)

README.md

@@ -76,7 +76,7 @@ configurable values.
 ```sh
 helm upgrade hyperbytedb-operator \
   oci://ghcr.io/hyperbyte-cloud/charts/hyperbytedb-operator \
-  --version 0.3.0 \
+  --version 0.3.1 \
   --namespace hyperbytedb-operator-system \
   --reuse-values
 ```

dist/chart/Chart.yaml

@@ -7,8 +7,8 @@ type: application
 # release workflow (.github/workflows/release.yml) stamps them with the git
 # tag (or a `0.0.0-sha.<short>` snapshot for main branch builds) when the
 # chart is packaged and pushed to ghcr.io.
-version: 0.3.0
-appVersion: "0.3.0"
+version: 0.3.1
+appVersion: "0.3.1"
 
 home: https://hyperbyte-cloud.github.io/hyperbytedb/
 sources:

docs/compatibility.md

@@ -32,11 +32,12 @@ Reasons: `Supported`, `Assumed` (no version found), `NewerThanKnown`
 
 | Operator | Renders for HyperbyteDB | Newest known |
 |---|---|---|
+| v0.3.1 | 0.8.0 and later | 0.9.1-beta |
 | v0.3.0 | 0.8.0 and later | 0.9.1-beta |
 | v0.2.0 | 0.8.0 and later | 0.9.0-beta |
 | v0.1.x | not version-aware: renders the newest keys for every cluster (0.8.x ignores the ones it does not know) | — |
 
-What v0.3.0 renders per version:
+What v0.3.1 renders per version:
 
 | Capability | HyperbyteDB | For older versions the operator |
 |---|---|---|
@@ -46,13 +47,21 @@ What v0.3.0 renders per version:
 | Control listener (`control-listener` annotation) | 0.9.0-beta | ignores the annotation; liveness stays on the server port |
 | Evacuate together (`evacuate-together` annotation) | 0.9.0-beta | ignores the annotation; departing nodes leave one at a time |
 | Cluster auth (`spec.auth.enabled`: auth Secret, cluster token, bootstrap admin) | 0.9.1-beta | renders `auth.enabled` only; no Secret, so a cluster with auth on cannot reach its own internal routes |
+| Proxy admin listener (probes, `/metrics`, `/admin` on port 8087; the proxy's version is its image tag, else the cluster's) | 0.9.0-beta (proxy) | probes the proxy and calls its admin API on the proxy's client port, where older proxies serve them |
 | Parallel pod management (`parallel-pods` annotation) | any | — (operator only) |
 
 The source of truth is `CompatibilityMatrix` in
 `internal/hyperbytedb/compat.go`; keep this page in step with it.
 
 ## Upgrading
 
+- v0.3.0 → v0.3.1: a proxy on 0.9.0-beta or later is probed on its admin
+  listener (port 8087, or 8088 when the proxy's client port is 8087).
+  v0.3.0 probed `/healthz` and `/readyz` on the client port, where those
+  proxies answer 404, so the proxy restarted forever, never became ready,
+  and every request through it failed with 502. The proxy Deployment of
+  such a cluster rolls once; a 0.8.x proxy's does not change. The admin
+  port is not on the proxy's Service.
 - v0.2.0 → v0.3.0: cluster auth needs HyperbyteDB 0.9.1-beta (0.9.0-beta
   has no cluster token and refuses the key). A cluster with
   `spec.auth.enabled` that resolves to 0.9.0-beta gets no auth Secret env,

internal/controller/hyperbytedbcluster_controller.go

@@ -562,7 +562,9 @@ func (r *HyperbytedbClusterReconciler) reconcileRollingRestart(
 		return nil, nil
 	}
 
-	proxyPort := hyperbytedb.ServerPort(cluster)
+	// The proxies' admin API: its own listener from proxy 0.9.0-beta, else
+	// the proxy's client port (spec.proxy.port, not the database's).
+	proxyPort := hyperbytedb.ProxyAdminPort(cluster)
 	stsName := hyperbytedb.StatefulSetName(cluster)
 
 	state, err := r.ensureRollingRestartState(ctx, cluster, replicas)

internal/hyperbytedb/compat.go

@@ -48,6 +48,10 @@ const (
 	// auth.cluster_token and auth.admin_username / admin_password: a
 	// cluster with auth on can reach its own internal routes.
 	CapabilityClusterAuth Capability = "cluster-auth"
+	// The proxy serves /healthz, /readyz, /metrics and /admin on its own
+	// admin listener (HYPERBYTEDB_PROXY_ADMIN_LISTEN) and answers 404 for
+	// them on its client port. A proxy capability: see ProxySupports.
+	CapabilityProxyAdminListener Capability = "proxy-admin-listener"
 )
 
 // CompatibilityEntry is one row of the compatibility matrix.
@@ -68,6 +72,7 @@ var CompatibilityMatrix = []CompatibilityEntry{
 	{CapabilityControlListener, "0.9.0-beta", "ignores the control-listener annotation (no control port, liveness on the server port)"},
 	{CapabilityEvacuateTogether, "0.9.0-beta", "ignores the evacuate-together annotation (departing nodes leave one at a time)"},
 	{CapabilityClusterAuth, "0.9.1-beta", "renders auth.enabled only; no auth Secret, so a cluster with auth on cannot reach its internal routes"},
+	{CapabilityProxyAdminListener, "0.9.0-beta", "probes the proxy and calls its admin API on its client port, where proxies before 0.9.0-beta serve them"},
 }
 
 // DatabaseVersion is the HyperbyteDB version a cluster is rendered for.
@@ -126,6 +131,22 @@ func ResolveDatabaseVersion(cluster *v1alpha1.HyperbytedbCluster) DatabaseVersio
 	return DatabaseVersion{Version: v, Source: "assumed", Raw: LatestDatabaseVersion}
 }
 
+// ResolveProxyVersion is the version of the proxy the cluster runs: the tag
+// of its proxy image when that is a version (spec.proxy.image may pin one),
+// else the cluster's database version, which the proxy image follows.
+func ResolveProxyVersion(cluster *v1alpha1.HyperbytedbCluster) *semver.Version {
+	if v, ok := parseDatabaseVersion(imageTag(ResolveProxyImage(cluster))); ok {
+		return v
+	}
+	return ResolveDatabaseVersion(cluster).Version
+}
+
+// ProxySupports reports whether the cluster's proxy version has the
+// capability.
+func ProxySupports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool {
+	return versionSupports(ResolveProxyVersion(cluster), capability)
+}
+
 // Supports reports whether the cluster's database version has the capability.
 func Supports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool {
 	return versionSupports(ResolveDatabaseVersion(cluster).Version, capability)

internal/hyperbytedb/proxy.go

@@ -57,6 +57,27 @@ func proxyLabels(cluster *v1alpha1.HyperbytedbCluster) map[string]string {
 	}
 }
 
+const (
+	// proxyAdminListenPort is where a proxy with its own admin listener
+	// serves probes, metrics and /admin: the proxy's default, set explicitly.
+	proxyAdminListenPort int32 = 8087
+	// proxyAdminPortName names that port on the proxy container.
+	proxyAdminPortName = "admin"
+)
+
+// ProxyAdminPort is the port of the proxy's /healthz, /readyz, /metrics and
+// /admin API: its own admin listener from 0.9.0-beta, the client port
+// before. It is not on the client Service; the operator calls pods by IP.
+func ProxyAdminPort(cluster *v1alpha1.HyperbytedbCluster) int32 {
+	if !ProxySupports(cluster, CapabilityProxyAdminListener) {
+		return proxyPort(cluster)
+	}
+	if proxyPort(cluster) == proxyAdminListenPort {
+		return proxyAdminListenPort + 1
+	}
+	return proxyAdminListenPort
+}
+
 func proxyPort(cluster *v1alpha1.HyperbytedbCluster) int32 {
 	if cluster.Spec.Proxy != nil && cluster.Spec.Proxy.Port > 0 {
 		return cluster.Spec.Proxy.Port
@@ -175,6 +196,29 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme
 		env = append(env, corev1.EnvVar{Name: "LOG_FORMAT", Value: "json"})
 	}
 
+	// From 0.9.0-beta the proxy serves probes on its admin listener and
+	// answers 404 for them on the client port; probing the client port kept
+	// such a proxy restarting and never ready.
+	ports := []corev1.ContainerPort{{
+		Name:          httpPortName,
+		ContainerPort: port,
+		Protocol:      corev1.ProtocolTCP,
+	}}
+	probePort := httpPortName
+	if ProxySupports(cluster, CapabilityProxyAdminListener) {
+		admin := ProxyAdminPort(cluster)
+		env = append(env, corev1.EnvVar{
+			Name:  "HYPERBYTEDB_PROXY_ADMIN_LISTEN",
+			Value: fmt.Sprintf("0.0.0.0:%d", admin),
+		})
+		ports = append(ports, corev1.ContainerPort{
+			Name:          proxyAdminPortName,
+			ContainerPort: admin,
+			Protocol:      corev1.ProtocolTCP,
+		})
+		probePort = proxyAdminPortName
+	}
+
 	podLabels := make(map[string]string)
 	maps.Copy(podLabels, proxyLabels(cluster))
 	maps.Copy(podLabels, spec.PodLabels)
@@ -187,18 +231,12 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme
 		Image:           image,
 		ImagePullPolicy: pullPolicy,
 		Env:             env,
-		Ports: []corev1.ContainerPort{
-			{
-				Name:          "http",
-				ContainerPort: port,
-				Protocol:      corev1.ProtocolTCP,
-			},
-		},
+		Ports:           ports,
 		LivenessProbe: &corev1.Probe{
 			ProbeHandler: corev1.ProbeHandler{
 				HTTPGet: &corev1.HTTPGetAction{
 					Path: "/healthz",
-					Port: intstr.FromString("http"),
+					Port: intstr.FromString(probePort),
 				},
 			},
 			InitialDelaySeconds: 2,
@@ -209,7 +247,7 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme
 			ProbeHandler: corev1.ProbeHandler{
 				HTTPGet: &corev1.HTTPGetAction{
 					Path: "/readyz",
-					Port: intstr.FromString("http"),
+					Port: intstr.FromString(probePort),
 				},
 			},
 			InitialDelaySeconds: 1,

internal/hyperbytedb/proxy_test.go

@@ -0,0 +1,136 @@
+package hyperbytedb
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+
+	v1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1"
+)
+
+func proxyCluster(version, proxyImage string, proxyPort int32) *v1alpha1.HyperbytedbCluster {
+	c := versionedCluster(version, "", nil)
+	c.Name = "shared-pool-1"
+	c.Spec.Proxy = &v1alpha1.ProxySpec{Enabled: true, Image: proxyImage, Port: proxyPort}
+	return c
+}
+
+func proxyContainer(t *testing.T, cluster *v1alpha1.HyperbytedbCluster) corev1.Container {
+	t.Helper()
+	cs := BuildProxyDeployment(cluster).Spec.Template.Spec.Containers
+	if len(cs) != 1 {
+		t.Fatalf("want one proxy container, got %d", len(cs))
+	}
+	return cs[0]
+}
+
+func envValue(c corev1.Container, name string) (string, bool) {
+	for _, e := range c.Env {
+		if e.Name == name {
+			return e.Value, true
+		}
+	}
+	return "", false
+}
+
+func portNumber(c corev1.Container, name string) int32 {
+	for _, p := range c.Ports {
+		if p.Name == name {
+			return p.ContainerPort
+		}
+	}
+	return 0
+}
+
+// Proxy 0.9.0-beta and later serve /healthz and /readyz only on their admin
+// listener and answer 404 on the client port. Probing the client port kept
+// shared-pool-1's 0.9.1-beta proxy restarting and never ready, so every
+// request through it failed with 502.
+func TestA091ProxyIsProbedOnItsAdminListener(t *testing.T) {
+	c := proxyContainer(t, proxyCluster("v0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy", 0))
+
+	if got := c.LivenessProbe.HTTPGet.Port.String(); got != proxyAdminPortName {
+		t.Errorf("liveness probe port = %s, want %s", got, proxyAdminPortName)
+	}
+	if got := c.ReadinessProbe.HTTPGet.Port.String(); got != proxyAdminPortName {
+		t.Errorf("readiness probe port = %s, want %s", got, proxyAdminPortName)
+	}
+	if got := portNumber(c, proxyAdminPortName); got != 8087 {
+		t.Errorf("admin container port = %d, want 8087", got)
+	}
+	if v, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); !ok || v != "0.0.0.0:8087" {
+		t.Errorf("HYPERBYTEDB_PROXY_ADMIN_LISTEN = %q (set %v), want 0.0.0.0:8087", v, ok)
+	}
+	if got := portNumber(c, httpPortName); got != 8086 {
+		t.Errorf("client port = %d, want 8086", got)
+	}
+}
+
+// A 0.8.5 proxy serves everything on one port: its pod is rendered as before,
+// so upgrading the operator does not restart it.
+func TestAn085ProxyKeepsItsProbesOnTheClientPort(t *testing.T) {
+	c := proxyContainer(t, proxyCluster("0.8.5-beta", "", 0))
+
+	if got := c.LivenessProbe.HTTPGet.Port.String(); got != httpPortName {
+		t.Errorf("liveness probe port = %s, want %s", got, httpPortName)
+	}
+	if got := c.ReadinessProbe.HTTPGet.Port.String(); got != httpPortName {
+		t.Errorf("readiness probe port = %s, want %s", got, httpPortName)
+	}
+	if _, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); ok {
+		t.Error("a 0.8.5 proxy got an admin listener")
+	}
+	if len(c.Ports) != 1 {
+		t.Errorf("ports = %v, want only the client port", c.Ports)
+	}
+}
+
+// The proxy's version is its image's: a proxy image pinned to its own tag
+// wins over the cluster's version, and an unparseable tag falls back to it.
+func TestTheProxyVersionFollowsItsOwnImageTag(t *testing.T) {
+	cases := []struct {
+		name, version, image string
+		admin                bool
+	}{
+		{"follows spec.version", "0.9.1-beta", "", true},
+		{"old proxy pinned on a new cluster", "0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.8.5-beta", false},
+		{"new proxy pinned on an old cluster", "0.8.5-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.9.1-beta", true},
+		{"unparseable tag follows the database", "0.8.5-beta", "registry.local/proxy:e2e", false},
+	}
+	for _, tc := range cases {
+		got := ProxySupports(proxyCluster(tc.version, tc.image, 0), CapabilityProxyAdminListener)
+		if got != tc.admin {
+			t.Errorf("%s: admin listener = %v, want %v", tc.name, got, tc.admin)
+		}
+	}
+}
+
+// The operator calls each proxy pod's admin API (exclude / include during
+// rolling restarts) on ProxyAdminPort: the admin listener for 0.9 proxies,
+// the proxy's own client port before. It used the database's port, which
+// is only right when spec.proxy.port is unset.
+func TestTheProxyAdminPort(t *testing.T) {
+	if got := ProxyAdminPort(proxyCluster("0.9.1-beta", "", 0)); got != 8087 {
+		t.Errorf("0.9.1 proxy: %d, want 8087", got)
+	}
+	if got := ProxyAdminPort(proxyCluster("0.8.5-beta", "", 9000)); got != 9000 {
+		t.Errorf("0.8.5 proxy on port 9000: %d, want 9000", got)
+	}
+	// A client port of 8087 moves the admin listener off it.
+	c := proxyCluster("0.9.1-beta", "", 8087)
+	if got := ProxyAdminPort(c); got != 8088 {
+		t.Errorf("0.9.1 proxy on client port 8087: admin %d, want 8088", got)
+	}
+	pc := proxyContainer(t, c)
+	if portNumber(pc, httpPortName) == portNumber(pc, proxyAdminPortName) {
+		t.Errorf("client and admin ports collide: %v", pc.Ports)
+	}
+}
+
+// The admin port is not on the client Service: /admin can exclude backends.
+func TestTheProxyServiceExposesOnlyTheClientPort(t *testing.T) {
+	svc := BuildProxyService(proxyCluster("0.9.1-beta", "", 0))
+	if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Name != httpPortName {
+		t.Errorf("service ports = %v, want only %s", svc.Spec.Ports, httpPortName)
+	}
+}