From cd19249185804546359f668db902e4eab792bf6b Mon Sep 17 00:00:00 2001 From: Austin Barrington Date: Sat, 17 Oct 2026 22:30:37 +0100 Subject: [PATCH] Probe 0.9 proxies on their admin listener (v0.3.1) From 0.9.0-beta the proxy serves /healthz, /readyz, /metrics and /admin only on its admin listener (HYPERBYTEDB_PROXY_ADMIN_LISTEN, 8087) and answers 404 for them on the client port. The operator probed the client port, so a 0.9.1-beta proxy (shared-pool-1 on n3) restarted forever, never became ready, and every request through it failed with 502. The proxy's version is its image tag, else the cluster's version. For 0.9.0-beta and later the operator sets the admin listener explicitly (8087, or 8088 when the client port is 8087), adds an admin container port and probes it; a 0.8.x proxy renders as before. Rolling-restart exclude/include calls go to the same admin port, and for older proxies to the proxy's own port rather than the database's. The admin port is not on the proxy Service. Chart 0.3.1. --- README.md | 2 +- dist/chart/Chart.yaml | 4 +- docs/compatibility.md | 11 +- .../hyperbytedbcluster_controller.go | 4 +- internal/hyperbytedb/compat.go | 21 +++ internal/hyperbytedb/proxy.go | 56 ++++++-- internal/hyperbytedb/proxy_test.go | 136 ++++++++++++++++++ 7 files changed, 220 insertions(+), 14 deletions(-) create mode 100644 internal/hyperbytedb/proxy_test.go diff --git a/README.md b/README.md index 92dba10..9e60ae7 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ configurable values. ```sh helm upgrade hyperbytedb-operator \ oci://ghcr.io/hyperbyte-cloud/charts/hyperbytedb-operator \ - --version 0.3.0 \ + --version 0.3.1 \ --namespace hyperbytedb-operator-system \ --reuse-values ``` diff --git a/dist/chart/Chart.yaml b/dist/chart/Chart.yaml index 7e332ee..dc1b195 100644 --- a/dist/chart/Chart.yaml +++ b/dist/chart/Chart.yaml @@ -7,8 +7,8 @@ type: application # release workflow (.github/workflows/release.yml) stamps them with the git # tag (or a `0.0.0-sha.` snapshot for main branch builds) when the # chart is packaged and pushed to ghcr.io. -version: 0.3.0 -appVersion: "0.3.0" +version: 0.3.1 +appVersion: "0.3.1" home: https://hyperbyte-cloud.github.io/hyperbytedb/ sources: diff --git a/docs/compatibility.md b/docs/compatibility.md index b197559..75c68b2 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -32,11 +32,12 @@ Reasons: `Supported`, `Assumed` (no version found), `NewerThanKnown` | Operator | Renders for HyperbyteDB | Newest known | |---|---|---| +| v0.3.1 | 0.8.0 and later | 0.9.1-beta | | v0.3.0 | 0.8.0 and later | 0.9.1-beta | | v0.2.0 | 0.8.0 and later | 0.9.0-beta | | v0.1.x | not version-aware: renders the newest keys for every cluster (0.8.x ignores the ones it does not know) | — | -What v0.3.0 renders per version: +What v0.3.1 renders per version: | Capability | HyperbyteDB | For older versions the operator | |---|---|---| @@ -46,6 +47,7 @@ What v0.3.0 renders per version: | Control listener (`control-listener` annotation) | 0.9.0-beta | ignores the annotation; liveness stays on the server port | | Evacuate together (`evacuate-together` annotation) | 0.9.0-beta | ignores the annotation; departing nodes leave one at a time | | Cluster auth (`spec.auth.enabled`: auth Secret, cluster token, bootstrap admin) | 0.9.1-beta | renders `auth.enabled` only; no Secret, so a cluster with auth on cannot reach its own internal routes | +| Proxy admin listener (probes, `/metrics`, `/admin` on port 8087; the proxy's version is its image tag, else the cluster's) | 0.9.0-beta (proxy) | probes the proxy and calls its admin API on the proxy's client port, where older proxies serve them | | Parallel pod management (`parallel-pods` annotation) | any | — (operator only) | The source of truth is `CompatibilityMatrix` in @@ -53,6 +55,13 @@ The source of truth is `CompatibilityMatrix` in ## Upgrading +- v0.3.0 → v0.3.1: a proxy on 0.9.0-beta or later is probed on its admin + listener (port 8087, or 8088 when the proxy's client port is 8087). + v0.3.0 probed `/healthz` and `/readyz` on the client port, where those + proxies answer 404, so the proxy restarted forever, never became ready, + and every request through it failed with 502. The proxy Deployment of + such a cluster rolls once; a 0.8.x proxy's does not change. The admin + port is not on the proxy's Service. - v0.2.0 → v0.3.0: cluster auth needs HyperbyteDB 0.9.1-beta (0.9.0-beta has no cluster token and refuses the key). A cluster with `spec.auth.enabled` that resolves to 0.9.0-beta gets no auth Secret env, diff --git a/internal/controller/hyperbytedbcluster_controller.go b/internal/controller/hyperbytedbcluster_controller.go index 937f3c7..91de1c3 100644 --- a/internal/controller/hyperbytedbcluster_controller.go +++ b/internal/controller/hyperbytedbcluster_controller.go @@ -562,7 +562,9 @@ func (r *HyperbytedbClusterReconciler) reconcileRollingRestart( return nil, nil } - proxyPort := hyperbytedb.ServerPort(cluster) + // The proxies' admin API: its own listener from proxy 0.9.0-beta, else + // the proxy's client port (spec.proxy.port, not the database's). + proxyPort := hyperbytedb.ProxyAdminPort(cluster) stsName := hyperbytedb.StatefulSetName(cluster) state, err := r.ensureRollingRestartState(ctx, cluster, replicas) diff --git a/internal/hyperbytedb/compat.go b/internal/hyperbytedb/compat.go index f3d6b4e..5df6670 100644 --- a/internal/hyperbytedb/compat.go +++ b/internal/hyperbytedb/compat.go @@ -48,6 +48,10 @@ const ( // auth.cluster_token and auth.admin_username / admin_password: a // cluster with auth on can reach its own internal routes. CapabilityClusterAuth Capability = "cluster-auth" + // The proxy serves /healthz, /readyz, /metrics and /admin on its own + // admin listener (HYPERBYTEDB_PROXY_ADMIN_LISTEN) and answers 404 for + // them on its client port. A proxy capability: see ProxySupports. + CapabilityProxyAdminListener Capability = "proxy-admin-listener" ) // CompatibilityEntry is one row of the compatibility matrix. @@ -68,6 +72,7 @@ var CompatibilityMatrix = []CompatibilityEntry{ {CapabilityControlListener, "0.9.0-beta", "ignores the control-listener annotation (no control port, liveness on the server port)"}, {CapabilityEvacuateTogether, "0.9.0-beta", "ignores the evacuate-together annotation (departing nodes leave one at a time)"}, {CapabilityClusterAuth, "0.9.1-beta", "renders auth.enabled only; no auth Secret, so a cluster with auth on cannot reach its internal routes"}, + {CapabilityProxyAdminListener, "0.9.0-beta", "probes the proxy and calls its admin API on its client port, where proxies before 0.9.0-beta serve them"}, } // DatabaseVersion is the HyperbyteDB version a cluster is rendered for. @@ -126,6 +131,22 @@ func ResolveDatabaseVersion(cluster *v1alpha1.HyperbytedbCluster) DatabaseVersio return DatabaseVersion{Version: v, Source: "assumed", Raw: LatestDatabaseVersion} } +// ResolveProxyVersion is the version of the proxy the cluster runs: the tag +// of its proxy image when that is a version (spec.proxy.image may pin one), +// else the cluster's database version, which the proxy image follows. +func ResolveProxyVersion(cluster *v1alpha1.HyperbytedbCluster) *semver.Version { + if v, ok := parseDatabaseVersion(imageTag(ResolveProxyImage(cluster))); ok { + return v + } + return ResolveDatabaseVersion(cluster).Version +} + +// ProxySupports reports whether the cluster's proxy version has the +// capability. +func ProxySupports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool { + return versionSupports(ResolveProxyVersion(cluster), capability) +} + // Supports reports whether the cluster's database version has the capability. func Supports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool { return versionSupports(ResolveDatabaseVersion(cluster).Version, capability) diff --git a/internal/hyperbytedb/proxy.go b/internal/hyperbytedb/proxy.go index 869b747..18fb0d5 100644 --- a/internal/hyperbytedb/proxy.go +++ b/internal/hyperbytedb/proxy.go @@ -57,6 +57,27 @@ func proxyLabels(cluster *v1alpha1.HyperbytedbCluster) map[string]string { } } +const ( + // proxyAdminListenPort is where a proxy with its own admin listener + // serves probes, metrics and /admin: the proxy's default, set explicitly. + proxyAdminListenPort int32 = 8087 + // proxyAdminPortName names that port on the proxy container. + proxyAdminPortName = "admin" +) + +// ProxyAdminPort is the port of the proxy's /healthz, /readyz, /metrics and +// /admin API: its own admin listener from 0.9.0-beta, the client port +// before. It is not on the client Service; the operator calls pods by IP. +func ProxyAdminPort(cluster *v1alpha1.HyperbytedbCluster) int32 { + if !ProxySupports(cluster, CapabilityProxyAdminListener) { + return proxyPort(cluster) + } + if proxyPort(cluster) == proxyAdminListenPort { + return proxyAdminListenPort + 1 + } + return proxyAdminListenPort +} + func proxyPort(cluster *v1alpha1.HyperbytedbCluster) int32 { if cluster.Spec.Proxy != nil && cluster.Spec.Proxy.Port > 0 { return cluster.Spec.Proxy.Port @@ -175,6 +196,29 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme env = append(env, corev1.EnvVar{Name: "LOG_FORMAT", Value: "json"}) } + // From 0.9.0-beta the proxy serves probes on its admin listener and + // answers 404 for them on the client port; probing the client port kept + // such a proxy restarting and never ready. + ports := []corev1.ContainerPort{{ + Name: httpPortName, + ContainerPort: port, + Protocol: corev1.ProtocolTCP, + }} + probePort := httpPortName + if ProxySupports(cluster, CapabilityProxyAdminListener) { + admin := ProxyAdminPort(cluster) + env = append(env, corev1.EnvVar{ + Name: "HYPERBYTEDB_PROXY_ADMIN_LISTEN", + Value: fmt.Sprintf("0.0.0.0:%d", admin), + }) + ports = append(ports, corev1.ContainerPort{ + Name: proxyAdminPortName, + ContainerPort: admin, + Protocol: corev1.ProtocolTCP, + }) + probePort = proxyAdminPortName + } + podLabels := make(map[string]string) maps.Copy(podLabels, proxyLabels(cluster)) maps.Copy(podLabels, spec.PodLabels) @@ -187,18 +231,12 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme Image: image, ImagePullPolicy: pullPolicy, Env: env, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: port, - Protocol: corev1.ProtocolTCP, - }, - }, + Ports: ports, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", - Port: intstr.FromString("http"), + Port: intstr.FromString(probePort), }, }, InitialDelaySeconds: 2, @@ -209,7 +247,7 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromString("http"), + Port: intstr.FromString(probePort), }, }, InitialDelaySeconds: 1, diff --git a/internal/hyperbytedb/proxy_test.go b/internal/hyperbytedb/proxy_test.go new file mode 100644 index 0000000..d8b002b --- /dev/null +++ b/internal/hyperbytedb/proxy_test.go @@ -0,0 +1,136 @@ +package hyperbytedb + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + + v1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1" +) + +func proxyCluster(version, proxyImage string, proxyPort int32) *v1alpha1.HyperbytedbCluster { + c := versionedCluster(version, "", nil) + c.Name = "shared-pool-1" + c.Spec.Proxy = &v1alpha1.ProxySpec{Enabled: true, Image: proxyImage, Port: proxyPort} + return c +} + +func proxyContainer(t *testing.T, cluster *v1alpha1.HyperbytedbCluster) corev1.Container { + t.Helper() + cs := BuildProxyDeployment(cluster).Spec.Template.Spec.Containers + if len(cs) != 1 { + t.Fatalf("want one proxy container, got %d", len(cs)) + } + return cs[0] +} + +func envValue(c corev1.Container, name string) (string, bool) { + for _, e := range c.Env { + if e.Name == name { + return e.Value, true + } + } + return "", false +} + +func portNumber(c corev1.Container, name string) int32 { + for _, p := range c.Ports { + if p.Name == name { + return p.ContainerPort + } + } + return 0 +} + +// Proxy 0.9.0-beta and later serve /healthz and /readyz only on their admin +// listener and answer 404 on the client port. Probing the client port kept +// shared-pool-1's 0.9.1-beta proxy restarting and never ready, so every +// request through it failed with 502. +func TestA091ProxyIsProbedOnItsAdminListener(t *testing.T) { + c := proxyContainer(t, proxyCluster("v0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy", 0)) + + if got := c.LivenessProbe.HTTPGet.Port.String(); got != proxyAdminPortName { + t.Errorf("liveness probe port = %s, want %s", got, proxyAdminPortName) + } + if got := c.ReadinessProbe.HTTPGet.Port.String(); got != proxyAdminPortName { + t.Errorf("readiness probe port = %s, want %s", got, proxyAdminPortName) + } + if got := portNumber(c, proxyAdminPortName); got != 8087 { + t.Errorf("admin container port = %d, want 8087", got) + } + if v, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); !ok || v != "0.0.0.0:8087" { + t.Errorf("HYPERBYTEDB_PROXY_ADMIN_LISTEN = %q (set %v), want 0.0.0.0:8087", v, ok) + } + if got := portNumber(c, httpPortName); got != 8086 { + t.Errorf("client port = %d, want 8086", got) + } +} + +// A 0.8.5 proxy serves everything on one port: its pod is rendered as before, +// so upgrading the operator does not restart it. +func TestAn085ProxyKeepsItsProbesOnTheClientPort(t *testing.T) { + c := proxyContainer(t, proxyCluster("0.8.5-beta", "", 0)) + + if got := c.LivenessProbe.HTTPGet.Port.String(); got != httpPortName { + t.Errorf("liveness probe port = %s, want %s", got, httpPortName) + } + if got := c.ReadinessProbe.HTTPGet.Port.String(); got != httpPortName { + t.Errorf("readiness probe port = %s, want %s", got, httpPortName) + } + if _, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); ok { + t.Error("a 0.8.5 proxy got an admin listener") + } + if len(c.Ports) != 1 { + t.Errorf("ports = %v, want only the client port", c.Ports) + } +} + +// The proxy's version is its image's: a proxy image pinned to its own tag +// wins over the cluster's version, and an unparseable tag falls back to it. +func TestTheProxyVersionFollowsItsOwnImageTag(t *testing.T) { + cases := []struct { + name, version, image string + admin bool + }{ + {"follows spec.version", "0.9.1-beta", "", true}, + {"old proxy pinned on a new cluster", "0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.8.5-beta", false}, + {"new proxy pinned on an old cluster", "0.8.5-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.9.1-beta", true}, + {"unparseable tag follows the database", "0.8.5-beta", "registry.local/proxy:e2e", false}, + } + for _, tc := range cases { + got := ProxySupports(proxyCluster(tc.version, tc.image, 0), CapabilityProxyAdminListener) + if got != tc.admin { + t.Errorf("%s: admin listener = %v, want %v", tc.name, got, tc.admin) + } + } +} + +// The operator calls each proxy pod's admin API (exclude / include during +// rolling restarts) on ProxyAdminPort: the admin listener for 0.9 proxies, +// the proxy's own client port before. It used the database's port, which +// is only right when spec.proxy.port is unset. +func TestTheProxyAdminPort(t *testing.T) { + if got := ProxyAdminPort(proxyCluster("0.9.1-beta", "", 0)); got != 8087 { + t.Errorf("0.9.1 proxy: %d, want 8087", got) + } + if got := ProxyAdminPort(proxyCluster("0.8.5-beta", "", 9000)); got != 9000 { + t.Errorf("0.8.5 proxy on port 9000: %d, want 9000", got) + } + // A client port of 8087 moves the admin listener off it. + c := proxyCluster("0.9.1-beta", "", 8087) + if got := ProxyAdminPort(c); got != 8088 { + t.Errorf("0.9.1 proxy on client port 8087: admin %d, want 8088", got) + } + pc := proxyContainer(t, c) + if portNumber(pc, httpPortName) == portNumber(pc, proxyAdminPortName) { + t.Errorf("client and admin ports collide: %v", pc.Ports) + } +} + +// The admin port is not on the client Service: /admin can exclude backends. +func TestTheProxyServiceExposesOnlyTheClientPort(t *testing.T) { + svc := BuildProxyService(proxyCluster("0.9.1-beta", "", 0)) + if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Name != httpPortName { + t.Errorf("service ports = %v, want only %s", svc.Spec.Ports, httpPortName) + } +}