diff --git a/README.md b/README.md index 92dba10..9e60ae7 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ configurable values. ```sh helm upgrade hyperbytedb-operator \ oci://ghcr.io/hyperbyte-cloud/charts/hyperbytedb-operator \ - --version 0.3.0 \ + --version 0.3.1 \ --namespace hyperbytedb-operator-system \ --reuse-values ``` diff --git a/dist/chart/Chart.yaml b/dist/chart/Chart.yaml index 7e332ee..dc1b195 100644 --- a/dist/chart/Chart.yaml +++ b/dist/chart/Chart.yaml @@ -7,8 +7,8 @@ type: application # release workflow (.github/workflows/release.yml) stamps them with the git # tag (or a `0.0.0-sha.` snapshot for main branch builds) when the # chart is packaged and pushed to ghcr.io. -version: 0.3.0 -appVersion: "0.3.0" +version: 0.3.1 +appVersion: "0.3.1" home: https://hyperbyte-cloud.github.io/hyperbytedb/ sources: diff --git a/docs/compatibility.md b/docs/compatibility.md index b197559..75c68b2 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -32,11 +32,12 @@ Reasons: `Supported`, `Assumed` (no version found), `NewerThanKnown` | Operator | Renders for HyperbyteDB | Newest known | |---|---|---| +| v0.3.1 | 0.8.0 and later | 0.9.1-beta | | v0.3.0 | 0.8.0 and later | 0.9.1-beta | | v0.2.0 | 0.8.0 and later | 0.9.0-beta | | v0.1.x | not version-aware: renders the newest keys for every cluster (0.8.x ignores the ones it does not know) | — | -What v0.3.0 renders per version: +What v0.3.1 renders per version: | Capability | HyperbyteDB | For older versions the operator | |---|---|---| @@ -46,6 +47,7 @@ What v0.3.0 renders per version: | Control listener (`control-listener` annotation) | 0.9.0-beta | ignores the annotation; liveness stays on the server port | | Evacuate together (`evacuate-together` annotation) | 0.9.0-beta | ignores the annotation; departing nodes leave one at a time | | Cluster auth (`spec.auth.enabled`: auth Secret, cluster token, bootstrap admin) | 0.9.1-beta | renders `auth.enabled` only; no Secret, so a cluster with auth on cannot reach its own internal routes | +| Proxy admin listener (probes, `/metrics`, `/admin` on port 8087; the proxy's version is its image tag, else the cluster's) | 0.9.0-beta (proxy) | probes the proxy and calls its admin API on the proxy's client port, where older proxies serve them | | Parallel pod management (`parallel-pods` annotation) | any | — (operator only) | The source of truth is `CompatibilityMatrix` in @@ -53,6 +55,13 @@ The source of truth is `CompatibilityMatrix` in ## Upgrading +- v0.3.0 → v0.3.1: a proxy on 0.9.0-beta or later is probed on its admin + listener (port 8087, or 8088 when the proxy's client port is 8087). + v0.3.0 probed `/healthz` and `/readyz` on the client port, where those + proxies answer 404, so the proxy restarted forever, never became ready, + and every request through it failed with 502. The proxy Deployment of + such a cluster rolls once; a 0.8.x proxy's does not change. The admin + port is not on the proxy's Service. - v0.2.0 → v0.3.0: cluster auth needs HyperbyteDB 0.9.1-beta (0.9.0-beta has no cluster token and refuses the key). A cluster with `spec.auth.enabled` that resolves to 0.9.0-beta gets no auth Secret env, diff --git a/internal/controller/hyperbytedbcluster_controller.go b/internal/controller/hyperbytedbcluster_controller.go index 937f3c7..91de1c3 100644 --- a/internal/controller/hyperbytedbcluster_controller.go +++ b/internal/controller/hyperbytedbcluster_controller.go @@ -562,7 +562,9 @@ func (r *HyperbytedbClusterReconciler) reconcileRollingRestart( return nil, nil } - proxyPort := hyperbytedb.ServerPort(cluster) + // The proxies' admin API: its own listener from proxy 0.9.0-beta, else + // the proxy's client port (spec.proxy.port, not the database's). + proxyPort := hyperbytedb.ProxyAdminPort(cluster) stsName := hyperbytedb.StatefulSetName(cluster) state, err := r.ensureRollingRestartState(ctx, cluster, replicas) diff --git a/internal/hyperbytedb/compat.go b/internal/hyperbytedb/compat.go index f3d6b4e..5df6670 100644 --- a/internal/hyperbytedb/compat.go +++ b/internal/hyperbytedb/compat.go @@ -48,6 +48,10 @@ const ( // auth.cluster_token and auth.admin_username / admin_password: a // cluster with auth on can reach its own internal routes. CapabilityClusterAuth Capability = "cluster-auth" + // The proxy serves /healthz, /readyz, /metrics and /admin on its own + // admin listener (HYPERBYTEDB_PROXY_ADMIN_LISTEN) and answers 404 for + // them on its client port. A proxy capability: see ProxySupports. + CapabilityProxyAdminListener Capability = "proxy-admin-listener" ) // CompatibilityEntry is one row of the compatibility matrix. @@ -68,6 +72,7 @@ var CompatibilityMatrix = []CompatibilityEntry{ {CapabilityControlListener, "0.9.0-beta", "ignores the control-listener annotation (no control port, liveness on the server port)"}, {CapabilityEvacuateTogether, "0.9.0-beta", "ignores the evacuate-together annotation (departing nodes leave one at a time)"}, {CapabilityClusterAuth, "0.9.1-beta", "renders auth.enabled only; no auth Secret, so a cluster with auth on cannot reach its internal routes"}, + {CapabilityProxyAdminListener, "0.9.0-beta", "probes the proxy and calls its admin API on its client port, where proxies before 0.9.0-beta serve them"}, } // DatabaseVersion is the HyperbyteDB version a cluster is rendered for. @@ -126,6 +131,22 @@ func ResolveDatabaseVersion(cluster *v1alpha1.HyperbytedbCluster) DatabaseVersio return DatabaseVersion{Version: v, Source: "assumed", Raw: LatestDatabaseVersion} } +// ResolveProxyVersion is the version of the proxy the cluster runs: the tag +// of its proxy image when that is a version (spec.proxy.image may pin one), +// else the cluster's database version, which the proxy image follows. +func ResolveProxyVersion(cluster *v1alpha1.HyperbytedbCluster) *semver.Version { + if v, ok := parseDatabaseVersion(imageTag(ResolveProxyImage(cluster))); ok { + return v + } + return ResolveDatabaseVersion(cluster).Version +} + +// ProxySupports reports whether the cluster's proxy version has the +// capability. +func ProxySupports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool { + return versionSupports(ResolveProxyVersion(cluster), capability) +} + // Supports reports whether the cluster's database version has the capability. func Supports(cluster *v1alpha1.HyperbytedbCluster, capability Capability) bool { return versionSupports(ResolveDatabaseVersion(cluster).Version, capability) diff --git a/internal/hyperbytedb/proxy.go b/internal/hyperbytedb/proxy.go index 869b747..18fb0d5 100644 --- a/internal/hyperbytedb/proxy.go +++ b/internal/hyperbytedb/proxy.go @@ -57,6 +57,27 @@ func proxyLabels(cluster *v1alpha1.HyperbytedbCluster) map[string]string { } } +const ( + // proxyAdminListenPort is where a proxy with its own admin listener + // serves probes, metrics and /admin: the proxy's default, set explicitly. + proxyAdminListenPort int32 = 8087 + // proxyAdminPortName names that port on the proxy container. + proxyAdminPortName = "admin" +) + +// ProxyAdminPort is the port of the proxy's /healthz, /readyz, /metrics and +// /admin API: its own admin listener from 0.9.0-beta, the client port +// before. It is not on the client Service; the operator calls pods by IP. +func ProxyAdminPort(cluster *v1alpha1.HyperbytedbCluster) int32 { + if !ProxySupports(cluster, CapabilityProxyAdminListener) { + return proxyPort(cluster) + } + if proxyPort(cluster) == proxyAdminListenPort { + return proxyAdminListenPort + 1 + } + return proxyAdminListenPort +} + func proxyPort(cluster *v1alpha1.HyperbytedbCluster) int32 { if cluster.Spec.Proxy != nil && cluster.Spec.Proxy.Port > 0 { return cluster.Spec.Proxy.Port @@ -175,6 +196,29 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme env = append(env, corev1.EnvVar{Name: "LOG_FORMAT", Value: "json"}) } + // From 0.9.0-beta the proxy serves probes on its admin listener and + // answers 404 for them on the client port; probing the client port kept + // such a proxy restarting and never ready. + ports := []corev1.ContainerPort{{ + Name: httpPortName, + ContainerPort: port, + Protocol: corev1.ProtocolTCP, + }} + probePort := httpPortName + if ProxySupports(cluster, CapabilityProxyAdminListener) { + admin := ProxyAdminPort(cluster) + env = append(env, corev1.EnvVar{ + Name: "HYPERBYTEDB_PROXY_ADMIN_LISTEN", + Value: fmt.Sprintf("0.0.0.0:%d", admin), + }) + ports = append(ports, corev1.ContainerPort{ + Name: proxyAdminPortName, + ContainerPort: admin, + Protocol: corev1.ProtocolTCP, + }) + probePort = proxyAdminPortName + } + podLabels := make(map[string]string) maps.Copy(podLabels, proxyLabels(cluster)) maps.Copy(podLabels, spec.PodLabels) @@ -187,18 +231,12 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme Image: image, ImagePullPolicy: pullPolicy, Env: env, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: port, - Protocol: corev1.ProtocolTCP, - }, - }, + Ports: ports, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", - Port: intstr.FromString("http"), + Port: intstr.FromString(probePort), }, }, InitialDelaySeconds: 2, @@ -209,7 +247,7 @@ func BuildProxyDeployment(cluster *v1alpha1.HyperbytedbCluster) *appsv1.Deployme ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromString("http"), + Port: intstr.FromString(probePort), }, }, InitialDelaySeconds: 1, diff --git a/internal/hyperbytedb/proxy_test.go b/internal/hyperbytedb/proxy_test.go new file mode 100644 index 0000000..d8b002b --- /dev/null +++ b/internal/hyperbytedb/proxy_test.go @@ -0,0 +1,136 @@ +package hyperbytedb + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + + v1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1" +) + +func proxyCluster(version, proxyImage string, proxyPort int32) *v1alpha1.HyperbytedbCluster { + c := versionedCluster(version, "", nil) + c.Name = "shared-pool-1" + c.Spec.Proxy = &v1alpha1.ProxySpec{Enabled: true, Image: proxyImage, Port: proxyPort} + return c +} + +func proxyContainer(t *testing.T, cluster *v1alpha1.HyperbytedbCluster) corev1.Container { + t.Helper() + cs := BuildProxyDeployment(cluster).Spec.Template.Spec.Containers + if len(cs) != 1 { + t.Fatalf("want one proxy container, got %d", len(cs)) + } + return cs[0] +} + +func envValue(c corev1.Container, name string) (string, bool) { + for _, e := range c.Env { + if e.Name == name { + return e.Value, true + } + } + return "", false +} + +func portNumber(c corev1.Container, name string) int32 { + for _, p := range c.Ports { + if p.Name == name { + return p.ContainerPort + } + } + return 0 +} + +// Proxy 0.9.0-beta and later serve /healthz and /readyz only on their admin +// listener and answer 404 on the client port. Probing the client port kept +// shared-pool-1's 0.9.1-beta proxy restarting and never ready, so every +// request through it failed with 502. +func TestA091ProxyIsProbedOnItsAdminListener(t *testing.T) { + c := proxyContainer(t, proxyCluster("v0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy", 0)) + + if got := c.LivenessProbe.HTTPGet.Port.String(); got != proxyAdminPortName { + t.Errorf("liveness probe port = %s, want %s", got, proxyAdminPortName) + } + if got := c.ReadinessProbe.HTTPGet.Port.String(); got != proxyAdminPortName { + t.Errorf("readiness probe port = %s, want %s", got, proxyAdminPortName) + } + if got := portNumber(c, proxyAdminPortName); got != 8087 { + t.Errorf("admin container port = %d, want 8087", got) + } + if v, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); !ok || v != "0.0.0.0:8087" { + t.Errorf("HYPERBYTEDB_PROXY_ADMIN_LISTEN = %q (set %v), want 0.0.0.0:8087", v, ok) + } + if got := portNumber(c, httpPortName); got != 8086 { + t.Errorf("client port = %d, want 8086", got) + } +} + +// A 0.8.5 proxy serves everything on one port: its pod is rendered as before, +// so upgrading the operator does not restart it. +func TestAn085ProxyKeepsItsProbesOnTheClientPort(t *testing.T) { + c := proxyContainer(t, proxyCluster("0.8.5-beta", "", 0)) + + if got := c.LivenessProbe.HTTPGet.Port.String(); got != httpPortName { + t.Errorf("liveness probe port = %s, want %s", got, httpPortName) + } + if got := c.ReadinessProbe.HTTPGet.Port.String(); got != httpPortName { + t.Errorf("readiness probe port = %s, want %s", got, httpPortName) + } + if _, ok := envValue(c, "HYPERBYTEDB_PROXY_ADMIN_LISTEN"); ok { + t.Error("a 0.8.5 proxy got an admin listener") + } + if len(c.Ports) != 1 { + t.Errorf("ports = %v, want only the client port", c.Ports) + } +} + +// The proxy's version is its image's: a proxy image pinned to its own tag +// wins over the cluster's version, and an unparseable tag falls back to it. +func TestTheProxyVersionFollowsItsOwnImageTag(t *testing.T) { + cases := []struct { + name, version, image string + admin bool + }{ + {"follows spec.version", "0.9.1-beta", "", true}, + {"old proxy pinned on a new cluster", "0.9.1-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.8.5-beta", false}, + {"new proxy pinned on an old cluster", "0.8.5-beta", "ghcr.io/hyperbyte-cloud/hyperbytedb-proxy:0.9.1-beta", true}, + {"unparseable tag follows the database", "0.8.5-beta", "registry.local/proxy:e2e", false}, + } + for _, tc := range cases { + got := ProxySupports(proxyCluster(tc.version, tc.image, 0), CapabilityProxyAdminListener) + if got != tc.admin { + t.Errorf("%s: admin listener = %v, want %v", tc.name, got, tc.admin) + } + } +} + +// The operator calls each proxy pod's admin API (exclude / include during +// rolling restarts) on ProxyAdminPort: the admin listener for 0.9 proxies, +// the proxy's own client port before. It used the database's port, which +// is only right when spec.proxy.port is unset. +func TestTheProxyAdminPort(t *testing.T) { + if got := ProxyAdminPort(proxyCluster("0.9.1-beta", "", 0)); got != 8087 { + t.Errorf("0.9.1 proxy: %d, want 8087", got) + } + if got := ProxyAdminPort(proxyCluster("0.8.5-beta", "", 9000)); got != 9000 { + t.Errorf("0.8.5 proxy on port 9000: %d, want 9000", got) + } + // A client port of 8087 moves the admin listener off it. + c := proxyCluster("0.9.1-beta", "", 8087) + if got := ProxyAdminPort(c); got != 8088 { + t.Errorf("0.9.1 proxy on client port 8087: admin %d, want 8088", got) + } + pc := proxyContainer(t, c) + if portNumber(pc, httpPortName) == portNumber(pc, proxyAdminPortName) { + t.Errorf("client and admin ports collide: %v", pc.Ports) + } +} + +// The admin port is not on the client Service: /admin can exclude backends. +func TestTheProxyServiceExposesOnlyTheClientPort(t *testing.T) { + svc := BuildProxyService(proxyCluster("0.9.1-beta", "", 0)) + if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Name != httpPortName { + t.Errorf("service ports = %v, want only %s", svc.Spec.Ports, httpPortName) + } +}