From ab4b2253d3660e843c52eeefff8ab2b5179062bf Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Thu, 6 Aug 2026 13:36:09 -0400 Subject: [PATCH] fix(chart): harden Kubernetes persistence Signed-off-by: Luke Tornquist --- deploy/charts/buzz/README.md | 47 +++++++++++++--- deploy/charts/buzz/examples/argocd-app.yaml | 6 +-- .../buzz/examples/flux-helmrelease.yaml | 8 +-- deploy/charts/buzz/templates/NOTES.txt | 5 +- deploy/charts/buzz/templates/_validate.tpl | 15 +++--- deploy/charts/buzz/templates/deployment.yaml | 4 ++ .../buzz/templates/quickstart-minio.yaml | 3 ++ .../charts/buzz/tests/fixtures/ha-values.yaml | 5 +- .../git_persistence_validation_test.yaml | 17 ++++++ deploy/charts/buzz/tests/hpa_test.yaml | 1 + .../buzz/tests/quickstart_bundled_test.yaml | 17 ++++++ deploy/charts/buzz/tests/render_test.yaml | 53 ++++++++++++++++--- deploy/charts/buzz/values.schema.json | 5 +- deploy/charts/buzz/values.yaml | 16 +++--- 14 files changed, 162 insertions(+), 40 deletions(-) create mode 100644 deploy/charts/buzz/tests/git_persistence_validation_test.yaml diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index b2778df28b..475153630c 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -52,6 +52,40 @@ See: The chart fails at `helm install` / `helm template` time with a clear message if any of these are missing or malformed (see `templates/_validate.tpl`). +## Identities and first client + +Buzz uses separate identities for the relay and its owner: + +- `BUZZ_RELAY_PRIVATE_KEY` is the service identity. Store it in the Secret named by `secrets.existingSecret` and back it up. +- `ownerPubkey` is the public key for the human operator. Keep the matching private key in your client or key manager, not in the relay Secret. + +Do not reuse the owner key as the relay key. Separate keys let you rotate infrastructure credentials without changing the operator identity, and keep service actions distinct from human actions. + +Joining and pairing solve different onboarding problems. An invite link joins a new identity to the community. Device pairing copies an existing identity to another device. Enable and expose the [device pairing relay](#device-pairing-relay) before pairing a phone or another desktop. + +## Storage and rollout behavior + +Set a storage class explicitly when the cluster has multiple defaults or when the default is unsuitable for databases. The quickstart profile accepts a class for each chart-managed volume: + +```yaml +persistence: + git: + storageClass: fast-block +postgresql: + persistence: + storageClass: fast-block +redis: + persistence: + storageClass: fast-block +minio: + persistence: + storageClass: fast-block +``` + +`storageClassName` is immutable after a PersistentVolumeClaim is created. Back up state before changing it, pause reconciliation, and recreate the affected claim through your deployment workflow. + +The relay Deployment uses `Recreate` when its Git scratch volume is `ReadWriteOnce` or `ReadWriteOncePod`. This releases a single-writer volume before Kubernetes starts the replacement pod. It uses `RollingUpdate` for `emptyDir` and multi-writer claims. + ## S3 URL addressing Buzz uses one URL style for both media and Git/CAS object-store requests: @@ -167,9 +201,9 @@ an Ingress or HTTPRoute for the pairing Service; route the public hostname to - Redis (`redis.enabled=true`, `externalRedis.url`, or `REDIS_URL` in `existingSecret`) — for `buzz-pubsub` fan-out -It does **not** require ReadWriteMany git storage. Git ref/object state is object-store-backed (each request hydrates an ephemeral repo from S3-compatible storage; writer serialization is the object-store pointer CAS — see `docs/git-on-object-storage.md`), and repo-name uniqueness lives in Postgres. Each replica can use its own `ReadWriteOnce` volume; no shared filesystem is needed. +It does **not** require persistent or ReadWriteMany Git storage. Git ref/object state is object-store-backed (each request hydrates an ephemeral repo from S3-compatible storage; writer serialization is the object-store pointer CAS — see `docs/git-on-object-storage.md`), and repo-name uniqueness lives in Postgres. Set `persistence.git.enabled=false` for multiple replicas so every pod receives its own `emptyDir`. The chart-managed PVC is one shared claim, not one `ReadWriteOnce` claim per replica. -The chart **template-fails** if the Redis invariant is broken at `replicaCount > 1`. No silent degradation. +The chart **template-fails** if Redis is missing or multiple replicas would share one chart-managed `ReadWriteOnce` or `ReadWriteOncePod` claim. No silent degradation. ### Relay autoscaling @@ -193,7 +227,9 @@ adapter (for example Prometheus Adapter) configured to expose the relay's `buzz_ws_connections_active` gauge as a pod metric with the name in `websocketMetricName`. The chart creates the HPA but deliberately does not install or configure a cluster-wide metrics adapter. Scale-down is gradual by -default so long-lived WebSocket connections have time to drain. +default so long-lived WebSocket connections have time to drain. Because the +default `autoscaling.minReplicas` is greater than one, set +`persistence.git.enabled=false` when turning autoscaling on. ## Upgrades @@ -207,9 +243,8 @@ Save these. Losing any of them is data loss. See NOTES.txt printed by `helm inst 1. `BUZZ_RELAY_PRIVATE_KEY` — relay identity. Rotating it = new identity (federation peers will not recognize the relay). 2. PostgreSQL database — the canonical event store. -3. S3 bucket — media blobs (chart default bucket: `buzz-media`). -4. Git PVC — repo on-disk state served by the relay's git endpoint. -5. Owner private key — held by the operator, not by this chart. Restore by re-installing with the same `ownerPubkey`. +3. S3 bucket — media and Git object-store state (chart default bucket: `buzz-media`). +4. Owner private key — held by the operator, not by this chart. Restore by re-installing with the same `ownerPubkey`. ## Honest limitations (v1) diff --git a/deploy/charts/buzz/examples/argocd-app.yaml b/deploy/charts/buzz/examples/argocd-app.yaml index a29a6919b7..98e9ad3fd9 100644 --- a/deploy/charts/buzz/examples/argocd-app.yaml +++ b/deploy/charts/buzz/examples/argocd-app.yaml @@ -52,9 +52,9 @@ spec: persistence: git: - enabled: true - accessMode: ReadWriteOnce # RWO is fine at any replicaCount (object-store-backed git) - storageClass: "" # any RWO class; no shared/RWX filesystem needed + # Git state is object-store-backed. emptyDir gives each relay pod + # independent scratch space and avoids sharing one PVC. + enabled: false size: 50Gi ingress: diff --git a/deploy/charts/buzz/examples/flux-helmrelease.yaml b/deploy/charts/buzz/examples/flux-helmrelease.yaml index 09a6bfeb6a..568196a746 100644 --- a/deploy/charts/buzz/examples/flux-helmrelease.yaml +++ b/deploy/charts/buzz/examples/flux-helmrelease.yaml @@ -25,7 +25,7 @@ spec: chart: spec: chart: buzz - version: "0.1.0" + version: "0.1.7" sourceRef: kind: HelmRepository name: buzz @@ -46,9 +46,9 @@ spec: persistence: git: - enabled: true - accessMode: ReadWriteOnce # RWO is fine at any replicaCount (object-store-backed git) - storageClass: "" # any RWO class; no shared/RWX filesystem needed + # Git state is object-store-backed. emptyDir gives each relay pod + # independent scratch space and avoids sharing one PVC. + enabled: false size: 50Gi ingress: diff --git a/deploy/charts/buzz/templates/NOTES.txt b/deploy/charts/buzz/templates/NOTES.txt index b409f4d942..865a9de8fd 100644 --- a/deploy/charts/buzz/templates/NOTES.txt +++ b/deploy/charts/buzz/templates/NOTES.txt @@ -47,9 +47,8 @@ 1. BUZZ_RELAY_PRIVATE_KEY — relay identity. Rotating it = identity change; federation peers will treat the relay as a new identity. 2. PostgreSQL database{{ if .Values.postgresql.enabled }} ({{ .Release.Name }}-postgresql PVC){{ end }} - 3. S3 bucket "{{ .Values.s3.bucket }}" — media blobs - 4. Git PVC ({{ include "buzz.fullname" . }}-git) — repo on-disk state - 5. Owner private key (held by the operator, NOT the chart) — restore by + 3. S3 bucket "{{ .Values.s3.bucket }}" — media and Git object-store state + 4. Owner private key (held by the operator, NOT the chart) — restore by re-installing with the same ownerPubkey. ────────────────────────────────────────────────────────────────────────────── diff --git a/deploy/charts/buzz/templates/_validate.tpl b/deploy/charts/buzz/templates/_validate.tpl index aa7f7ac13c..8d432efd74 100644 --- a/deploy/charts/buzz/templates/_validate.tpl +++ b/deploy/charts/buzz/templates/_validate.tpl @@ -16,22 +16,25 @@ surface at template time regardless of which manifest helm renders first. {{- if and (not .Values.redis.enabled) (not .Values.externalRedis.url) (not .Values.secrets.existingSecret) -}} {{- fail (printf "minimum replica count %d requires Redis for buzz-pubsub. Enable redis.enabled=true, set externalRedis.url, or provide secrets.existingSecret with key REDIS_URL." $minimumReplicas) -}} {{- end -}} + {{- if and .Values.persistence.git.enabled (or (eq .Values.persistence.git.accessMode "ReadWriteOnce") (eq .Values.persistence.git.accessMode "ReadWriteOncePod")) -}} + {{- fail (printf "minimum replica count %d: multiple replicas cannot share one %s Git scratch claim. Set persistence.git.enabled=false for a per-pod emptyDir, or use a multi-writer claim." $minimumReplicas .Values.persistence.git.accessMode) -}} + {{- end -}} {{- end -}} -{{/* Multiple replicas do NOT require ReadWriteMany git storage. +{{/* Multiple replicas do NOT require persistent or ReadWriteMany git storage. Git ref/object state is object-store-backed: every read and write hydrates an ephemeral bare repo from S3-compatible storage per request, and writer serialization is the object-store pointer CAS (docs/git-on-object-storage.md, Inv_NoFork). No persistent git state lives - on the PVC, so replicas do not need a shared ReadWriteMany volume to agree - on refs. Repo-name uniqueness — the last shared-state need — now lives in - Postgres (git_repo_names), not on local disk. + on the PVC, so the recommended HA configuration gives every replica an + independent emptyDir. Repo-name uniqueness — the last shared-state need — + now lives in Postgres (git_repo_names), not on local disk. The prior hard-fail requiring persistence.git.accessMode=ReadWriteMany was removed here: its stated reason ("git on-disk state must be shared across - replicas") is no longer true. Redis (validated above) remains the real - multi-pod requirement for buzz-pubsub. */}} + replicas") is no longer true. The guard above rejects one chart-managed + single-writer claim shared by every Deployment replica. */}} {{/* Autoscaling bounds must be coherent. */}} {{- if .Values.autoscaling.enabled -}} diff --git a/deploy/charts/buzz/templates/deployment.yaml b/deploy/charts/buzz/templates/deployment.yaml index 5c876f7d24..a2941a3721 100644 --- a/deploy/charts/buzz/templates/deployment.yaml +++ b/deploy/charts/buzz/templates/deployment.yaml @@ -10,10 +10,14 @@ spec: replicas: {{ .Values.replicaCount }} {{- end }} strategy: + {{- if and .Values.persistence.git.enabled (or (eq .Values.persistence.git.accessMode "ReadWriteOnce") (eq .Values.persistence.git.accessMode "ReadWriteOncePod")) }} + type: Recreate + {{- else }} type: RollingUpdate rollingUpdate: maxSurge: 1 maxUnavailable: 0 + {{- end }} selector: matchLabels: {{- include "buzz.relaySelectorLabels" . | nindent 6 }} diff --git a/deploy/charts/buzz/templates/quickstart-minio.yaml b/deploy/charts/buzz/templates/quickstart-minio.yaml index 9685a7ff34..7ca90fe0e8 100644 --- a/deploy/charts/buzz/templates/quickstart-minio.yaml +++ b/deploy/charts/buzz/templates/quickstart-minio.yaml @@ -92,6 +92,9 @@ metadata: helm.sh/resource-policy: keep spec: accessModes: ["ReadWriteOnce"] + {{- if .Values.minio.persistence.storageClass }} + storageClassName: {{ .Values.minio.persistence.storageClass | quote }} + {{- end }} resources: requests: storage: {{ .Values.minio.persistence.size | quote }} diff --git a/deploy/charts/buzz/tests/fixtures/ha-values.yaml b/deploy/charts/buzz/tests/fixtures/ha-values.yaml index 14e28b999b..f043cf7b7e 100644 --- a/deploy/charts/buzz/tests/fixtures/ha-values.yaml +++ b/deploy/charts/buzz/tests/fixtures/ha-values.yaml @@ -1,4 +1,4 @@ -# HA shape: replicas=3 + Redis + RWO git (object-store-backed; no RWM). Render-only check. +# HA shape: replicas=3 + Redis + per-pod Git scratch space. Render-only check. relayUrl: wss://buzz.example.com ownerPubkey: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" replicaCount: 3 @@ -12,8 +12,7 @@ s3: bucket: "buzz-media" persistence: git: - enabled: true - accessMode: ReadWriteOnce + enabled: false size: 50Gi podDisruptionBudget: enabled: true diff --git a/deploy/charts/buzz/tests/git_persistence_validation_test.yaml b/deploy/charts/buzz/tests/git_persistence_validation_test.yaml new file mode 100644 index 0000000000..dff6f40f2d --- /dev/null +++ b/deploy/charts/buzz/tests/git_persistence_validation_test.yaml @@ -0,0 +1,17 @@ +suite: Git scratch persistence validation +templates: + - templates/pvc-git.yaml +tests: + - it: rejects multiple replicas sharing one chart-managed single-writer claim + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: http://minio:9000 + replicaCount: 3 + persistence.git.enabled: true + persistence.git.accessMode: ReadWriteOnce + asserts: + - failedTemplate: + errorPattern: "multiple replicas cannot share one ReadWriteOnce Git scratch claim" diff --git a/deploy/charts/buzz/tests/hpa_test.yaml b/deploy/charts/buzz/tests/hpa_test.yaml index 9cdbc8dcd7..5f30af2413 100644 --- a/deploy/charts/buzz/tests/hpa_test.yaml +++ b/deploy/charts/buzz/tests/hpa_test.yaml @@ -11,6 +11,7 @@ set: s3.endpoint: http://minio:9000 s3.accessKey: a s3.secretKey: s + persistence.git.enabled: false tests: - it: omits the HPA by default template: templates/hpa.yaml diff --git a/deploy/charts/buzz/tests/quickstart_bundled_test.yaml b/deploy/charts/buzz/tests/quickstart_bundled_test.yaml index ecb7877e9f..3927bd96d7 100644 --- a/deploy/charts/buzz/tests/quickstart_bundled_test.yaml +++ b/deploy/charts/buzz/tests/quickstart_bundled_test.yaml @@ -37,6 +37,23 @@ tests: path: metadata.annotations["helm.sh/hook"] template: templates/quickstart-minio-init.yaml + - it: applies the configured storage class to bundled MinIO + release: + name: rel + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + postgresql.enabled: true + redis.enabled: true + minio.enabled: true + minio.persistence.storageClass: fast-block + asserts: + - equal: + path: spec.storageClassName + value: fast-block + template: templates/quickstart-minio.yaml + documentIndex: 2 + - it: relay S3 endpoint resolves to the bundled MinIO Service release: name: rel diff --git a/deploy/charts/buzz/tests/render_test.yaml b/deploy/charts/buzz/tests/render_test.yaml index cf08210781..1322fa661a 100644 --- a/deploy/charts/buzz/tests/render_test.yaml +++ b/deploy/charts/buzz/tests/render_test.yaml @@ -59,6 +59,44 @@ tests: value: "true" template: templates/deployment.yaml + - it: recreates a single-replica deployment that mounts a ReadWriteOnce PVC + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: http://minio:9000 + s3.accessKey: a + s3.secretKey: s + asserts: + - equal: + path: spec.strategy.type + value: Recreate + template: templates/deployment.yaml + - notExists: + path: spec.strategy.rollingUpdate + template: templates/deployment.yaml + + - it: rolls a deployment that mounts a multi-writer PVC + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: http://minio:9000 + s3.accessKey: a + s3.secretKey: s + persistence.git.accessMode: ReadWriteMany + asserts: + - equal: + path: spec.strategy.type + value: RollingUpdate + template: templates/deployment.yaml + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + template: templates/deployment.yaml + - it: renders virtual-hosted S3 addressing for providers that require it set: relayUrl: wss://buzz.example.com @@ -131,7 +169,7 @@ tests: s3.accessKey: a s3.secretKey: s replicaCount: 3 - persistence.git.accessMode: ReadWriteOnce + persistence.git.enabled: false relay.huddleAudioAvailable: true asserts: - contains: @@ -141,7 +179,7 @@ tests: value: "true" template: templates/deployment.yaml - - it: renders HA cleanly with replicaCount=3 + RWO git + Redis (no RWM needed) + - it: renders HA cleanly with replicaCount=3 + per-pod git scratch + Redis set: relayUrl: wss://buzz.example.com ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" @@ -151,15 +189,14 @@ tests: s3.accessKey: a s3.secretKey: s replicaCount: 3 - persistence.git.accessMode: ReadWriteOnce + persistence.git.enabled: false asserts: - equal: path: spec.replicas value: 3 template: templates/deployment.yaml - - equal: - path: spec.accessModes[0] - value: ReadWriteOnce + - hasDocuments: + count: 0 template: templates/pvc-git.yaml - contains: path: spec.template.spec.containers[0].env @@ -203,6 +240,10 @@ tests: - hasDocuments: count: 0 template: templates/pvc-git.yaml + - equal: + path: spec.strategy.type + value: RollingUpdate + template: templates/deployment.yaml - it: preserves image defaults when Pod extensions are empty set: diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index e1e362a531..32b39ddd72 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -7,7 +7,7 @@ "properties": { "quickstart": { "type": "boolean", - "description": "Master toggle for evaluation profile (enables postgresql + redis subcharts and chart-side autogen). Not GitOps-safe." + "description": "Intent marker for the evaluation profile. Enable the bundled Postgres, Redis, and MinIO components separately. Not GitOps-safe." }, "image": { "type": "object", @@ -26,7 +26,7 @@ "replicaCount": { "type": "integer", "minimum": 1, - "description": "Replica count for the relay Deployment. replicaCount > 1 requires Redis (for buzz-pubsub) — enforced by _validate.tpl. Git storage does NOT need ReadWriteMany: git state is object-store-backed and repo names live in Postgres, so ReadWriteOnce is fine per replica." + "description": "Replica count for the relay Deployment. replicaCount > 1 requires Redis (for buzz-pubsub) — enforced by _validate.tpl. Git state is object-store-backed; disable git persistence to give each replica independent emptyDir scratch space." }, "relayUrl": { "type": "string", @@ -224,6 +224,7 @@ "additionalProperties": false, "properties": { "enabled": { "type": "boolean" }, + "storageClass": { "type": "string" }, "size": { "type": "string", "pattern": "^[0-9]+(\\.[0-9]+)?(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)?$" } } } diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 42b09f1b3e..c00bdfd7fd 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -5,7 +5,7 @@ # PRODUCTION (default) — external Postgres/Redis/S3, existingSecret # refs everywhere, no chart-side autogeneration, GitOps-safe (ArgoCD/Flux). # HA-ready: replicaCount >= 2 (requires Redis; git state is object-store- -# backed, so no ReadWriteMany volume is needed — RWO per replica is fine). +# backed, so disable git persistence for a per-pod emptyDir). # # QUICKSTART — bundles in-cluster Postgres + Redis + MinIO and # auto-generates relay secrets via the `lookup` pattern (NOT GitOps-safe — @@ -30,10 +30,11 @@ image: # ── Topology ──────────────────────────────────────────────────────────────── # replicaCount > 1 hard-requires Redis for buzz-pubsub (in-cluster or external). -# It does NOT require ReadWriteMany git storage: git ref/object state is -# object-store-backed (each request hydrates an ephemeral repo from S3; writer -# serialization is the object-store pointer CAS), and repo-name uniqueness lives -# in Postgres. Each replica can use its own ReadWriteOnce volume (or none). +# It does NOT require persistent or ReadWriteMany git storage: git ref/object +# state is object-store-backed (each request hydrates an ephemeral repo from S3; +# writer serialization is the object-store pointer CAS), and repo-name +# uniqueness lives in Postgres. Set persistence.git.enabled=false so each +# replica receives its own emptyDir. replicaCount: 1 # ── Autoscaling ────────────────────────────────────────────────────────────── @@ -82,8 +83,8 @@ ownerPubkey: "" # ── Chart-managed secrets ──────────────────────────────────────────────────── # Production / GitOps path: create a Secret out-of-band with these keys and -# point `secrets.existingSecret` at it. Any key omitted from the existing -# Secret falls back to chart-side autogen (only effective at first install). +# point `secrets.existingSecret` at it. The chart does not generate a second +# Secret or fill missing keys when an existing Secret is selected. # # Expected keys (all optional unless required by relay config): # BUZZ_RELAY_PRIVATE_KEY — 64-char hex; relay identity (rotation = identity change) @@ -365,6 +366,7 @@ minio: mcImage: minio/mc:RELEASE.2025-08-13T08-35-41Z persistence: enabled: true + storageClass: "" size: 10Gi # ── Git server config ────────────────────────────────────────────────────────