diff --git a/.github/workflows/validate-scaffold.yaml b/.github/workflows/validate-scaffold.yaml index 8e2c9c8..8915a8b 100644 --- a/.github/workflows/validate-scaffold.yaml +++ b/.github/workflows/validate-scaffold.yaml @@ -63,6 +63,12 @@ jobs: # aliases, and fails on anything that would bring the retired stacks back. run: ./scripts/validate-observability-option.sh + - name: 🩺 Validate the backup, database and certificate check + # Runs the production platform-health-alert script against recorded API + # responses: every problem shape is reported, healthy and uninstalled + # kinds stay quiet, and a broken read or delivery fails the check. + run: ./scripts/tests/test-platform-health-alert.sh + - name: 🔔 Validate Flux alert coverage # The production Alert is the only notification for a failed HelmRelease, # and it has no namespace wildcard: every namespace holding a release or a diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md index 7b5b0c6..31e1e93 100644 --- a/docs/TEMPLATING.md +++ b/docs/TEMPLATING.md @@ -144,8 +144,9 @@ deny policy from blocking the new workload during an upgrade. The invalid URL fallback keeps local and unconfigured instances inert. Flux notification-controller now owns reconciliation errors in production, including dependency waits. -Some Prometheus alert rules have no replacement: certificate expiry, backup -freshness and autoscaler failures are not alerted today. +In production, a `platform-health-alert` CronJob reports failed or stale backups, +degraded CloudNativePG clusters and certificates close to expiry through the same +webhook. Cluster Autoscaler failures are left to Coroot. [`docs/dr/alerting.md`](dr/alerting.md) lists what is and is not alerted. Open the Coroot UI at `https://observability.`. Community Edition diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md index f309cc8..29ece99 100644 --- a/docs/dr/alerting.md +++ b/docs/dr/alerting.md @@ -4,7 +4,7 @@ and incidents, collected by an eBPF node agent with no application changes. Self-hosted, no SaaS metrics tier. There is no kube-prometheus-stack: no Prometheus operator, Alertmanager, node-exporter or kube-state-metrics, and no -`monitoring` namespace. Alerting has three parts: +`monitoring` namespace. Alerting has four parts: 1. **Coroot incidents go to Slack.** Coroot inspects every workload and node and posts incident and resolution notifications in production. @@ -12,6 +12,10 @@ Prometheus operator, Alertmanager, node-exporter or kube-state-metrics, and no reconciliation errors for every `Kustomization` and every `HelmRelease`. 3. **One off-cluster dead-man check.** The `cluster-heartbeat` CronJob pings an external monitor, so a cluster that is down still gets noticed. +4. **A health check for backups, databases and certificates.** The + `platform-health-alert` CronJob posts what Coroot and Flux do not see: a + failed or stale backup, a degraded CloudNativePG cluster and a certificate + close to expiry. See [Backup, database and certificate check](#backup-database-and-certificate-check). There is one UI, behind SSO: Coroot at `https://observability.${domain}`, reached only through oauth2-proxy. @@ -23,18 +27,19 @@ says how an instance that ran them migrates. ## Components -| Component | Role | Persistence (prod) | -| ------------------- | -------------------------------------------------- | ------------------ | -| Coroot | UI, incidents, service map, alert inspections | PVC, 2 Gi | -| Coroot Prometheus | Coroot's own metric store | PVC, 10 Gi | -| Coroot ClickHouse | Logs, traces and profiles, 3 d retention | PVC, 10 Gi | -| Coroot node agent | eBPF telemetry from every node (DaemonSet) | n/a | -| Audit log forwarder | Ships Talos host audit logs to Coroot (prod only) | n/a | -| Flux notifications | `Provider` and two `Alert` objects (prod only) | n/a | -| cluster-heartbeat | Dead-man ping every five minutes (CronJob) | n/a | - -Local/CI runs the same stack without the audit log forwarder and without the -Flux notification path. Coroot's bundled Prometheus is its own metric store; it +| Component | Role | Persistence (prod) | +| --------------------- | -------------------------------------------------- | ------------------ | +| Coroot | UI, incidents, service map, alert inspections | PVC, 2 Gi | +| Coroot Prometheus | Coroot's own metric store | PVC, 10 Gi | +| Coroot ClickHouse | Logs, traces and profiles, 3 d retention | PVC, 10 Gi | +| Coroot node agent | eBPF telemetry from every node (DaemonSet) | n/a | +| Audit log forwarder | Ships Talos host audit logs to Coroot (prod only) | n/a | +| Flux notifications | `Provider` and two `Alert` objects (prod only) | n/a | +| cluster-heartbeat | Dead-man ping every five minutes (CronJob) | n/a | +| platform-health-alert | Backup, database and certificate check (prod only) | n/a | + +Local/CI runs the same stack without the audit log forwarder, the health check and +the Flux notification path. Coroot's bundled Prometheus is its own metric store; it is not a Prometheus operator and installs no `monitoring.coreos.com` resources. ## Alert routing → Slack @@ -98,26 +103,54 @@ and ~10 min grace and connect it to Slack. The CronJob deliberately accepts only ## What is alerted, and what is not -| Signal | Where it shows | -| ------------------------------------------- | ------------------------------------------- | -| A workload misses its objectives | Coroot incident → Slack | -| Node, pod, memory, disk and restart health | Coroot inspections, in the Coroot UI | -| A `Kustomization` fails to build or apply | Flux `Alert` → Slack | -| A `HelmRelease` fails to install or upgrade | Flux `Alert` → Slack | -| A prerequisite layer is not ready | Flux `Alert` → Slack (dependency wait) | -| The cluster stops answering | External monitor → Slack | - -**Not alerted today.** The Prometheus rules that left with kube-prometheus-stack -had no direct equivalent in Coroot's model, and these are not replaced: - -- certificate expiry (`CertificateExpiringSoon`); -- a failed or stale Velero backup, and a CloudNativePG cluster without a recent - backup or a streaming replica; -- cluster-autoscaler scale-up failures. - -Check backups with the restore drill in [restore-drill.md](./restore-drill.md) -and certificates with `kubectl get certificates -A` until an instance adds its -own check for them. +| Signal | Where it shows | +| -------------------------------------------- | --------------------------------------------- | +| A workload misses its objectives | Coroot incident → Slack | +| Node, pod, memory, disk and restart health | Coroot inspections, in the Coroot UI | +| A `Kustomization` fails to build or apply | Flux `Alert` → Slack | +| A `HelmRelease` fails to install or upgrade | Flux `Alert` → Slack | +| A prerequisite layer is not ready | Flux `Alert` → Slack (dependency wait) | +| The cluster stops answering | External monitor → Slack | +| A Velero backup failed or is stale | Health check → Slack | +| A CloudNativePG cluster is degraded or stale | Health check → Slack | +| A certificate expires within 14 days | Health check → Slack | +| Cluster Autoscaler cannot add a node | Coroot incident for the workload left Pending | + +**Cluster Autoscaler failures are left to Coroot.** KSail deploys and owns the +autoscaler, not this repository's manifests, so a check here would have to +guess its namespace and its event format. What matters to an operator is the +effect: pods that stay Pending, which Coroot reports as the workload missing +its objectives. Read the autoscaler's own state from its +`cluster-autoscaler-status` ConfigMap. + +## Backup, database and certificate check + +The `platform-health-alert` CronJob replaces the Prometheus rules that left +with kube-prometheus-stack. It runs every six hours in production, in the +`observability` namespace, and posts one Slack message listing every problem +it finds: + +- a Velero schedule whose latest backup failed, or that has no completed + backup in the last 26 hours; +- a CloudNativePG cluster with no completed `Backup` in the last 26 hours, + fewer ready instances than it asks for, or a configured WAL archive that is + not working, each for longer than 15 minutes; +- a cert-manager `Certificate` that expires within 14 days, which means + cert-manager has not renewed it. + +It posts to the same webhook as Coroot and Flux, so there is nothing new to +provision. Its network path is the `observability` namespace's `allow-coroot` +policy, which already lets every pod but the heartbeat reach the API server, DNS +and, in production, `hooks.slack.com:443`; the validator fails if that changes. A problem is reported again on every run until it is fixed. Its +identity can only list Velero schedules and backups, CloudNativePG clusters +and backups, and certificates. A kind whose API is not installed is skipped; +any other read or delivery failure fails the Job, so check +`kubectl -n observability get jobs` after a quiet week if you doubt it. +`scripts/tests/test-platform-health-alert.sh` runs the check against recorded +API responses in CI. + +Every CloudNativePG cluster is expected to have a recent backup. A database you +add without a `ScheduledBackup` is reported until it has one. ## Off-cluster backup diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 7194e05..443f330 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -41,6 +41,8 @@ resources: - kubelet-serving-cert-approver/ - longhorn/ - origin-ca-issuer/ + # Backup, database and certificate checks that post to Slack (#423). + - platform-health-alert/ patches: - path: openbao/patches/store-data-on-hcloud.yaml - path: coroot/patches/allow-slack-webhook-egress.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role-binding.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role-binding.yaml new file mode 100644 index 0000000..a59e443 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role-binding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: platform-health-alert +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: platform-health-alert +subjects: + - kind: ServiceAccount + name: platform-health-alert + namespace: observability diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role.yaml new file mode 100644 index 0000000..0c5c7d6 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cluster-role.yaml @@ -0,0 +1,16 @@ +--- +# Read-only: the check only lists the objects whose status it judges. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: platform-health-alert +rules: + - apiGroups: ["velero.io"] + resources: ["schedules", "backups"] + verbs: ["list"] + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters", "backups"] + verbs: ["list"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["list"] diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cron-job.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cron-job.yaml new file mode 100644 index 0000000..c9d9368 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/cron-job.yaml @@ -0,0 +1,228 @@ +--- +# Posts to Slack when a backup, a database or a certificate needs attention: +# +# - a Velero schedule whose latest backup failed, or with no completed backup +# within BACKUP_MAX_AGE_SECONDS; +# - a CloudNativePG cluster with no completed backup within +# BACKUP_MAX_AGE_SECONDS, fewer ready instances than requested, or a +# configured WAL archive that is not working, each past GRACE_SECONDS; +# - a cert-manager Certificate that expires within CERT_WARN_SECONDS. +# +# A kind whose API is not installed (HTTP 404) is skipped. Every other API or +# delivery failure fails the Job, because nothing else notices a check that +# silently sends nothing. A problem that persists is reported again on every +# run until it is fixed; that is deliberate. +# +# The script uses its own shell variables, so Flux substitution is disabled for +# this object. The webhook comes from the Secret next to it, mounted as a file +# and passed to curl as stdin configuration, so it is in neither the PodSpec +# nor an argument vector. scripts/tests/test-platform-health-alert.sh runs this +# script against recorded API responses. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: platform-health-alert + namespace: observability + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled +spec: + schedule: "41 */6 * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + app: platform-health-alert + spec: + restartPolicy: Never + serviceAccountName: platform-health-alert + # The check lists objects through the Kubernetes API, so it needs its + # token; its ClusterRole only lists the kinds it judges. + automountServiceAccountToken: true + securityContext: + runAsNonRoot: true + runAsUser: 65532 + # Makes the mounted webhook file group-readable by this user. + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: check + image: docker.io/badouralix/curl-jq:latest@sha256:1e7c0284e24572ace7170df9fc91f15fd3b79ebf056d4dde17244d5d74bbfabc + imagePullPolicy: IfNotPresent + env: + - name: BACKUP_MAX_AGE_SECONDS + value: "93600" # 26 hours: one daily backup plus slack. + - name: CERT_WARN_SECONDS + value: "1209600" # 14 days. + - name: GRACE_SECONDS + value: "900" + command: + - /bin/sh + - -c + - | + set -eu + API=${API:-https://kubernetes.default.svc} + SA=${SA_DIR:-/var/run/secrets/kubernetes.io/serviceaccount} + WORK=${WORK_DIR:-/tmp} + NOW=${NOW:-$(date -u +%s)} + WEBHOOK_URL=$(cat "${WEBHOOK_FILE:-/etc/platform-health-alert/url}") + + # The URL becomes a curl config line, where a newline would + # start another directive; and the token in its path must not + # travel over plain http. + if [ "$(printf '%s' "$WEBHOOK_URL" | tr -d '[:cntrl:]')" != "$WEBHOOK_URL" ]; then + echo "ERROR: webhook URL contains control characters" >&2 + exit 1 + fi + case "$WEBHOOK_URL" in + https://*) ;; + *) echo "ERROR: webhook URL must use https://" >&2; exit 1 ;; + esac + + # list PATH NAME: writes the list to $WORK/NAME.json. Returns 1 + # when the kind is not installed; exits on any other failure, so + # a broken read never looks like a healthy cluster. + list() { + code=$(curl -sS -o "$WORK/$2.json" -w '%{http_code}' \ + --cacert "$SA/ca.crt" \ + -H "Authorization: Bearer $(cat "$SA/token")" \ + "$API$1") || code="curl-failed" + case "$code" in + 200) ;; + 404) echo "$2: API not installed; skipped."; return 1 ;; + *) echo "ERROR: listing $1 returned $code" >&2; exit 1 ;; + esac + if ! jq -e '(.items | type) == "array"' "$WORK/$2.json" >/dev/null; then + echo "ERROR: unexpected response shape listing $1" >&2 + exit 1 + fi + echo "$2: $(jq '.items | length' "$WORK/$2.json") listed." + } + + DEFS='def ts: if . == null then null else (sub("\\.[0-9]+Z$"; "Z") | fromdate) end; + def ref: "`\(.metadata.namespace)/\(.metadata.name)`";' + : >"$WORK/findings" + + if list /apis/velero.io/v1/schedules velero-schedules; then + list /apis/velero.io/v1/backups velero-backups + jq -r --argjson now "$NOW" --argjson max "$BACKUP_MAX_AGE_SECONDS" \ + --slurpfile backups "$WORK/velero-backups.json" "$DEFS"' + .items[] | . as $s + | [$backups[0].items[] + | select(.metadata.namespace == $s.metadata.namespace + and .metadata.labels["velero.io/schedule-name"] == $s.metadata.name)] as $mine + | ($mine | sort_by(.metadata.creationTimestamp) | last) as $latest + | ([$mine[] | select(.status.phase == "Completed") | .status.completionTimestamp | ts] | max) as $done + | ($done // ($s.metadata.creationTimestamp | ts)) as $since + | (if $latest != null and (($latest.status.phase // "") | IN("Failed", "PartiallyFailed", "FailedValidation")) + then "• Velero schedule \($s | ref): latest backup `\($latest.metadata.name)` is \($latest.status.phase)" + else empty end), + (if $since <= $now - $max + then "• Velero schedule \($s | ref): no completed backup since \($since | todate)" + else empty end) + ' "$WORK/velero-schedules.json" >>"$WORK/findings" + fi + + if list /apis/postgresql.cnpg.io/v1/clusters cnpg-clusters; then + list /apis/postgresql.cnpg.io/v1/backups cnpg-backups + jq -r --argjson now "$NOW" --argjson max "$BACKUP_MAX_AGE_SECONDS" \ + --argjson grace "$GRACE_SECONDS" \ + --slurpfile backups "$WORK/cnpg-backups.json" "$DEFS"' + .items[] | . as $c + | ($c.metadata.creationTimestamp | ts) as $created + | (($c.status.conditions // []) | map(select(.type == "Ready")) | first) as $ready + | (($c.status.conditions // []) | map(select(.type == "ContinuousArchiving")) | first) as $archive + | ([($c.spec.plugins // [])[] | select(.enabled != false and .isWALArchiver == true)] | length > 0 + or $c.spec.backup.barmanObjectStore != null) as $archiveConfigured + | ([$backups[0].items[] + | select(.metadata.namespace == $c.metadata.namespace + and .spec.cluster.name == $c.metadata.name + and .status.phase == "completed") + | .status.stoppedAt | ts] | max) as $done + | ($done // $created) as $since + | (($ready.lastTransitionTime | ts) // $created) as $readySince + | (($archive.lastTransitionTime | ts) // $created) as $archiveSince + | (if $since <= $now - $max + then "• CloudNativePG cluster \($c | ref): no completed backup since \($since | todate)" + else empty end), + (if (($c.status.readyInstances // 0) < ($c.spec.instances // 1) + or ($ready.status // "Unknown") != "True") + and $readySince <= $now - $grace + then "• CloudNativePG cluster \($c | ref): \($c.status.readyInstances // 0)/\($c.spec.instances // 1) instances ready since \($readySince | todate)" + else empty end), + (if $archiveConfigured and ($archive.status // "Unknown") != "True" + and $archiveSince <= $now - $grace + then "• CloudNativePG cluster \($c | ref): WAL archiving is \($archive.status // "Unknown") since \($archiveSince | todate)" + else empty end) + ' "$WORK/cnpg-clusters.json" >>"$WORK/findings" + fi + + if list /apis/cert-manager.io/v1/certificates certificates; then + jq -r --argjson now "$NOW" --argjson warn "$CERT_WARN_SECONDS" "$DEFS"' + .items[] + | (.status.notAfter | ts) as $notAfter + | select($notAfter != null and $notAfter - $now < $warn) + | "• Certificate \(ref) expires \($notAfter | todate); cert-manager has not renewed it" + ' "$WORK/certificates.json" >>"$WORK/findings" + fi + + if [ ! -s "$WORK/findings" ]; then + echo "No backup, database or certificate problem found." + exit 0 + fi + cat "$WORK/findings" + + jq -n --rawfile findings "$WORK/findings" '{text: ("*Platform health check*\n\n" + $findings + "\n_See docs/dr/alerting.md for what each line means._")}' >"$WORK/payload.json" + + # Only the reserved placeholder may skip delivery. + case "$WEBHOOK_URL" in + https://example.invalid/*) + echo "No Slack webhook configured; alert not delivered." + exit 0 + ;; + esac + printf 'url = %s\n' "$WEBHOOK_URL" | + curl -sf --max-time 15 \ + --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \ + -H 'Content-Type: application/json' \ + --data @"$WORK/payload.json" \ + --config - + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + - name: webhook + mountPath: /etc/platform-health-alert + readOnly: true + resources: + requests: + cpu: 5m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + volumes: + - name: tmp + emptyDir: {} + # 288 is 0440 written in decimal, so YAML 1.1 and 1.2 readers agree: + # readable by root and the pod's fsGroup only. + - name: webhook + secret: + secretName: platform-health-alert-webhook + defaultMode: 288 diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/kustomization.yaml new file mode 100644 index 0000000..6f24d28 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/kustomization.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Production-only health check for the signals that left with +# kube-prometheus-stack (#423): failed or stale Velero backups, CloudNativePG +# clusters without a recent backup, a streaming replica or WAL archiving, and +# certificates close to expiry. It lives in the observability namespace to +# reuse the allow-coroot network policy, which already permits the +# kube-apiserver, DNS and, in production, hooks.slack.com:443. +resources: + - service-account.yaml + - cluster-role.yaml + - cluster-role-binding.yaml + - secret.yaml + - cron-job.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/secret.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/secret.yaml new file mode 100644 index 0000000..74758e3 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/secret.yaml @@ -0,0 +1,13 @@ +--- +# The same Slack webhook Coroot and the Flux notifications post to, so these +# alerts land in the same channel with no new secret to provision. The +# placeholder keeps static validation usable without the decrypted input, and +# the check refuses to deliver to it. +apiVersion: v1 +kind: Secret +metadata: + name: platform-health-alert-webhook + namespace: observability +type: Opaque +stringData: + url: ${alertmanager_webhook_url:=https://example.invalid/no-slack-configured} diff --git a/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/service-account.yaml b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/service-account.yaml new file mode 100644 index 0000000..867c78d --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/platform-health-alert/service-account.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: platform-health-alert + namespace: observability diff --git a/scripts/tests/test-platform-health-alert.sh b/scripts/tests/test-platform-health-alert.sh new file mode 100755 index 0000000..5494131 --- /dev/null +++ b/scripts/tests/test-platform-health-alert.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Runs the production platform-health-alert check, exactly as rendered, against +# recorded Kubernetes API responses. curl is stubbed: API reads are served from +# fixtures and the Slack delivery is captured instead of sent. +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source_root="$(cd "${script_dir}/../.." && pwd)" +test_dir="$(mktemp -d)" +trap 'rm -rf "${test_dir}"' EXIT + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +kubectl kustomize "${source_root}/k8s/providers/hetzner/infrastructure/controllers" >"${test_dir}/controllers.yaml" +yq 'select(.kind == "CronJob" and .metadata.name == "platform-health-alert") | .spec.jobTemplate.spec.template.spec.containers[0].command[2]' \ + "${test_dir}/controllers.yaml" >"${test_dir}/check.sh" +[[ -s "${test_dir}/check.sh" ]] || fail "the production render has no platform-health-alert script" +container_env() { + yq "select(.kind == \"CronJob\" and .metadata.name == \"platform-health-alert\") | .spec.jobTemplate.spec.template.spec.containers[0].env[] | select(.name == \"$1\") | .value" \ + "${test_dir}/controllers.yaml" +} +BACKUP_MAX_AGE_SECONDS="$(container_env BACKUP_MAX_AGE_SECONDS)" +CERT_WARN_SECONDS="$(container_env CERT_WARN_SECONDS)" +GRACE_SECONDS="$(container_env GRACE_SECONDS)" +[[ "${BACKUP_MAX_AGE_SECONDS}" == 93600 && "${CERT_WARN_SECONDS}" == 1209600 && "${GRACE_SECONDS}" == 900 ]] || + fail "unexpected thresholds: ${BACKUP_MAX_AGE_SECONDS} ${CERT_WARN_SECONDS} ${GRACE_SECONDS}" +export BACKUP_MAX_AGE_SECONDS CERT_WARN_SECONDS GRACE_SECONDS + +mkdir -p "${test_dir}/bin" "${test_dir}/sa" +printf 'token' >"${test_dir}/sa/token" +printf 'ca' >"${test_dir}/sa/ca.crt" +cat >"${test_dir}/bin/curl" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +out="" data="" config=0 url="" +while [[ $# -gt 0 ]]; do + case "$1" in + -o) out="$2"; shift ;; + --data) data="${2#@}"; shift ;; + --config) config=1; shift ;; + -w|--cacert|-H|--max-time|--retry|--retry-delay) shift ;; + http*) url="$1" ;; + esac + shift +done +if [[ "${config}" == 1 ]]; then + cat >"${FIXTURES}/posted.config" + cp "${data}" "${FIXTURES}/posted.json" + [[ ! -e "${FIXTURES}/post-fails" ]] || exit 22 + exit 0 +fi +path="${url#https://api.test}" +fixture="${FIXTURES}/$(printf '%s' "${path}" | tr '/' '_').json" +if [[ -e "${FIXTURES}/server-error" ]]; then printf '500'; exit 0; fi +if [[ -e "${fixture}" ]]; then cp "${fixture}" "${out}"; printf '200'; else printf '404'; fi +STUB +chmod +x "${test_dir}/bin/curl" + +now=1794355200 # 2026-11-11T00:00:00Z +iso() { jq -n -r --argjson t "$1" '$t | todate'; } +hours_ago() { iso $((now - $1 * 3600)); } + +new_case() { + FIXTURES="${test_dir}/case" + rm -rf "${FIXTURES}" "${test_dir}/work" + mkdir -p "${FIXTURES}" "${test_dir}/work" + printf 'https://hooks.slack.test/services/x' >"${test_dir}/webhook" + export FIXTURES +} +fixture() { printf '%s' "$2" >"${FIXTURES}/$(printf '%s' "$1" | tr '/' '_').json"; } + +run_check() { + PATH="${test_dir}/bin:${PATH}" API=https://api.test SA_DIR="${test_dir}/sa" WORK_DIR="${test_dir}/work" \ + NOW="${now}" WEBHOOK_FILE="${test_dir}/webhook" sh "${test_dir}/check.sh" >"${test_dir}/out" 2>&1 +} + +healthy_fixtures() { + fixture /apis/velero.io/v1/schedules "{\"items\":[{\"metadata\":{\"namespace\":\"velero\",\"name\":\"daily-full\",\"creationTimestamp\":\"$(hours_ago 500)\"}}]}" + fixture /apis/velero.io/v1/backups "{\"items\":[ + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"daily-full-1\",\"creationTimestamp\":\"$(hours_ago 30)\",\"labels\":{\"velero.io/schedule-name\":\"daily-full\"}},\"status\":{\"phase\":\"Completed\",\"completionTimestamp\":\"$(hours_ago 29)\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"daily-full-2\",\"creationTimestamp\":\"$(hours_ago 6)\",\"labels\":{\"velero.io/schedule-name\":\"daily-full\"}},\"status\":{\"phase\":\"Completed\",\"completionTimestamp\":\"$(hours_ago 5)\"}}]}" + fixture /apis/postgresql.cnpg.io/v1/clusters "{\"items\":[{\"metadata\":{\"namespace\":\"app\",\"name\":\"db\",\"creationTimestamp\":\"$(hours_ago 500)\"}, + \"spec\":{\"instances\":3,\"backup\":{\"barmanObjectStore\":{}}}, + \"status\":{\"readyInstances\":3,\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\",\"lastTransitionTime\":\"$(hours_ago 100)\"},{\"type\":\"ContinuousArchiving\",\"status\":\"True\",\"lastTransitionTime\":\"$(hours_ago 100)\"}]}}]}" + fixture /apis/postgresql.cnpg.io/v1/backups "{\"items\":[{\"metadata\":{\"namespace\":\"app\",\"name\":\"db-1\"},\"spec\":{\"cluster\":{\"name\":\"db\"}},\"status\":{\"phase\":\"completed\",\"stoppedAt\":\"$(hours_ago 3)\"}}]}" + fixture /apis/cert-manager.io/v1/certificates "{\"items\":[{\"metadata\":{\"namespace\":\"gateway\",\"name\":\"wildcard\"},\"status\":{\"notAfter\":\"$(hours_ago -1440)\"}}]}" +} + +# 1. A healthy cluster posts nothing. +new_case +healthy_fixtures +run_check || fail "healthy run failed: $(cat "${test_dir}/out")" +grep -q 'No backup, database or certificate problem found.' "${test_dir}/out" || fail "healthy run did not report a clean check" +[[ ! -e "${FIXTURES}/posted.json" ]] || fail "healthy run posted to Slack" + +# 2. Every problem shape is reported in one Slack post, and nothing within its grace. +new_case +fixture /apis/velero.io/v1/schedules "{\"items\":[ + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"failing\",\"creationTimestamp\":\"$(hours_ago 500)\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"stale\",\"creationTimestamp\":\"$(hours_ago 500)\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"never\",\"creationTimestamp\":\"$(hours_ago 27)\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"new\",\"creationTimestamp\":\"$(hours_ago 2)\"}}]}" +fixture /apis/velero.io/v1/backups "{\"items\":[ + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"failing-1\",\"creationTimestamp\":\"$(hours_ago 10)\",\"labels\":{\"velero.io/schedule-name\":\"failing\"}},\"status\":{\"phase\":\"Completed\",\"completionTimestamp\":\"$(hours_ago 9)\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"failing-2\",\"creationTimestamp\":\"$(hours_ago 1)\",\"labels\":{\"velero.io/schedule-name\":\"failing\"}},\"status\":{\"phase\":\"PartiallyFailed\"}}, + {\"metadata\":{\"namespace\":\"velero\",\"name\":\"stale-1\",\"creationTimestamp\":\"$(hours_ago 40)\",\"labels\":{\"velero.io/schedule-name\":\"stale\"}},\"status\":{\"phase\":\"Completed\",\"completionTimestamp\":\"2026-11-09T09:00:00.123Z\"}}]}" +fixture /apis/postgresql.cnpg.io/v1/clusters "{\"items\":[ + {\"metadata\":{\"namespace\":\"app\",\"name\":\"unbacked\",\"creationTimestamp\":\"$(hours_ago 500)\"},\"spec\":{\"instances\":1},\"status\":{\"readyInstances\":1,\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\",\"lastTransitionTime\":\"$(hours_ago 100)\"}]}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"short\",\"creationTimestamp\":\"$(hours_ago 500)\"},\"spec\":{\"instances\":3},\"status\":{\"readyInstances\":2,\"conditions\":[{\"type\":\"Ready\",\"status\":\"False\",\"lastTransitionTime\":\"$(hours_ago 1)\"}]}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"settling\",\"creationTimestamp\":\"$(hours_ago 500)\"},\"spec\":{\"instances\":3},\"status\":{\"readyInstances\":2,\"conditions\":[{\"type\":\"Ready\",\"status\":\"False\",\"lastTransitionTime\":\"$(iso $((now - 300)))\"}]}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"archive\",\"creationTimestamp\":\"$(hours_ago 500)\"},\"spec\":{\"instances\":1,\"plugins\":[{\"name\":\"barman-cloud.cloudnative-pg.io\",\"isWALArchiver\":true}]},\"status\":{\"readyInstances\":1,\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\",\"lastTransitionTime\":\"$(hours_ago 100)\"},{\"type\":\"ContinuousArchiving\",\"status\":\"False\",\"lastTransitionTime\":\"$(hours_ago 2)\"}]}}]}" +fixture /apis/postgresql.cnpg.io/v1/backups "{\"items\":[ + {\"metadata\":{\"namespace\":\"app\",\"name\":\"short-1\"},\"spec\":{\"cluster\":{\"name\":\"short\"}},\"status\":{\"phase\":\"completed\",\"stoppedAt\":\"$(hours_ago 3)\"}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"settling-1\"},\"spec\":{\"cluster\":{\"name\":\"settling\"}},\"status\":{\"phase\":\"completed\",\"stoppedAt\":\"$(hours_ago 3)\"}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"archive-1\"},\"spec\":{\"cluster\":{\"name\":\"archive\"}},\"status\":{\"phase\":\"completed\",\"stoppedAt\":\"$(hours_ago 3)\"}}, + {\"metadata\":{\"namespace\":\"app\",\"name\":\"unbacked-failed\"},\"spec\":{\"cluster\":{\"name\":\"unbacked\"}},\"status\":{\"phase\":\"failed\",\"stoppedAt\":\"$(hours_ago 1)\"}}, + {\"metadata\":{\"namespace\":\"other\",\"name\":\"unbacked-elsewhere\"},\"spec\":{\"cluster\":{\"name\":\"unbacked\"}},\"status\":{\"phase\":\"completed\",\"stoppedAt\":\"$(hours_ago 1)\"}}]}" +fixture /apis/cert-manager.io/v1/certificates "{\"items\":[ + {\"metadata\":{\"namespace\":\"gateway\",\"name\":\"expiring\"},\"status\":{\"notAfter\":\"$(hours_ago -240)\"}}, + {\"metadata\":{\"namespace\":\"gateway\",\"name\":\"fine\"},\"status\":{\"notAfter\":\"$(hours_ago -400)\"}}, + {\"metadata\":{\"namespace\":\"gateway\",\"name\":\"pending\"},\"status\":{}}]}" +run_check || fail "problem run failed: $(cat "${test_dir}/out")" +expected="$(cat <"${test_dir}/webhook" +rm -f "${FIXTURES}/posted.json" +run_check || fail "placeholder run failed" +grep -q 'alert not delivered' "${test_dir}/out" || fail "placeholder run did not say it skipped delivery" +[[ ! -e "${FIXTURES}/posted.json" ]] || fail "the placeholder webhook was posted to" + +# 4. Kinds that are not installed are skipped. +new_case +run_check || fail "a cluster without these APIs failed: $(cat "${test_dir}/out")" +[[ "$(grep -c 'API not installed; skipped.' "${test_dir}/out")" == 3 ]] || fail "expected three skipped kinds" + +# 5. Fail closed on a broken read, a malformed list, a failed delivery or an unsafe webhook. +new_case +healthy_fixtures +touch "${FIXTURES}/server-error" +run_check && fail "an API error was reported as healthy" +new_case +healthy_fixtures +fixture /apis/cert-manager.io/v1/certificates '{"items":{}}' +run_check && fail "a malformed list was reported as healthy" +new_case +healthy_fixtures +fixture /apis/cert-manager.io/v1/certificates "{\"items\":[{\"metadata\":{\"namespace\":\"a\",\"name\":\"b\"},\"status\":{\"notAfter\":\"$(hours_ago -1)\"}}]}" +touch "${FIXTURES}/post-fails" +run_check && fail "a failed Slack delivery succeeded" +new_case +healthy_fixtures +printf 'http://hooks.slack.test/services/x' >"${test_dir}/webhook" +run_check && fail "a plain-http webhook was accepted" +printf 'https://hooks.slack.test/x\noutput = /tmp/x' >"${test_dir}/webhook" +run_check && fail "a webhook with a control character was accepted" + +echo "platform-health-alert check passed." diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh index 1141a1d..29c3ca7 100755 --- a/scripts/validate-observability-option.sh +++ b/scripts/validate-observability-option.sh @@ -530,6 +530,59 @@ assert_flux_notification_contract() { fi } +# Validates the production backup, database and certificate check (#423): one +# read-only identity that can only list the kinds it judges, the shared +# webhook input, and a CronJob wired to both. +assert_platform_health_alert_contract() { + local rendered_path="$1" + local contract + + contract="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s ' + def one($kind; $ns; $name): [.[] | select(type == "object" and .kind == $kind and (.metadata.namespace // "") == $ns and .metadata.name == $name)]; + (one("ServiceAccount"; "observability"; "platform-health-alert") | length == 1) and + (one("ClusterRole"; ""; "platform-health-alert") | length == 1 and .[0].rules == [ + {"apiGroups":["velero.io"],"resources":["schedules","backups"],"verbs":["list"]}, + {"apiGroups":["postgresql.cnpg.io"],"resources":["clusters","backups"],"verbs":["list"]}, + {"apiGroups":["cert-manager.io"],"resources":["certificates"],"verbs":["list"]}]) and + (one("ClusterRoleBinding"; ""; "platform-health-alert") | length == 1 and + .[0].roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"platform-health-alert"} and + .[0].subjects == [{"kind":"ServiceAccount","name":"platform-health-alert","namespace":"observability"}]) and + (one("Secret"; "observability"; "platform-health-alert-webhook") | length == 1 and + .[0].stringData == {"url":"${alertmanager_webhook_url:=https://example.invalid/no-slack-configured}"}) and + (one("CronJob"; "observability"; "platform-health-alert") | length == 1 and + .[0].metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled" and + (.[0].spec.jobTemplate.spec.template.spec as $pod | + $pod.serviceAccountName == "platform-health-alert" and + $pod.securityContext.runAsNonRoot == true and + ($pod.containers | length == 1) and + $pod.containers[0].securityContext.readOnlyRootFilesystem == true and + $pod.containers[0].securityContext.allowPrivilegeEscalation == false and + ($pod.containers[0].image | test("@sha256:[0-9a-f]{64}$")) and + ($pod.containers[0].env // [] | map(.valueFrom // empty) | length == 0) and + ([$pod.volumes[] | select(.secret != null) | .secret] == [{"secretName":"platform-health-alert-webhook","defaultMode":288}]) and + # The pod must stay outside the heartbeat carve-out, so the namespace-wide + # allow-coroot policy below selects it. + ((.[0].spec.jobTemplate.spec.template.metadata.labels // {}) | has("platform-heartbeat") | not))) and + # allow-coroot is the only policy that lets this pod out of the namespace default-deny: + # it must select every pod but the heartbeat and allow the API server, Slack on 443 and DNS. + (one("CiliumNetworkPolicy"; "observability"; "allow-coroot") | length == 1 and + .[0].spec.endpointSelector == {"matchExpressions":[{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]} and + (.[0].spec.egress | any(.toEntities == ["kube-apiserver"] and (has("toPorts") | not))) and + (.[0].spec.egress | any((.toFQDNs // [] | any(.matchName == "hooks.slack.com")) and + .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) and + (.[0].spec.egress | any(.toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and + (.toPorts[0].ports | map(.protocol) | sort) == ["TCP","UDP"] and + .toPorts[0].rules.dns == [{"matchPattern":"*"}]))) + ' + )" + if [[ "${contract}" != "true" ]]; then + echo "production controllers must render the read-only platform-health-alert check wired to the existing Slack webhook" >&2 + return 1 + fi +} + # Validates that only the observability namespace's generated policies defer # the dedicated heartbeat label to its narrower workload policy. assert_heartbeat_policy_exclusion() { @@ -905,6 +958,12 @@ assert_resource_count "${docker_controllers}" Alert flux-system platform-reconci assert_resource_count "${docker_controllers}" Alert flux-system platform-dependency-wait 0 assert_resource_count "${docker_controllers}" Secret flux-system slack-webhook 0 assert_flux_notification_contract "${hetzner_controllers}" +assert_platform_health_alert_contract "${hetzner_controllers}" +for health_alert_kind in ServiceAccount CronJob; do + assert_resource_count "${docker_controllers}" "${health_alert_kind}" observability platform-health-alert 0 +done +assert_resource_count "${docker_controllers}" Secret observability platform-health-alert-webhook 0 +assert_resource_count "${docker_controllers}" ClusterRole "" platform-health-alert 0 assert_heartbeat_staged_after_policy_exclusion "${docker_controllers}" "${docker_apps}" assert_heartbeat_staged_after_policy_exclusion "${hetzner_controllers}" "${hetzner_apps}" @@ -1144,7 +1203,8 @@ for documented_boundary in \ "narrow info-severity path for dependency waits" \ "scripts/validate-alert-coverage.sh" \ "adds that namespace to the alert" \ - "are not alerted today" \ + "a \`platform-health-alert\` CronJob reports failed or stale backups" \ + "Cluster Autoscaler failures are left to Coroot." \ "https://observability." \ "Dex-backed oauth2-proxy" \ "no direct Gateway route to the Coroot service" \ @@ -1179,8 +1239,10 @@ for documented_heartbeat_boundary in \ "Why HelmRelease is watched directly" \ "scripts/validate-alert-coverage.sh" \ "Local / Docker renders no Slack" \ - "Not alerted today" \ - "certificate expiry"; do + "Backup, database and certificate check" \ + "Cluster Autoscaler failures are left to Coroot." \ + "scripts/tests/test-platform-health-alert.sh" \ + "A certificate expires within 14 days"; do if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then echo "alerting guide does not retain alerting boundary: ${documented_heartbeat_boundary}" >&2 exit 1