feat(gke-test): test keep on the shared faberline GKE cluster

.github/workflows/gke-test.yml

@@ -0,0 +1,40 @@
+# Deploy keep to the shared faberline GKE test cluster and run the test Job
+# in k8s/gke-test-job/ against it. faberline/aw owns the cluster, the pools
+# (pool-keep-server for the server, pool-keep-client for the test Job), and the
+# reusable workflow; keep owns k8s/overlays/gke-test/ and k8s/gke-test-job/.
+# Rules: aw's docs/standards/test-cluster.md.
+#
+# Every pool is at 0 nodes between runs, so the first rollout waits a few
+# minutes for the autoscaler to add a node. The run always deletes the
+# namespace, even when the test fails.
+name: gke-test
+
+on:
+  pull_request:
+    paths:
+      - "k8s/**"
+      - ".github/workflows/gke-test.yml"
+      - ".github/workflows/keep-test-image.yml"
+      - "Dockerfile"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image:
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/keep-test-image.yml
+
+  gke-test:
+    needs: image
+    permissions:
+      id-token: write
+      contents: read
+      packages: read
+    uses: faberline/aw/.github/workflows/gke-test.yml@main
+    with:
+      image: ${{ needs.image.outputs.image }}
+      test-job: k8s/gke-test-job

k8s/gke-test-job/job.yaml

@@ -0,0 +1,69 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: keep-test
+spec:
+  # A retry would hide a flaky failure. The run fails on the first one.
+  backoffLimit: 0
+  activeDeadlineSeconds: 1200
+  template:
+    spec:
+      serviceAccountName: keep-test
+      restartPolicy: Never
+      nodeSelector:
+        faberline.dev/pool: keep-client
+      tolerations:
+        - key: faberline.dev/pool
+          operator: Equal
+          value: keep-client
+          effect: NoSchedule
+      imagePullSecrets:
+        - name: ghcr-pull
+      # One pod per node: no other pod of the namespace shares this node.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: kubernetes.io/hostname
+              labelSelector: {}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: verify
+          # kubectl, curl, jq, and bash. Match the minor version of the
+          # cluster (1.36).
+          image: alpine/k8s:1.36.4
+          command: ["bash", "/scripts/verify.sh"]
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: KEEP_URL
+              value: http://keep:7117
+            # kubectl writes its cache under HOME.
+            - name: HOME
+              value: /tmp
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: scripts
+          configMap:
+            name: keep-test-script

k8s/gke-test-job/kustomization.yaml

@@ -0,0 +1,18 @@
+# Test Job for the shared GKE test cluster. faberline/aw's
+# .github/workflows/gke-test.yml applies it after the keep StatefulSet is
+# rolled out, and the run passes only when the Job completes. Rules: aw's
+# docs/standards/test-cluster.md.
+#
+# The Job runs on pool-keep-client, so the test client never takes CPU from
+# the keep server it tests. It reaches keep through the Service DNS name, not
+# through a port-forward. The run sets the namespace.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: keep
+resources:
+  - rbac.yaml
+  - job.yaml
+configMapGenerator:
+  - name: keep-test-script
+    files:
+      - verify.sh

k8s/gke-test-job/rbac.yaml

@@ -0,0 +1,26 @@
+# Step 3 of verify.sh deletes keep-0 and waits for it to come back.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: keep-test
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: keep-test
+rules:
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list, watch, delete]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: keep-test
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: keep-test
+subjects:
+  - kind: ServiceAccount
+    name: keep-test

k8s/gke-test-job/verify.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Runs inside the keep-test Job on pool-keep-client. The Job's logs are the
+# evidence; the shared run copies them into its artifact.
+#
+# Three steps:
+#   1. /readyz answers 200 through the keep Service.
+#   2. PUT /kv/{key} then GET /kv/{key} returns the same value.
+#   3. keep-0 is deleted and comes back Ready; the same GET still returns the
+#      value. This proves the PVC keeps the data.
+set -euo pipefail
+
+: "${NAMESPACE:?}" "${POD_NAME:?}" "${KEEP_URL:?}"
+
+pod="keep-0"
+key="rt-${POD_NAME}"
+body="$(mktemp)"
+
+wait_ready() {
+  local deadline=$((SECONDS + 300))
+  until curl --max-time 5 --silent --fail "$KEEP_URL/readyz" >/dev/null 2>&1; do
+    if ((SECONDS >= deadline)); then
+      echo "keep: /readyz did not answer through the Service in 300s" >&2
+      return 1
+    fi
+    sleep 2
+  done
+}
+
+# http_json <expected-status> <method> <path> [body]; the response is in $body.
+http_json() {
+  local want="$1" method="$2" path="$3" data="${4:-}"
+  local args=(--max-time 15 --silent --show-error -o "$body" -w '%{http_code}'
+    -X "$method" "$KEEP_URL$path")
+  [[ -n "$data" ]] && args+=(-H 'content-type: application/json' --data "$data")
+  local status
+  status="$(curl "${args[@]}")"
+  echo "keep: $method $path -> $status $(cat "$body")"
+  if [[ "$status" != "$want" ]]; then
+    echo "keep: expected $want" >&2
+    return 1
+  fi
+}
+
+read_back() {
+  http_json 200 GET "/kv/$key"
+  jq -e --arg run "$POD_NAME" '.value.run == $run and .value.n == 1' "$body" >/dev/null || {
+    echo "keep: read-back value for $key is wrong" >&2
+    return 1
+  }
+}
+
+echo "[keep] step 1: readyz"
+wait_ready
+
+echo "[keep] step 2: PUT and GET round trip"
+http_json 200 PUT "/kv/$key" "{\"value\":{\"n\":1,\"run\":\"$POD_NAME\"}}"
+read_back
+
+echo "[keep] step 3: delete $pod; the data must survive"
+kubectl -n "$NAMESPACE" delete pod "$pod" --wait=true --timeout=180s
+# The StatefulSet makes the new pod a moment after the old one is gone, and
+# `kubectl wait` on a pod that does not exist fails at once. Wait for it first.
+deadline=$((SECONDS + 120))
+until kubectl -n "$NAMESPACE" get pod "$pod" >/dev/null 2>&1; do
+  if ((SECONDS >= deadline)); then
+    echo "keep: the StatefulSet did not make $pod again in 120s" >&2
+    exit 1
+  fi
+  sleep 2
+done
+kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$pod" --timeout=300s
+wait_ready
+read_back
+
+echo "[keep] PASS: readyz, round trip, data survives a pod restart"

k8s/overlays/gke-test/kustomization.yaml

@@ -0,0 +1,53 @@
+# Shared GKE test cluster overlay. faberline/aw deploys it through
+# .github/workflows/gke-test.yml; the rules are in aw's
+# docs/standards/test-cluster.md.
+#
+# - nodeSelector + toleration: run only on pool-keep-server (aw's
+#   infra/gke-test/pools.yaml: e2-standard-4, because keep requests 2 CPU).
+# - One pod per node: a required pod anti-affinity that matches every pod of
+#   the namespace, so each replica has a node of its own. keep has 1 replica
+#   today; pool-keep-server needs max_nodes >= replicas when that grows.
+# - Requests without limits: the base sets limits for real clusters; the
+#   overlay removes them so the pod can use the whole node.
+# - The test client is a Job in k8s/gke-test-job/ on pool-keep-client.
+# - ghcr-pull: the run creates this pull secret from its GITHUB_TOKEN.
+# - The run sets the image; a REPLACE_ME left here fails the run.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources: [../../base]
+images:
+  - name: keep
+    newName: ghcr.io/faberline/keep
+    newTag: REPLACE_ME__TAG
+patches:
+  - target:
+      kind: StatefulSet
+      name: keep
+    patch: |-
+      apiVersion: apps/v1
+      kind: StatefulSet
+      metadata:
+        name: keep
+      spec:
+        template:
+          spec:
+            nodeSelector:
+              faberline.dev/pool: keep-server
+            tolerations:
+              - key: faberline.dev/pool
+                operator: Equal
+                value: keep-server
+                effect: NoSchedule
+            imagePullSecrets:
+              - name: ghcr-pull
+            affinity:
+              podAntiAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  - topologyKey: kubernetes.io/hostname
+                    labelSelector: {}
+  - target:
+      kind: StatefulSet
+      name: keep
+    patch: |-
+      - op: remove
+        path: /spec/template/spec/containers/0/resources/limits