diff --git a/.github/workflows/gke-test.yml b/.github/workflows/gke-test.yml new file mode 100644 index 0000000..9693168 --- /dev/null +++ b/.github/workflows/gke-test.yml @@ -0,0 +1,40 @@ +# Deploy keep to the shared faberline GKE test cluster and run the test Job +# in k8s/gke-test-job/ against it. faberline/aw owns the cluster, the pools +# (pool-keep-server for the server, pool-keep-client for the test Job), and the +# reusable workflow; keep owns k8s/overlays/gke-test/ and k8s/gke-test-job/. +# Rules: aw's docs/standards/test-cluster.md. +# +# Every pool is at 0 nodes between runs, so the first rollout waits a few +# minutes for the autoscaler to add a node. The run always deletes the +# namespace, even when the test fails. +name: gke-test + +on: + pull_request: + paths: + - "k8s/**" + - ".github/workflows/gke-test.yml" + - ".github/workflows/keep-test-image.yml" + - "Dockerfile" + workflow_dispatch: + +permissions: + contents: read + +jobs: + image: + permissions: + contents: read + packages: write + uses: ./.github/workflows/keep-test-image.yml + + gke-test: + needs: image + permissions: + id-token: write + contents: read + packages: read + uses: faberline/aw/.github/workflows/gke-test.yml@main + with: + image: ${{ needs.image.outputs.image }} + test-job: k8s/gke-test-job diff --git a/k8s/gke-test-job/job.yaml b/k8s/gke-test-job/job.yaml new file mode 100644 index 0000000..7914a4b --- /dev/null +++ b/k8s/gke-test-job/job.yaml @@ -0,0 +1,69 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: keep-test +spec: + # A retry would hide a flaky failure. The run fails on the first one. + backoffLimit: 0 + activeDeadlineSeconds: 1200 + template: + spec: + serviceAccountName: keep-test + restartPolicy: Never + nodeSelector: + faberline.dev/pool: keep-client + tolerations: + - key: faberline.dev/pool + operator: Equal + value: keep-client + effect: NoSchedule + imagePullSecrets: + - name: ghcr-pull + # One pod per node: no other pod of the namespace shares this node. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: {} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: verify + # kubectl, curl, jq, and bash. Match the minor version of the + # cluster (1.36). + image: alpine/k8s:1.36.4 + command: ["bash", "/scripts/verify.sh"] + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KEEP_URL + value: http://keep:7117 + # kubectl writes its cache under HOME. + - name: HOME + value: /tmp + resources: + requests: + cpu: 250m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: scripts + configMap: + name: keep-test-script diff --git a/k8s/gke-test-job/kustomization.yaml b/k8s/gke-test-job/kustomization.yaml new file mode 100644 index 0000000..80a00fe --- /dev/null +++ b/k8s/gke-test-job/kustomization.yaml @@ -0,0 +1,18 @@ +# Test Job for the shared GKE test cluster. faberline/aw's +# .github/workflows/gke-test.yml applies it after the keep StatefulSet is +# rolled out, and the run passes only when the Job completes. Rules: aw's +# docs/standards/test-cluster.md. +# +# The Job runs on pool-keep-client, so the test client never takes CPU from +# the keep server it tests. It reaches keep through the Service DNS name, not +# through a port-forward. The run sets the namespace. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: keep +resources: + - rbac.yaml + - job.yaml +configMapGenerator: + - name: keep-test-script + files: + - verify.sh diff --git a/k8s/gke-test-job/rbac.yaml b/k8s/gke-test-job/rbac.yaml new file mode 100644 index 0000000..e54f3eb --- /dev/null +++ b/k8s/gke-test-job/rbac.yaml @@ -0,0 +1,26 @@ +# Step 3 of verify.sh deletes keep-0 and waits for it to come back. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: keep-test +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: keep-test +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: keep-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: keep-test +subjects: + - kind: ServiceAccount + name: keep-test diff --git a/k8s/gke-test-job/verify.sh b/k8s/gke-test-job/verify.sh new file mode 100644 index 0000000..3c8f768 --- /dev/null +++ b/k8s/gke-test-job/verify.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Runs inside the keep-test Job on pool-keep-client. The Job's logs are the +# evidence; the shared run copies them into its artifact. +# +# Three steps: +# 1. /readyz answers 200 through the keep Service. +# 2. PUT /kv/{key} then GET /kv/{key} returns the same value. +# 3. keep-0 is deleted and comes back Ready; the same GET still returns the +# value. This proves the PVC keeps the data. +set -euo pipefail + +: "${NAMESPACE:?}" "${POD_NAME:?}" "${KEEP_URL:?}" + +pod="keep-0" +key="rt-${POD_NAME}" +body="$(mktemp)" + +wait_ready() { + local deadline=$((SECONDS + 300)) + until curl --max-time 5 --silent --fail "$KEEP_URL/readyz" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "keep: /readyz did not answer through the Service in 300s" >&2 + return 1 + fi + sleep 2 + done +} + +# http_json [body]; the response is in $body. +http_json() { + local want="$1" method="$2" path="$3" data="${4:-}" + local args=(--max-time 15 --silent --show-error -o "$body" -w '%{http_code}' + -X "$method" "$KEEP_URL$path") + [[ -n "$data" ]] && args+=(-H 'content-type: application/json' --data "$data") + local status + status="$(curl "${args[@]}")" + echo "keep: $method $path -> $status $(cat "$body")" + if [[ "$status" != "$want" ]]; then + echo "keep: expected $want" >&2 + return 1 + fi +} + +read_back() { + http_json 200 GET "/kv/$key" + jq -e --arg run "$POD_NAME" '.value.run == $run and .value.n == 1' "$body" >/dev/null || { + echo "keep: read-back value for $key is wrong" >&2 + return 1 + } +} + +echo "[keep] step 1: readyz" +wait_ready + +echo "[keep] step 2: PUT and GET round trip" +http_json 200 PUT "/kv/$key" "{\"value\":{\"n\":1,\"run\":\"$POD_NAME\"}}" +read_back + +echo "[keep] step 3: delete $pod; the data must survive" +kubectl -n "$NAMESPACE" delete pod "$pod" --wait=true --timeout=180s +# The StatefulSet makes the new pod a moment after the old one is gone, and +# `kubectl wait` on a pod that does not exist fails at once. Wait for it first. +deadline=$((SECONDS + 120)) +until kubectl -n "$NAMESPACE" get pod "$pod" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "keep: the StatefulSet did not make $pod again in 120s" >&2 + exit 1 + fi + sleep 2 +done +kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$pod" --timeout=300s +wait_ready +read_back + +echo "[keep] PASS: readyz, round trip, data survives a pod restart" diff --git a/k8s/overlays/gke-test/kustomization.yaml b/k8s/overlays/gke-test/kustomization.yaml new file mode 100644 index 0000000..69be5a3 --- /dev/null +++ b/k8s/overlays/gke-test/kustomization.yaml @@ -0,0 +1,53 @@ +# Shared GKE test cluster overlay. faberline/aw deploys it through +# .github/workflows/gke-test.yml; the rules are in aw's +# docs/standards/test-cluster.md. +# +# - nodeSelector + toleration: run only on pool-keep-server (aw's +# infra/gke-test/pools.yaml: e2-standard-4, because keep requests 2 CPU). +# - One pod per node: a required pod anti-affinity that matches every pod of +# the namespace, so each replica has a node of its own. keep has 1 replica +# today; pool-keep-server needs max_nodes >= replicas when that grows. +# - Requests without limits: the base sets limits for real clusters; the +# overlay removes them so the pod can use the whole node. +# - The test client is a Job in k8s/gke-test-job/ on pool-keep-client. +# - ghcr-pull: the run creates this pull secret from its GITHUB_TOKEN. +# - The run sets the image; a REPLACE_ME left here fails the run. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [../../base] +images: + - name: keep + newName: ghcr.io/faberline/keep + newTag: REPLACE_ME__TAG +patches: + - target: + kind: StatefulSet + name: keep + patch: |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: keep + spec: + template: + spec: + nodeSelector: + faberline.dev/pool: keep-server + tolerations: + - key: faberline.dev/pool + operator: Equal + value: keep-server + effect: NoSchedule + imagePullSecrets: + - name: ghcr-pull + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: {} + - target: + kind: StatefulSet + name: keep + patch: |- + - op: remove + path: /spec/template/spec/containers/0/resources/limits