From 2845f0fcbb8536820e46f40297e1ed13999890ea Mon Sep 17 00:00:00 2001 From: chrischeng-c4 Date: Thu, 15 Oct 2026 14:59:14 +0800 Subject: [PATCH 1/4] feat(gke-test): test keep on the shared faberline GKE cluster Local kind clusters fight for the same CPU and memory. keep now has a pull request run on the shared cluster faberline-test, which faberline/aw owns. Its pool pool-keep has 0 nodes between runs. - k8s/overlays/gke-test: base plus the GHCR image, and a StatefulSet patch that pins keep to pool-keep (nodeSelector and toleration) and pulls with the ghcr-pull secret that the run creates. - scripts/gke-test/verify.sh: /readyz, a PUT and GET round trip on /kv/{key}, then delete keep-0 and read the same key again to prove the PVC keeps the data. - .github/workflows/gke-test.yml: build the image with keep-test-image.yml, then call aw's reusable gke-test.yml, which deploys, runs verify.sh, and always deletes the namespace. The aw refs point at claude/gke-test-cluster until faberline/aw#32 merges. --- .github/workflows/gke-test.yml | 43 +++++++++ k8s/overlays/gke-test/kustomization.yaml | 36 ++++++++ scripts/gke-test/verify.sh | 106 +++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 .github/workflows/gke-test.yml create mode 100644 k8s/overlays/gke-test/kustomization.yaml create mode 100755 scripts/gke-test/verify.sh diff --git a/.github/workflows/gke-test.yml b/.github/workflows/gke-test.yml new file mode 100644 index 0000000..d5e5930 --- /dev/null +++ b/.github/workflows/gke-test.yml @@ -0,0 +1,43 @@ +# Deploy keep to the shared faberline GKE test cluster and run +# scripts/gke-test/verify.sh against it. faberline/aw owns the cluster, the +# pool (pool-keep), and the reusable workflow; keep owns +# k8s/overlays/gke-test/ and the test command. Rules: aw's +# docs/standards/test-cluster.md. +# +# Every pool is at 0 nodes between runs, so the first rollout waits a few +# minutes for the autoscaler to add a node. The run always deletes the +# namespace, even when the test fails. +name: gke-test + +on: + pull_request: + paths: + - "k8s/**" + - "scripts/gke-test/**" + - ".github/workflows/gke-test.yml" + - ".github/workflows/keep-test-image.yml" + - "Dockerfile" + workflow_dispatch: + +permissions: + contents: read + +jobs: + image: + permissions: + contents: read + packages: write + uses: ./.github/workflows/keep-test-image.yml + + gke-test: + needs: image + permissions: + id-token: write + contents: read + packages: read + # TODO: switch both refs to main after faberline/aw#32 merges. + uses: faberline/aw/.github/workflows/gke-test.yml@claude/gke-test-cluster + with: + image: ${{ needs.image.outputs.image }} + test-command: scripts/gke-test/verify.sh + aw-ref: claude/gke-test-cluster diff --git a/k8s/overlays/gke-test/kustomization.yaml b/k8s/overlays/gke-test/kustomization.yaml new file mode 100644 index 0000000..f460501 --- /dev/null +++ b/k8s/overlays/gke-test/kustomization.yaml @@ -0,0 +1,36 @@ +# Shared GKE test cluster overlay. faberline/aw deploys it through +# .github/workflows/gke-test.yml; the rules are in aw's +# docs/standards/test-cluster.md. +# +# - nodeSelector + toleration: run only on pool-keep (aw's +# infra/gke-test/pools.yaml: e2-standard-4, because keep requests 2 CPU). +# - ghcr-pull: the run creates this pull secret from its GITHUB_TOKEN. +# - The run sets the image; a REPLACE_ME left here fails the run. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [../../base] +images: + - name: keep + newName: ghcr.io/faberline/keep + newTag: REPLACE_ME__TAG +patches: + - target: + kind: StatefulSet + name: keep + patch: |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: keep + spec: + template: + spec: + nodeSelector: + faberline.dev/pool: keep + tolerations: + - key: faberline.dev/pool + operator: Equal + value: keep + effect: NoSchedule + imagePullSecrets: + - name: ghcr-pull diff --git a/scripts/gke-test/verify.sh b/scripts/gke-test/verify.sh new file mode 100755 index 0000000..a1d636d --- /dev/null +++ b/scripts/gke-test/verify.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Test command for the shared GKE test cluster (faberline/aw +# .github/workflows/gke-test.yml). The run calls it from the repository root +# after the keep StatefulSet is rolled out, with NAMESPACE, EVIDENCE_DIR, +# RUN_LABEL, and KUBECONFIG set. +# +# Three steps, each leaves evidence in EVIDENCE_DIR: +# 1. /readyz answers 200 through a port-forward. +# 2. PUT /kv/{key} then GET /kv/{key} returns the same value. +# 3. keep-0 is deleted and comes back Ready; the same GET still returns the +# value. This proves the PVC keeps the data. +set -euo pipefail + +: "${NAMESPACE:?}" "${EVIDENCE_DIR:?}" "${RUN_LABEL:?}" + +ns="$NAMESPACE" +pod="keep-0" +local_port=17117 +base_url="http://127.0.0.1:${local_port}" +key="rt-${RUN_LABEL}" +forward_pid="" + +mkdir -p "$EVIDENCE_DIR" + +stop_forward() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" >/dev/null 2>&1 || true + wait "$forward_pid" >/dev/null 2>&1 || true + forward_pid="" + fi +} +trap stop_forward EXIT + +# A port-forward pins one pod. Step 3 replaces that pod, so a dead forward is +# started again instead of treated as a failure. +start_forward() { + stop_forward + local deadline=$((SECONDS + 120)) + while ((SECONDS < deadline)); do + if [[ -z "$forward_pid" ]] || ! kill -0 "$forward_pid" >/dev/null 2>&1; then + kubectl -n "$ns" port-forward service/keep "${local_port}:7117" \ + >>"$EVIDENCE_DIR/port-forward.log" 2>&1 & + forward_pid="$!" + sleep 1 + fi + if curl --max-time 5 --silent --fail "$base_url/readyz" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "keep: /readyz did not answer through the port-forward in 120s" >&2 + return 1 +} + +# http_json [body] +http_json() { + local out="$1" want="$2" method="$3" path="$4" body="${5:-}" + local args=(--max-time 15 --silent --show-error -o "$EVIDENCE_DIR/$out" + -w '%{http_code}' -X "$method" "$base_url$path") + [[ -n "$body" ]] && args+=(-H 'content-type: application/json' --data "$body") + local status + status="$(curl "${args[@]}")" + if [[ "$status" != "$want" ]]; then + echo "keep: $method $path returned $status, expected $want" >&2 + cat "$EVIDENCE_DIR/$out" >&2 || true + return 1 + fi +} + +read_back() { + local out="$1" + http_json "$out" 200 GET "/kv/$key" + jq -e --arg run "$RUN_LABEL" '.value.run == $run and .value.n == 1' \ + "$EVIDENCE_DIR/$out" >/dev/null || { + echo "keep: read-back value for $key is wrong" >&2 + cat "$EVIDENCE_DIR/$out" >&2 + return 1 + } +} + +echo "[keep] step 1: readyz" >&2 +start_forward +curl --max-time 5 --silent "$base_url/readyz" >"$EVIDENCE_DIR/step1-readyz.txt" || true + +echo "[keep] step 2: PUT and GET round trip" >&2 +http_json step2-write.json 200 PUT "/kv/$key" "{\"value\":{\"n\":1,\"run\":\"$RUN_LABEL\"}}" +read_back step2-read.json + +echo "[keep] step 3: delete $pod; the data must survive" >&2 +kubectl -n "$ns" delete pod "$pod" --wait=true --timeout=180s >&2 +# The StatefulSet makes the new pod a moment after the old one is gone, and +# `kubectl wait` on a pod that does not exist fails at once. Wait for it first. +deadline=$((SECONDS + 120)) +until kubectl -n "$ns" get pod "$pod" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "keep: the StatefulSet did not make $pod again in 120s" >&2 + exit 1 + fi + sleep 2 +done +kubectl -n "$ns" wait --for=condition=Ready "pod/$pod" --timeout=300s >&2 +start_forward +read_back step3-read-after-restart.json + +echo "[keep] PASS: readyz, round trip, data survives a pod restart" | + tee "$EVIDENCE_DIR/verdict.txt" >&2 From b5c68ce22e3bc2397e9add64d4ee54278b3b2583 Mon Sep 17 00:00:00 2001 From: chrischeng-c4 Date: Thu, 15 Oct 2026 16:41:30 +0800 Subject: [PATCH 2/4] ci(gke-test): use the shared workflow from faberline/aw main faberline/aw#32 is merged, so the pilot no longer needs the feature branch. aw-ref defaults to main. --- .github/workflows/gke-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/gke-test.yml b/.github/workflows/gke-test.yml index d5e5930..6d8ce48 100644 --- a/.github/workflows/gke-test.yml +++ b/.github/workflows/gke-test.yml @@ -35,9 +35,7 @@ jobs: id-token: write contents: read packages: read - # TODO: switch both refs to main after faberline/aw#32 merges. - uses: faberline/aw/.github/workflows/gke-test.yml@claude/gke-test-cluster + uses: faberline/aw/.github/workflows/gke-test.yml@main with: image: ${{ needs.image.outputs.image }} test-command: scripts/gke-test/verify.sh - aw-ref: claude/gke-test-cluster From c90355170e61dbad1f54d2efd44f56dea9ade895 Mon Sep 17 00:00:00 2001 From: chrischeng-c4 Date: Thu, 15 Oct 2026 23:57:44 +0800 Subject: [PATCH 3/4] feat(gke-test): run the check as an in-cluster test Job The check moves from a runner script with a port-forward to the Job in k8s/gke-test-job/ on pool-keep-client, so the client never takes CPU from the keep server. The server runs on pool-keep-server, with one replica per node (topologySpreadConstraints). Needs faberline/aw#35 applied first. --- .github/workflows/gke-test.yml | 13 ++- k8s/gke-test-job/job.yaml | 65 ++++++++++++++ k8s/gke-test-job/kustomization.yaml | 18 ++++ k8s/gke-test-job/rbac.yaml | 26 ++++++ k8s/gke-test-job/verify.sh | 75 ++++++++++++++++ k8s/overlays/gke-test/kustomization.yaml | 18 +++- scripts/gke-test/verify.sh | 106 ----------------------- 7 files changed, 205 insertions(+), 116 deletions(-) create mode 100644 k8s/gke-test-job/job.yaml create mode 100644 k8s/gke-test-job/kustomization.yaml create mode 100644 k8s/gke-test-job/rbac.yaml create mode 100644 k8s/gke-test-job/verify.sh delete mode 100755 scripts/gke-test/verify.sh diff --git a/.github/workflows/gke-test.yml b/.github/workflows/gke-test.yml index 6d8ce48..9693168 100644 --- a/.github/workflows/gke-test.yml +++ b/.github/workflows/gke-test.yml @@ -1,8 +1,8 @@ -# Deploy keep to the shared faberline GKE test cluster and run -# scripts/gke-test/verify.sh against it. faberline/aw owns the cluster, the -# pool (pool-keep), and the reusable workflow; keep owns -# k8s/overlays/gke-test/ and the test command. Rules: aw's -# docs/standards/test-cluster.md. +# Deploy keep to the shared faberline GKE test cluster and run the test Job +# in k8s/gke-test-job/ against it. faberline/aw owns the cluster, the pools +# (pool-keep-server for the server, pool-keep-client for the test Job), and the +# reusable workflow; keep owns k8s/overlays/gke-test/ and k8s/gke-test-job/. +# Rules: aw's docs/standards/test-cluster.md. # # Every pool is at 0 nodes between runs, so the first rollout waits a few # minutes for the autoscaler to add a node. The run always deletes the @@ -13,7 +13,6 @@ on: pull_request: paths: - "k8s/**" - - "scripts/gke-test/**" - ".github/workflows/gke-test.yml" - ".github/workflows/keep-test-image.yml" - "Dockerfile" @@ -38,4 +37,4 @@ jobs: uses: faberline/aw/.github/workflows/gke-test.yml@main with: image: ${{ needs.image.outputs.image }} - test-command: scripts/gke-test/verify.sh + test-job: k8s/gke-test-job diff --git a/k8s/gke-test-job/job.yaml b/k8s/gke-test-job/job.yaml new file mode 100644 index 0000000..4eed4e1 --- /dev/null +++ b/k8s/gke-test-job/job.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: keep-test +spec: + # A retry would hide a flaky failure. The run fails on the first one. + backoffLimit: 0 + activeDeadlineSeconds: 1200 + template: + spec: + serviceAccountName: keep-test + restartPolicy: Never + nodeSelector: + faberline.dev/pool: keep-client + tolerations: + - key: faberline.dev/pool + operator: Equal + value: keep-client + effect: NoSchedule + imagePullSecrets: + - name: ghcr-pull + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: verify + # kubectl, curl, jq, and bash. Match the minor version of the + # cluster (1.36). + image: alpine/k8s:1.36.4 + command: ["bash", "/scripts/verify.sh"] + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KEEP_URL + value: http://keep:7117 + # kubectl writes its cache under HOME. + - name: HOME + value: /tmp + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: scripts + configMap: + name: keep-test-script diff --git a/k8s/gke-test-job/kustomization.yaml b/k8s/gke-test-job/kustomization.yaml new file mode 100644 index 0000000..80a00fe --- /dev/null +++ b/k8s/gke-test-job/kustomization.yaml @@ -0,0 +1,18 @@ +# Test Job for the shared GKE test cluster. faberline/aw's +# .github/workflows/gke-test.yml applies it after the keep StatefulSet is +# rolled out, and the run passes only when the Job completes. Rules: aw's +# docs/standards/test-cluster.md. +# +# The Job runs on pool-keep-client, so the test client never takes CPU from +# the keep server it tests. It reaches keep through the Service DNS name, not +# through a port-forward. The run sets the namespace. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: keep +resources: + - rbac.yaml + - job.yaml +configMapGenerator: + - name: keep-test-script + files: + - verify.sh diff --git a/k8s/gke-test-job/rbac.yaml b/k8s/gke-test-job/rbac.yaml new file mode 100644 index 0000000..e54f3eb --- /dev/null +++ b/k8s/gke-test-job/rbac.yaml @@ -0,0 +1,26 @@ +# Step 3 of verify.sh deletes keep-0 and waits for it to come back. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: keep-test +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: keep-test +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: keep-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: keep-test +subjects: + - kind: ServiceAccount + name: keep-test diff --git a/k8s/gke-test-job/verify.sh b/k8s/gke-test-job/verify.sh new file mode 100644 index 0000000..3c8f768 --- /dev/null +++ b/k8s/gke-test-job/verify.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Runs inside the keep-test Job on pool-keep-client. The Job's logs are the +# evidence; the shared run copies them into its artifact. +# +# Three steps: +# 1. /readyz answers 200 through the keep Service. +# 2. PUT /kv/{key} then GET /kv/{key} returns the same value. +# 3. keep-0 is deleted and comes back Ready; the same GET still returns the +# value. This proves the PVC keeps the data. +set -euo pipefail + +: "${NAMESPACE:?}" "${POD_NAME:?}" "${KEEP_URL:?}" + +pod="keep-0" +key="rt-${POD_NAME}" +body="$(mktemp)" + +wait_ready() { + local deadline=$((SECONDS + 300)) + until curl --max-time 5 --silent --fail "$KEEP_URL/readyz" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "keep: /readyz did not answer through the Service in 300s" >&2 + return 1 + fi + sleep 2 + done +} + +# http_json [body]; the response is in $body. +http_json() { + local want="$1" method="$2" path="$3" data="${4:-}" + local args=(--max-time 15 --silent --show-error -o "$body" -w '%{http_code}' + -X "$method" "$KEEP_URL$path") + [[ -n "$data" ]] && args+=(-H 'content-type: application/json' --data "$data") + local status + status="$(curl "${args[@]}")" + echo "keep: $method $path -> $status $(cat "$body")" + if [[ "$status" != "$want" ]]; then + echo "keep: expected $want" >&2 + return 1 + fi +} + +read_back() { + http_json 200 GET "/kv/$key" + jq -e --arg run "$POD_NAME" '.value.run == $run and .value.n == 1' "$body" >/dev/null || { + echo "keep: read-back value for $key is wrong" >&2 + return 1 + } +} + +echo "[keep] step 1: readyz" +wait_ready + +echo "[keep] step 2: PUT and GET round trip" +http_json 200 PUT "/kv/$key" "{\"value\":{\"n\":1,\"run\":\"$POD_NAME\"}}" +read_back + +echo "[keep] step 3: delete $pod; the data must survive" +kubectl -n "$NAMESPACE" delete pod "$pod" --wait=true --timeout=180s +# The StatefulSet makes the new pod a moment after the old one is gone, and +# `kubectl wait` on a pod that does not exist fails at once. Wait for it first. +deadline=$((SECONDS + 120)) +until kubectl -n "$NAMESPACE" get pod "$pod" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "keep: the StatefulSet did not make $pod again in 120s" >&2 + exit 1 + fi + sleep 2 +done +kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$pod" --timeout=300s +wait_ready +read_back + +echo "[keep] PASS: readyz, round trip, data survives a pod restart" diff --git a/k8s/overlays/gke-test/kustomization.yaml b/k8s/overlays/gke-test/kustomization.yaml index f460501..967e137 100644 --- a/k8s/overlays/gke-test/kustomization.yaml +++ b/k8s/overlays/gke-test/kustomization.yaml @@ -2,8 +2,12 @@ # .github/workflows/gke-test.yml; the rules are in aw's # docs/standards/test-cluster.md. # -# - nodeSelector + toleration: run only on pool-keep (aw's +# - nodeSelector + toleration: run only on pool-keep-server (aw's # infra/gke-test/pools.yaml: e2-standard-4, because keep requests 2 CPU). +# - topologySpreadConstraints: one server replica per node, as on a real +# cluster. keep has 1 replica today; the rule holds when that grows, and +# pool-keep-server then needs max_nodes >= replicas. +# - The test client is a Job in k8s/gke-test-job/ on pool-keep-client. # - ghcr-pull: the run creates this pull secret from its GITHUB_TOKEN. # - The run sets the image; a REPLACE_ME left here fails the run. apiVersion: kustomize.config.k8s.io/v1beta1 @@ -26,11 +30,19 @@ patches: template: spec: nodeSelector: - faberline.dev/pool: keep + faberline.dev/pool: keep-server tolerations: - key: faberline.dev/pool operator: Equal - value: keep + value: keep-server effect: NoSchedule imagePullSecrets: - name: ghcr-pull + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: keep + role: server diff --git a/scripts/gke-test/verify.sh b/scripts/gke-test/verify.sh deleted file mode 100755 index a1d636d..0000000 --- a/scripts/gke-test/verify.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# Test command for the shared GKE test cluster (faberline/aw -# .github/workflows/gke-test.yml). The run calls it from the repository root -# after the keep StatefulSet is rolled out, with NAMESPACE, EVIDENCE_DIR, -# RUN_LABEL, and KUBECONFIG set. -# -# Three steps, each leaves evidence in EVIDENCE_DIR: -# 1. /readyz answers 200 through a port-forward. -# 2. PUT /kv/{key} then GET /kv/{key} returns the same value. -# 3. keep-0 is deleted and comes back Ready; the same GET still returns the -# value. This proves the PVC keeps the data. -set -euo pipefail - -: "${NAMESPACE:?}" "${EVIDENCE_DIR:?}" "${RUN_LABEL:?}" - -ns="$NAMESPACE" -pod="keep-0" -local_port=17117 -base_url="http://127.0.0.1:${local_port}" -key="rt-${RUN_LABEL}" -forward_pid="" - -mkdir -p "$EVIDENCE_DIR" - -stop_forward() { - if [[ -n "$forward_pid" ]]; then - kill "$forward_pid" >/dev/null 2>&1 || true - wait "$forward_pid" >/dev/null 2>&1 || true - forward_pid="" - fi -} -trap stop_forward EXIT - -# A port-forward pins one pod. Step 3 replaces that pod, so a dead forward is -# started again instead of treated as a failure. -start_forward() { - stop_forward - local deadline=$((SECONDS + 120)) - while ((SECONDS < deadline)); do - if [[ -z "$forward_pid" ]] || ! kill -0 "$forward_pid" >/dev/null 2>&1; then - kubectl -n "$ns" port-forward service/keep "${local_port}:7117" \ - >>"$EVIDENCE_DIR/port-forward.log" 2>&1 & - forward_pid="$!" - sleep 1 - fi - if curl --max-time 5 --silent --fail "$base_url/readyz" >/dev/null 2>&1; then - return 0 - fi - sleep 2 - done - echo "keep: /readyz did not answer through the port-forward in 120s" >&2 - return 1 -} - -# http_json [body] -http_json() { - local out="$1" want="$2" method="$3" path="$4" body="${5:-}" - local args=(--max-time 15 --silent --show-error -o "$EVIDENCE_DIR/$out" - -w '%{http_code}' -X "$method" "$base_url$path") - [[ -n "$body" ]] && args+=(-H 'content-type: application/json' --data "$body") - local status - status="$(curl "${args[@]}")" - if [[ "$status" != "$want" ]]; then - echo "keep: $method $path returned $status, expected $want" >&2 - cat "$EVIDENCE_DIR/$out" >&2 || true - return 1 - fi -} - -read_back() { - local out="$1" - http_json "$out" 200 GET "/kv/$key" - jq -e --arg run "$RUN_LABEL" '.value.run == $run and .value.n == 1' \ - "$EVIDENCE_DIR/$out" >/dev/null || { - echo "keep: read-back value for $key is wrong" >&2 - cat "$EVIDENCE_DIR/$out" >&2 - return 1 - } -} - -echo "[keep] step 1: readyz" >&2 -start_forward -curl --max-time 5 --silent "$base_url/readyz" >"$EVIDENCE_DIR/step1-readyz.txt" || true - -echo "[keep] step 2: PUT and GET round trip" >&2 -http_json step2-write.json 200 PUT "/kv/$key" "{\"value\":{\"n\":1,\"run\":\"$RUN_LABEL\"}}" -read_back step2-read.json - -echo "[keep] step 3: delete $pod; the data must survive" >&2 -kubectl -n "$ns" delete pod "$pod" --wait=true --timeout=180s >&2 -# The StatefulSet makes the new pod a moment after the old one is gone, and -# `kubectl wait` on a pod that does not exist fails at once. Wait for it first. -deadline=$((SECONDS + 120)) -until kubectl -n "$ns" get pod "$pod" >/dev/null 2>&1; do - if ((SECONDS >= deadline)); then - echo "keep: the StatefulSet did not make $pod again in 120s" >&2 - exit 1 - fi - sleep 2 -done -kubectl -n "$ns" wait --for=condition=Ready "pod/$pod" --timeout=300s >&2 -start_forward -read_back step3-read-after-restart.json - -echo "[keep] PASS: readyz, round trip, data survives a pod restart" | - tee "$EVIDENCE_DIR/verdict.txt" >&2 From 26754016398ec5ae4b35af68dd52d0414b029a1e Mon Sep 17 00:00:00 2001 From: chrischeng-c4 Date: Fri, 16 Oct 2026 01:03:49 +0800 Subject: [PATCH 4/4] feat(gke-test): one pod per node, requests without limits The server and the test Job each get a node of their own: a required pod anti-affinity on the hostname that matches every pod of the namespace. The overlay removes the base's limits, and the Job sets no limit, so each pod can use the whole node. Rules: aw docs/standards/test-cluster.md. --- k8s/gke-test-job/job.yaml | 8 +++++-- k8s/overlays/gke-test/kustomization.yaml | 27 ++++++++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/k8s/gke-test-job/job.yaml b/k8s/gke-test-job/job.yaml index 4eed4e1..7914a4b 100644 --- a/k8s/gke-test-job/job.yaml +++ b/k8s/gke-test-job/job.yaml @@ -19,6 +19,12 @@ spec: effect: NoSchedule imagePullSecrets: - name: ghcr-pull + # One pod per node: no other pod of the namespace shares this node. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: {} securityContext: runAsNonRoot: true runAsUser: 65532 @@ -49,8 +55,6 @@ spec: requests: cpu: 250m memory: 256Mi - limits: - memory: 256Mi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/k8s/overlays/gke-test/kustomization.yaml b/k8s/overlays/gke-test/kustomization.yaml index 967e137..69be5a3 100644 --- a/k8s/overlays/gke-test/kustomization.yaml +++ b/k8s/overlays/gke-test/kustomization.yaml @@ -4,9 +4,11 @@ # # - nodeSelector + toleration: run only on pool-keep-server (aw's # infra/gke-test/pools.yaml: e2-standard-4, because keep requests 2 CPU). -# - topologySpreadConstraints: one server replica per node, as on a real -# cluster. keep has 1 replica today; the rule holds when that grows, and -# pool-keep-server then needs max_nodes >= replicas. +# - One pod per node: a required pod anti-affinity that matches every pod of +# the namespace, so each replica has a node of its own. keep has 1 replica +# today; pool-keep-server needs max_nodes >= replicas when that grows. +# - Requests without limits: the base sets limits for real clusters; the +# overlay removes them so the pod can use the whole node. # - The test client is a Job in k8s/gke-test-job/ on pool-keep-client. # - ghcr-pull: the run creates this pull secret from its GITHUB_TOKEN. # - The run sets the image; a REPLACE_ME left here fails the run. @@ -38,11 +40,14 @@ patches: effect: NoSchedule imagePullSecrets: - name: ghcr-pull - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: keep - role: server + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: {} + - target: + kind: StatefulSet + name: keep + patch: |- + - op: remove + path: /spec/template/spec/containers/0/resources/limits