[codex] ci: route builds to stable BuildKit cache owners

.github/workflows/docker-agent.yml

@@ -60,15 +60,43 @@ jobs:
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_PASSWORD }}
 
+      # Keep heavy agent layers on ordinal 1, separate from the server push
+      # cache on ordinal 0. The local DinD daemon is still used for the fresh
+      # ffmpeg smoke run below; only BuildKit execution moves to the persistent
+      # remote cache owner.
+      - name: Select BuildKit endpoint
+        id: buildkit-endpoint
+        env:
+          PREFERRED_ORDINAL: "1"
+        run: |
+          set -euo pipefail
+          if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then
+            echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2
+            exit 1
+          fi
+
+          probe_buildkit() {
+            local host="$1"
+            local timeout_seconds="$2"
+            timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234
+          }
+
+          preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local"
+          fallback="buildkit-amd64.ci.svc.cluster.local"
+          host="${preferred}"
+          if ! probe_buildkit "${preferred}" 5; then
+            echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint"
+            host="${fallback}"
+            probe_buildkit "${fallback}" 10
+          fi
+          echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT"
+          echo "Using BuildKit endpoint tcp://${host}:1234"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
-          # Same rationale as docker.yml — ephemeral builder per run so a
-          # corrupt content-store blob can't propagate across builds. The
-          # agent buildcache (paperclip-agent:buildcache) hasn't surfaced
-          # the same `short read` symptom yet, but it's the same runner +
-          # same persistent-builder pattern, so apply the fix preemptively.
-          cleanup: true
+          driver: remote
+          endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}
 
       - name: Resolve build SHA
         id: sha

.github/workflows/docker.yml

@@ -144,11 +144,43 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      # BuildKit cache is local to each StatefulSet ordinal. Ephemeral ARC
+      # client IPs make Service session affinity ineffective between runs, so
+      # route each build class to a stable cache owner. The ClusterIP remains a
+      # bounded availability fallback when the preferred ordinal is unhealthy.
+      - name: Select BuildKit endpoint
+        id: buildkit-endpoint
+        env:
+          PREFERRED_ORDINAL: ${{ github.event_name == 'workflow_dispatch' && '1' || '0' }}
+        run: |
+          set -euo pipefail
+          if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then
+            echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2
+            exit 1
+          fi
+
+          probe_buildkit() {
+            local host="$1"
+            local timeout_seconds="$2"
+            timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234
+          }
+
+          preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local"
+          fallback="buildkit-amd64.ci.svc.cluster.local"
+          host="${preferred}"
+          if ! probe_buildkit "${preferred}" 5; then
+            echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint"
+            host="${fallback}"
+            probe_buildkit "${fallback}" 10
+          fi
+          echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT"
+          echo "Using BuildKit endpoint tcp://${host}:1234"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
           driver: remote
-          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+          endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}
 
       # Runtime packages and global CLIs change far less often than /app.
       # Address the base by its inputs and only build it when that exact image

scripts/__tests__/ci-cache-routing.test.mjs

@@ -4,6 +4,7 @@ import test from "node:test";
 
 const prWorkflowPath = ".github/workflows/pr.yml";
 const dockerWorkflowPath = ".github/workflows/docker.yml";
+const dockerAgentWorkflowPath = ".github/workflows/docker-agent.yml";
 const refreshLockfileWorkflowPath = ".github/workflows/refresh-lockfile.yml";
 
 test("PR jobs do not install an ineffective compiler cache", async () => {
@@ -14,13 +15,41 @@ test("PR jobs do not install an ineffective compiler cache", async () => {
 
 test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded compression", async () => {
   const workflow = await readFile(dockerWorkflowPath, "utf8");
+  const agentWorkflow = await readFile(dockerAgentWorkflowPath, "utf8");
   const buildJob = workflow.match(/\n  build-and-push:\n([\s\S]*?)\n  deploy:\n/)?.[1];
 
   assert.ok(buildJob, "build-and-push job is missing");
   assert.match(buildJob, /\n    runs-on: arc-paperclip-buildkit\n/);
   assert.match(
     buildJob,
-    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: tcp:\/\/buildkit-amd64\.ci\.svc\.cluster\.local:1234\n/,
+    /PREFERRED_ORDINAL: \$\{\{ github\.event_name == 'workflow_dispatch' && '1' \|\| '0' \}\}/,
+  );
+  assert.match(
+    buildJob,
+    /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/,
+  );
+  assert.match(buildJob, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/);
+  assert.match(
+    buildJob,
+    /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/,
+  );
+  assert.match(
+    buildJob,
+    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/,
+  );
+  assert.match(agentWorkflow, /PREFERRED_ORDINAL: "1"/);
+  assert.match(
+    agentWorkflow,
+    /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/,
+  );
+  assert.match(agentWorkflow, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/);
+  assert.match(
+    agentWorkflow,
+    /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/,
+  );
+  assert.match(
+    agentWorkflow,
+    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/,
   );
   assert.match(
     buildJob,

server/src/__tests__/docker-opencode-runtime-pin.test.ts

@@ -130,12 +130,22 @@ describe("production Dockerfile k8s adapter runtime pins", () => {
     expect(dockerWorkflow.match(/runs-on: arc-paperclip-buildkit/g)).toHaveLength(1);
     expect(dockerWorkflow).not.toContain("runs-on: arc-dind");
     expect(dockerWorkflow).toContain("driver: remote");
-    expect(dockerWorkflow).toContain("endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234");
+    expect(dockerWorkflow).toContain(
+      "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}",
+    );
+    expect(dockerWorkflow).toContain(
+      "buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local",
+    );
     expect(dockerWorkflow.match(/runs-on: arc-deploy/g)).toHaveLength(1);
     expect(dockerWorkflow).toContain(
       "if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}",
     );
     expect(dockerAgentWorkflow.match(/runs-on: arc-dind/g)).toHaveLength(1);
+    expect(dockerAgentWorkflow).toContain("driver: remote");
+    expect(dockerAgentWorkflow).toContain('PREFERRED_ORDINAL: "1"');
+    expect(dockerAgentWorkflow).toContain(
+      "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}",
+    );
     expect(dockerAgentWorkflow).not.toContain("runs-on: arc-deploy");
     expect(dockerWorkflow).not.toContain("runs-on: self-hosted");
     expect(dockerAgentWorkflow).not.toContain("runs-on: self-hosted");