ci: seed pnpm cache after lockfile refresh

.github/workflows/docker-agent.yml

@@ -60,15 +60,43 @@ jobs:
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_PASSWORD }}
 
+      # Keep heavy agent layers on ordinal 1, separate from the server push
+      # cache on ordinal 0. The local DinD daemon is still used for the fresh
+      # ffmpeg smoke run below; only BuildKit execution moves to the persistent
+      # remote cache owner.
+      - name: Select BuildKit endpoint
+        id: buildkit-endpoint
+        env:
+          PREFERRED_ORDINAL: "1"
+        run: |
+          set -euo pipefail
+          if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then
+            echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2
+            exit 1
+          fi
+
+          probe_buildkit() {
+            local host="$1"
+            local timeout_seconds="$2"
+            timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234
+          }
+
+          preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local"
+          fallback="buildkit-amd64.ci.svc.cluster.local"
+          host="${preferred}"
+          if ! probe_buildkit "${preferred}" 5; then
+            echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint"
+            host="${fallback}"
+            probe_buildkit "${fallback}" 10
+          fi
+          echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT"
+          echo "Using BuildKit endpoint tcp://${host}:1234"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
-          # Same rationale as docker.yml — ephemeral builder per run so a
-          # corrupt content-store blob can't propagate across builds. The
-          # agent buildcache (paperclip-agent:buildcache) hasn't surfaced
-          # the same `short read` symptom yet, but it's the same runner +
-          # same persistent-builder pattern, so apply the fix preemptively.
-          cleanup: true
+          driver: remote
+          endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}
 
       - name: Resolve build SHA
         id: sha

.github/workflows/docker.yml

@@ -144,11 +144,43 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      # BuildKit cache is local to each StatefulSet ordinal. Ephemeral ARC
+      # client IPs make Service session affinity ineffective between runs, so
+      # route each build class to a stable cache owner. The ClusterIP remains a
+      # bounded availability fallback when the preferred ordinal is unhealthy.
+      - name: Select BuildKit endpoint
+        id: buildkit-endpoint
+        env:
+          PREFERRED_ORDINAL: ${{ github.event_name == 'workflow_dispatch' && '1' || '0' }}
+        run: |
+          set -euo pipefail
+          if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then
+            echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2
+            exit 1
+          fi
+
+          probe_buildkit() {
+            local host="$1"
+            local timeout_seconds="$2"
+            timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234
+          }
+
+          preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local"
+          fallback="buildkit-amd64.ci.svc.cluster.local"
+          host="${preferred}"
+          if ! probe_buildkit "${preferred}" 5; then
+            echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint"
+            host="${fallback}"
+            probe_buildkit "${fallback}" 10
+          fi
+          echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT"
+          echo "Using BuildKit endpoint tcp://${host}:1234"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
           driver: remote
-          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+          endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}
 
       # Runtime packages and global CLIs change far less often than /app.
       # Address the base by its inputs and only build it when that exact image

.github/workflows/refresh-lockfile.yml

@@ -32,11 +32,34 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 24
-          cache: pnpm
 
       - name: Refresh pnpm lockfile
         run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile
 
+      - name: Resolve pnpm store cache
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+          echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT"
+
+      - name: Restore pnpm store
+        id: pnpm-cache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ steps.pnpm-cache.outputs.path }}
+          key: node-cache-${{ runner.os }}-${{ steps.pnpm-cache.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
+
+      - name: Populate pnpm store
+        run: pnpm fetch --frozen-lockfile
+
+      - name: Save populated pnpm store
+        if: success() && steps.pnpm-cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.pnpm-cache.outputs.path }}
+          key: node-cache-${{ runner.os }}-${{ steps.pnpm-cache.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
+
       - name: Fail on unexpected file changes
         run: |
           changed="$(git status --porcelain)"

scripts/__tests__/ci-cache-routing.test.mjs

@@ -4,6 +4,8 @@ import test from "node:test";
 
 const prWorkflowPath = ".github/workflows/pr.yml";
 const dockerWorkflowPath = ".github/workflows/docker.yml";
+const dockerAgentWorkflowPath = ".github/workflows/docker-agent.yml";
+const refreshLockfileWorkflowPath = ".github/workflows/refresh-lockfile.yml";
 
 test("PR jobs do not install an ineffective compiler cache", async () => {
   const workflow = await readFile(prWorkflowPath, "utf8");
@@ -13,13 +15,41 @@ test("PR jobs do not install an ineffective compiler cache", async () => {
 
 test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded compression", async () => {
   const workflow = await readFile(dockerWorkflowPath, "utf8");
+  const agentWorkflow = await readFile(dockerAgentWorkflowPath, "utf8");
   const buildJob = workflow.match(/\n  build-and-push:\n([\s\S]*?)\n  deploy:\n/)?.[1];
 
   assert.ok(buildJob, "build-and-push job is missing");
   assert.match(buildJob, /\n    runs-on: arc-paperclip-buildkit\n/);
   assert.match(
     buildJob,
-    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: tcp:\/\/buildkit-amd64\.ci\.svc\.cluster\.local:1234\n/,
+    /PREFERRED_ORDINAL: \$\{\{ github\.event_name == 'workflow_dispatch' && '1' \|\| '0' \}\}/,
+  );
+  assert.match(
+    buildJob,
+    /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/,
+  );
+  assert.match(buildJob, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/);
+  assert.match(
+    buildJob,
+    /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/,
+  );
+  assert.match(
+    buildJob,
+    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/,
+  );
+  assert.match(agentWorkflow, /PREFERRED_ORDINAL: "1"/);
+  assert.match(
+    agentWorkflow,
+    /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/,
+  );
+  assert.match(agentWorkflow, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/);
+  assert.match(
+    agentWorkflow,
+    /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/,
+  );
+  assert.match(
+    agentWorkflow,
+    /uses: docker\/setup-buildx-action@v4\n        with:\n          driver: remote\n          endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/,
   );
   assert.match(
     buildJob,
@@ -36,3 +66,49 @@ test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded
   );
   assert.doesNotMatch(buildJob, /outputs: [^\n]*force-compression/);
 });
+
+test("Refresh Lockfile populates the setup-node-compatible pnpm cache before saving", async () => {
+  const workflow = await readFile(refreshLockfileWorkflowPath, "utf8");
+  const setupNode = workflow.match(
+    /\n      - name: Setup Node\.js\n([\s\S]*?)(?=\n      - name:)/,
+  )?.[1];
+
+  assert.ok(setupNode, "Refresh Lockfile Setup Node.js step is missing");
+  assert.doesNotMatch(setupNode, /\n\s+cache: pnpm\n/);
+
+  const cacheKey =
+    /key: node-cache-\$\{\{ runner\.os \}\}-\$\{\{ steps\.pnpm-cache\.outputs\.arch \}\}-pnpm-\$\{\{ hashFiles\('pnpm-lock\.yaml'\) \}\}/g;
+  assert.equal(
+    [...workflow.matchAll(cacheKey)].length,
+    2,
+    "restore and save must use setup-node's exact PNPM cache key shape",
+  );
+  assert.match(
+    workflow,
+    /- name: Resolve pnpm store cache\n        id: pnpm-cache[\s\S]*echo "path=\$\(pnpm store path --silent\)"[\s\S]*echo "arch=\$\(node -p 'process\.arch'\)"/,
+  );
+  assert.match(
+    workflow,
+    /- name: Restore pnpm store\n        id: pnpm-cache-restore\n        uses: actions\/cache\/restore@v4\n        with:\n          path: \$\{\{ steps\.pnpm-cache\.outputs\.path \}\}/,
+  );
+  assert.match(workflow, /- name: Populate pnpm store\n        run: pnpm fetch --frozen-lockfile/);
+  assert.match(
+    workflow,
+    /- name: Save populated pnpm store\n        if: success\(\) && steps\.pnpm-cache-restore\.outputs\.cache-hit != 'true'\n        uses: actions\/cache\/save@v4\n        with:\n          path: \$\{\{ steps\.pnpm-cache\.outputs\.path \}\}/,
+  );
+
+  const orderedSteps = [
+    "- name: Refresh pnpm lockfile",
+    "- name: Resolve pnpm store cache",
+    "- name: Restore pnpm store",
+    "- name: Populate pnpm store",
+    "- name: Save populated pnpm store",
+    "- name: Fail on unexpected file changes",
+  ].map((step) => workflow.indexOf(step));
+  assert.ok(orderedSteps.every((index) => index >= 0), "cache seeding steps are incomplete");
+  assert.deepEqual(
+    [...orderedSteps].sort((left, right) => left - right),
+    orderedSteps,
+    "the refreshed lockfile must be restored, fetched, and saved in order",
+  );
+});

server/src/__tests__/docker-opencode-runtime-pin.test.ts

@@ -130,12 +130,22 @@ describe("production Dockerfile k8s adapter runtime pins", () => {
     expect(dockerWorkflow.match(/runs-on: arc-paperclip-buildkit/g)).toHaveLength(1);
     expect(dockerWorkflow).not.toContain("runs-on: arc-dind");
     expect(dockerWorkflow).toContain("driver: remote");
-    expect(dockerWorkflow).toContain("endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234");
+    expect(dockerWorkflow).toContain(
+      "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}",
+    );
+    expect(dockerWorkflow).toContain(
+      "buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local",
+    );
     expect(dockerWorkflow.match(/runs-on: arc-deploy/g)).toHaveLength(1);
     expect(dockerWorkflow).toContain(
       "if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}",
     );
     expect(dockerAgentWorkflow.match(/runs-on: arc-dind/g)).toHaveLength(1);
+    expect(dockerAgentWorkflow).toContain("driver: remote");
+    expect(dockerAgentWorkflow).toContain('PREFERRED_ORDINAL: "1"');
+    expect(dockerAgentWorkflow).toContain(
+      "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}",
+    );
     expect(dockerAgentWorkflow).not.toContain("runs-on: arc-deploy");
     expect(dockerWorkflow).not.toContain("runs-on: self-hosted");
     expect(dockerAgentWorkflow).not.toContain("runs-on: self-hosted");