From ab31b281fe1464dcae3944eb168c73683c535cdd Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Sun, 9 Aug 2026 02:24:45 -0700 Subject: [PATCH 1/2] ci: seed pnpm cache after lockfile refresh --- .github/workflows/refresh-lockfile.yml | 25 ++++++++++- scripts/__tests__/ci-cache-routing.test.mjs | 47 +++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/refresh-lockfile.yml b/.github/workflows/refresh-lockfile.yml index 9e86ff1e59a0..b8fa494d6846 100644 --- a/.github/workflows/refresh-lockfile.yml +++ b/.github/workflows/refresh-lockfile.yml @@ -32,11 +32,34 @@ jobs: uses: actions/setup-node@v4 with: node-version: 24 - cache: pnpm - name: Refresh pnpm lockfile run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile + - name: Resolve pnpm store cache + id: pnpm-cache + shell: bash + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store + id: pnpm-cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ steps.pnpm-cache.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm-cache.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Populate pnpm store + run: pnpm fetch --frozen-lockfile + + - name: Save populated pnpm store + if: success() && steps.pnpm-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ steps.pnpm-cache.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm-cache.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + - name: Fail on unexpected file changes run: | changed="$(git status --porcelain)" diff --git a/scripts/__tests__/ci-cache-routing.test.mjs b/scripts/__tests__/ci-cache-routing.test.mjs index b1e22ef84d39..9a8806059a52 100644 --- a/scripts/__tests__/ci-cache-routing.test.mjs +++ b/scripts/__tests__/ci-cache-routing.test.mjs @@ -4,6 +4,7 @@ import test from "node:test"; const prWorkflowPath = ".github/workflows/pr.yml"; const dockerWorkflowPath = ".github/workflows/docker.yml"; +const refreshLockfileWorkflowPath = ".github/workflows/refresh-lockfile.yml"; test("PR jobs do not install an ineffective compiler cache", async () => { const workflow = await readFile(prWorkflowPath, "utf8"); @@ -36,3 +37,49 @@ test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded ); assert.doesNotMatch(buildJob, /outputs: [^\n]*force-compression/); }); + +test("Refresh Lockfile populates the setup-node-compatible pnpm cache before saving", async () => { + const workflow = await readFile(refreshLockfileWorkflowPath, "utf8"); + const setupNode = workflow.match( + /\n - name: Setup Node\.js\n([\s\S]*?)(?=\n - name:)/, + )?.[1]; + + assert.ok(setupNode, "Refresh Lockfile Setup Node.js step is missing"); + assert.doesNotMatch(setupNode, /\n\s+cache: pnpm\n/); + + const cacheKey = + /key: node-cache-\$\{\{ runner\.os \}\}-\$\{\{ steps\.pnpm-cache\.outputs\.arch \}\}-pnpm-\$\{\{ hashFiles\('pnpm-lock\.yaml'\) \}\}/g; + assert.equal( + [...workflow.matchAll(cacheKey)].length, + 2, + "restore and save must use setup-node's exact PNPM cache key shape", + ); + assert.match( + workflow, + /- name: Resolve pnpm store cache\n id: pnpm-cache[\s\S]*echo "path=\$\(pnpm store path --silent\)"[\s\S]*echo "arch=\$\(node -p 'process\.arch'\)"/, + ); + assert.match( + workflow, + /- name: Restore pnpm store\n id: pnpm-cache-restore\n uses: actions\/cache\/restore@v4\n with:\n path: \$\{\{ steps\.pnpm-cache\.outputs\.path \}\}/, + ); + assert.match(workflow, /- name: Populate pnpm store\n run: pnpm fetch --frozen-lockfile/); + assert.match( + workflow, + /- name: Save populated pnpm store\n if: success\(\) && steps\.pnpm-cache-restore\.outputs\.cache-hit != 'true'\n uses: actions\/cache\/save@v4\n with:\n path: \$\{\{ steps\.pnpm-cache\.outputs\.path \}\}/, + ); + + const orderedSteps = [ + "- name: Refresh pnpm lockfile", + "- name: Resolve pnpm store cache", + "- name: Restore pnpm store", + "- name: Populate pnpm store", + "- name: Save populated pnpm store", + "- name: Fail on unexpected file changes", + ].map((step) => workflow.indexOf(step)); + assert.ok(orderedSteps.every((index) => index >= 0), "cache seeding steps are incomplete"); + assert.deepEqual( + [...orderedSteps].sort((left, right) => left - right), + orderedSteps, + "the refreshed lockfile must be restored, fetched, and saved in order", + ); +}); From 3644848fb4dae94c379a38764b0178e39b4939ed Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Sun, 9 Aug 2026 04:29:49 -0700 Subject: [PATCH 2/2] ci: route builds to stable BuildKit cache owners --- .github/workflows/docker-agent.yml | 40 ++++++++++++++++--- .github/workflows/docker.yml | 34 +++++++++++++++- scripts/__tests__/ci-cache-routing.test.mjs | 31 +++++++++++++- .../docker-opencode-runtime-pin.test.ts | 12 +++++- 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-agent.yml b/.github/workflows/docker-agent.yml index bbee83d503ea..2f039337a9f5 100644 --- a/.github/workflows/docker-agent.yml +++ b/.github/workflows/docker-agent.yml @@ -60,15 +60,43 @@ jobs: username: ${{ secrets.HARBOR_USERNAME }} password: ${{ secrets.HARBOR_PASSWORD }} + # Keep heavy agent layers on ordinal 1, separate from the server push + # cache on ordinal 0. The local DinD daemon is still used for the fresh + # ffmpeg smoke run below; only BuildKit execution moves to the persistent + # remote cache owner. + - name: Select BuildKit endpoint + id: buildkit-endpoint + env: + PREFERRED_ORDINAL: "1" + run: | + set -euo pipefail + if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then + echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2 + exit 1 + fi + + probe_buildkit() { + local host="$1" + local timeout_seconds="$2" + timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234 + } + + preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local" + fallback="buildkit-amd64.ci.svc.cluster.local" + host="${preferred}" + if ! probe_buildkit "${preferred}" 5; then + echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint" + host="${fallback}" + probe_buildkit "${fallback}" 10 + fi + echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT" + echo "Using BuildKit endpoint tcp://${host}:1234" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: - # Same rationale as docker.yml — ephemeral builder per run so a - # corrupt content-store blob can't propagate across builds. The - # agent buildcache (paperclip-agent:buildcache) hasn't surfaced - # the same `short read` symptom yet, but it's the same runner + - # same persistent-builder pattern, so apply the fix preemptively. - cleanup: true + driver: remote + endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }} - name: Resolve build SHA id: sha diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f8ce3515264e..f34ca2891fb7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -144,11 +144,43 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # BuildKit cache is local to each StatefulSet ordinal. Ephemeral ARC + # client IPs make Service session affinity ineffective between runs, so + # route each build class to a stable cache owner. The ClusterIP remains a + # bounded availability fallback when the preferred ordinal is unhealthy. + - name: Select BuildKit endpoint + id: buildkit-endpoint + env: + PREFERRED_ORDINAL: ${{ github.event_name == 'workflow_dispatch' && '1' || '0' }} + run: | + set -euo pipefail + if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then + echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2 + exit 1 + fi + + probe_buildkit() { + local host="$1" + local timeout_seconds="$2" + timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234 + } + + preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local" + fallback="buildkit-amd64.ci.svc.cluster.local" + host="${preferred}" + if ! probe_buildkit "${preferred}" 5; then + echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint" + host="${fallback}" + probe_buildkit "${fallback}" 10 + fi + echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT" + echo "Using BuildKit endpoint tcp://${host}:1234" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver: remote - endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }} # Runtime packages and global CLIs change far less often than /app. # Address the base by its inputs and only build it when that exact image diff --git a/scripts/__tests__/ci-cache-routing.test.mjs b/scripts/__tests__/ci-cache-routing.test.mjs index 9a8806059a52..1c4b6b511a73 100644 --- a/scripts/__tests__/ci-cache-routing.test.mjs +++ b/scripts/__tests__/ci-cache-routing.test.mjs @@ -4,6 +4,7 @@ import test from "node:test"; const prWorkflowPath = ".github/workflows/pr.yml"; const dockerWorkflowPath = ".github/workflows/docker.yml"; +const dockerAgentWorkflowPath = ".github/workflows/docker-agent.yml"; const refreshLockfileWorkflowPath = ".github/workflows/refresh-lockfile.yml"; test("PR jobs do not install an ineffective compiler cache", async () => { @@ -14,13 +15,41 @@ test("PR jobs do not install an ineffective compiler cache", async () => { test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded compression", async () => { const workflow = await readFile(dockerWorkflowPath, "utf8"); + const agentWorkflow = await readFile(dockerAgentWorkflowPath, "utf8"); const buildJob = workflow.match(/\n build-and-push:\n([\s\S]*?)\n deploy:\n/)?.[1]; assert.ok(buildJob, "build-and-push job is missing"); assert.match(buildJob, /\n runs-on: arc-paperclip-buildkit\n/); assert.match( buildJob, - /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: tcp:\/\/buildkit-amd64\.ci\.svc\.cluster\.local:1234\n/, + /PREFERRED_ORDINAL: \$\{\{ github\.event_name == 'workflow_dispatch' && '1' \|\| '0' \}\}/, + ); + assert.match( + buildJob, + /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/, + ); + assert.match(buildJob, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/); + assert.match( + buildJob, + /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/, + ); + assert.match( + buildJob, + /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/, + ); + assert.match(agentWorkflow, /PREFERRED_ORDINAL: "1"/); + assert.match( + agentWorkflow, + /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/, + ); + assert.match(agentWorkflow, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/); + assert.match( + agentWorkflow, + /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/, + ); + assert.match( + agentWorkflow, + /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/, ); assert.match( buildJob, diff --git a/server/src/__tests__/docker-opencode-runtime-pin.test.ts b/server/src/__tests__/docker-opencode-runtime-pin.test.ts index 0e0ebbc7558b..8ad9c9a19dc6 100644 --- a/server/src/__tests__/docker-opencode-runtime-pin.test.ts +++ b/server/src/__tests__/docker-opencode-runtime-pin.test.ts @@ -130,12 +130,22 @@ describe("production Dockerfile k8s adapter runtime pins", () => { expect(dockerWorkflow.match(/runs-on: arc-paperclip-buildkit/g)).toHaveLength(1); expect(dockerWorkflow).not.toContain("runs-on: arc-dind"); expect(dockerWorkflow).toContain("driver: remote"); - expect(dockerWorkflow).toContain("endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234"); + expect(dockerWorkflow).toContain( + "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}", + ); + expect(dockerWorkflow).toContain( + "buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local", + ); expect(dockerWorkflow.match(/runs-on: arc-deploy/g)).toHaveLength(1); expect(dockerWorkflow).toContain( "if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}", ); expect(dockerAgentWorkflow.match(/runs-on: arc-dind/g)).toHaveLength(1); + expect(dockerAgentWorkflow).toContain("driver: remote"); + expect(dockerAgentWorkflow).toContain('PREFERRED_ORDINAL: "1"'); + expect(dockerAgentWorkflow).toContain( + "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}", + ); expect(dockerAgentWorkflow).not.toContain("runs-on: arc-deploy"); expect(dockerWorkflow).not.toContain("runs-on: self-hosted"); expect(dockerAgentWorkflow).not.toContain("runs-on: self-hosted");