From 3644848fb4dae94c379a38764b0178e39b4939ed Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Sun, 9 Aug 2026 04:29:49 -0700 Subject: [PATCH] ci: route builds to stable BuildKit cache owners --- .github/workflows/docker-agent.yml | 40 ++++++++++++++++--- .github/workflows/docker.yml | 34 +++++++++++++++- scripts/__tests__/ci-cache-routing.test.mjs | 31 +++++++++++++- .../docker-opencode-runtime-pin.test.ts | 12 +++++- 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-agent.yml b/.github/workflows/docker-agent.yml index bbee83d503ea..2f039337a9f5 100644 --- a/.github/workflows/docker-agent.yml +++ b/.github/workflows/docker-agent.yml @@ -60,15 +60,43 @@ jobs: username: ${{ secrets.HARBOR_USERNAME }} password: ${{ secrets.HARBOR_PASSWORD }} + # Keep heavy agent layers on ordinal 1, separate from the server push + # cache on ordinal 0. The local DinD daemon is still used for the fresh + # ffmpeg smoke run below; only BuildKit execution moves to the persistent + # remote cache owner. + - name: Select BuildKit endpoint + id: buildkit-endpoint + env: + PREFERRED_ORDINAL: "1" + run: | + set -euo pipefail + if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then + echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2 + exit 1 + fi + + probe_buildkit() { + local host="$1" + local timeout_seconds="$2" + timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234 + } + + preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local" + fallback="buildkit-amd64.ci.svc.cluster.local" + host="${preferred}" + if ! probe_buildkit "${preferred}" 5; then + echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint" + host="${fallback}" + probe_buildkit "${fallback}" 10 + fi + echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT" + echo "Using BuildKit endpoint tcp://${host}:1234" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: - # Same rationale as docker.yml — ephemeral builder per run so a - # corrupt content-store blob can't propagate across builds. The - # agent buildcache (paperclip-agent:buildcache) hasn't surfaced - # the same `short read` symptom yet, but it's the same runner + - # same persistent-builder pattern, so apply the fix preemptively. - cleanup: true + driver: remote + endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }} - name: Resolve build SHA id: sha diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f8ce3515264e..f34ca2891fb7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -144,11 +144,43 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # BuildKit cache is local to each StatefulSet ordinal. Ephemeral ARC + # client IPs make Service session affinity ineffective between runs, so + # route each build class to a stable cache owner. The ClusterIP remains a + # bounded availability fallback when the preferred ordinal is unhealthy. + - name: Select BuildKit endpoint + id: buildkit-endpoint + env: + PREFERRED_ORDINAL: ${{ github.event_name == 'workflow_dispatch' && '1' || '0' }} + run: | + set -euo pipefail + if [[ ! "${PREFERRED_ORDINAL}" =~ ^[01]$ ]]; then + echo "Invalid BuildKit ordinal: ${PREFERRED_ORDINAL}" >&2 + exit 1 + fi + + probe_buildkit() { + local host="$1" + local timeout_seconds="$2" + timeout "${timeout_seconds}" nc -z -w "${timeout_seconds}" "${host}" 1234 + } + + preferred="buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local" + fallback="buildkit-amd64.ci.svc.cluster.local" + host="${preferred}" + if ! probe_buildkit "${preferred}" 5; then + echo "::warning::Preferred BuildKit ${preferred} is unavailable; using the healthy service endpoint" + host="${fallback}" + probe_buildkit "${fallback}" 10 + fi + echo "endpoint=tcp://${host}:1234" >> "$GITHUB_OUTPUT" + echo "Using BuildKit endpoint tcp://${host}:1234" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver: remote - endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }} # Runtime packages and global CLIs change far less often than /app. # Address the base by its inputs and only build it when that exact image diff --git a/scripts/__tests__/ci-cache-routing.test.mjs b/scripts/__tests__/ci-cache-routing.test.mjs index 9a8806059a52..1c4b6b511a73 100644 --- a/scripts/__tests__/ci-cache-routing.test.mjs +++ b/scripts/__tests__/ci-cache-routing.test.mjs @@ -4,6 +4,7 @@ import test from "node:test"; const prWorkflowPath = ".github/workflows/pr.yml"; const dockerWorkflowPath = ".github/workflows/docker.yml"; +const dockerAgentWorkflowPath = ".github/workflows/docker-agent.yml"; const refreshLockfileWorkflowPath = ".github/workflows/refresh-lockfile.yml"; test("PR jobs do not install an ineffective compiler cache", async () => { @@ -14,13 +15,41 @@ test("PR jobs do not install an ineffective compiler cache", async () => { test("Docker builds use persistent remote BuildKit, exact-SHA cache, and bounded compression", async () => { const workflow = await readFile(dockerWorkflowPath, "utf8"); + const agentWorkflow = await readFile(dockerAgentWorkflowPath, "utf8"); const buildJob = workflow.match(/\n build-and-push:\n([\s\S]*?)\n deploy:\n/)?.[1]; assert.ok(buildJob, "build-and-push job is missing"); assert.match(buildJob, /\n runs-on: arc-paperclip-buildkit\n/); assert.match( buildJob, - /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: tcp:\/\/buildkit-amd64\.ci\.svc\.cluster\.local:1234\n/, + /PREFERRED_ORDINAL: \$\{\{ github\.event_name == 'workflow_dispatch' && '1' \|\| '0' \}\}/, + ); + assert.match( + buildJob, + /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/, + ); + assert.match(buildJob, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/); + assert.match( + buildJob, + /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/, + ); + assert.match( + buildJob, + /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/, + ); + assert.match(agentWorkflow, /PREFERRED_ORDINAL: "1"/); + assert.match( + agentWorkflow, + /preferred="buildkit-amd64-\$\{PREFERRED_ORDINAL\}\.buildkit-amd64-headless\.ci\.svc\.cluster\.local"/, + ); + assert.match(agentWorkflow, /fallback="buildkit-amd64\.ci\.svc\.cluster\.local"/); + assert.match( + agentWorkflow, + /timeout "\$\{timeout_seconds\}" nc -z -w "\$\{timeout_seconds\}" "\$\{host\}" 1234/, + ); + assert.match( + agentWorkflow, + /uses: docker\/setup-buildx-action@v4\n with:\n driver: remote\n endpoint: \$\{\{ steps\.buildkit-endpoint\.outputs\.endpoint \}\}\n/, ); assert.match( buildJob, diff --git a/server/src/__tests__/docker-opencode-runtime-pin.test.ts b/server/src/__tests__/docker-opencode-runtime-pin.test.ts index 0e0ebbc7558b..8ad9c9a19dc6 100644 --- a/server/src/__tests__/docker-opencode-runtime-pin.test.ts +++ b/server/src/__tests__/docker-opencode-runtime-pin.test.ts @@ -130,12 +130,22 @@ describe("production Dockerfile k8s adapter runtime pins", () => { expect(dockerWorkflow.match(/runs-on: arc-paperclip-buildkit/g)).toHaveLength(1); expect(dockerWorkflow).not.toContain("runs-on: arc-dind"); expect(dockerWorkflow).toContain("driver: remote"); - expect(dockerWorkflow).toContain("endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234"); + expect(dockerWorkflow).toContain( + "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}", + ); + expect(dockerWorkflow).toContain( + "buildkit-amd64-${PREFERRED_ORDINAL}.buildkit-amd64-headless.ci.svc.cluster.local", + ); expect(dockerWorkflow.match(/runs-on: arc-deploy/g)).toHaveLength(1); expect(dockerWorkflow).toContain( "if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}", ); expect(dockerAgentWorkflow.match(/runs-on: arc-dind/g)).toHaveLength(1); + expect(dockerAgentWorkflow).toContain("driver: remote"); + expect(dockerAgentWorkflow).toContain('PREFERRED_ORDINAL: "1"'); + expect(dockerAgentWorkflow).toContain( + "endpoint: ${{ steps.buildkit-endpoint.outputs.endpoint }}", + ); expect(dockerAgentWorkflow).not.toContain("runs-on: arc-deploy"); expect(dockerWorkflow).not.toContain("runs-on: self-hosted"); expect(dockerAgentWorkflow).not.toContain("runs-on: self-hosted");