Skip to content

ci(docker): cache BuildKit layers in a registry the runner advertises #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hbrombeer merged 1 commit into from
Sep 29, 2026
Merged

ci(docker): cache BuildKit layers in a registry the runner advertises #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 43 additions & 13 deletions .github/workflows/docker-gradle-build-push.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,47 @@ jobs:
mkdir -p "$docker_config"
echo "DOCKER_CONFIG=${docker_config}" >> "$GITHUB_ENV"

# Where BuildKit keeps its layer cache, decided on the runner:
# - GitHub-hosted: the repository-scoped GHA cache.
# - self-hosted with BUILDKIT_CACHE_REGISTRY: that registry. The hbr1 ARC
# runners advertise one next to them (plain HTTP, cluster-internal),
# the same way they advertise GRADLE_BUILD_CACHE_URL. The GHA cache is
# no option there: pulled from hbr1 it crawled, 2026-09-29 one 93 MB
# layer took 20+ minutes.
# - anything else: no layer cache.
# Exports to the registry use ignore-error so a cache hiccup costs a cold
# build next time rather than failing this one.
- name: Resolve layer cache
id: layer_cache
run: |
set -euo pipefail
repo="${GITHUB_REPOSITORY#*/}"
from=""; to=""; config=""
if [ "${RUNNER_ENVIRONMENT:-}" = "github-hosted" ]; then
from="type=gha,scope=${GITHUB_REPOSITORY}-amd64"
to="type=gha,scope=${GITHUB_REPOSITORY}-amd64,mode=max"
elif [ -n "${BUILDKIT_CACHE_REGISTRY:-}" ]; then
ref="${BUILDKIT_CACHE_REGISTRY}/cache/${repo}:amd64"
from="type=registry,ref=${ref}"
to="type=registry,ref=${ref},mode=max,ignore-error=true"
config="[registry.\"${BUILDKIT_CACHE_REGISTRY}\"]
http = true
insecure = true"
fi
{
echo "from=${from}"
echo "to=${to}"
echo "buildkitd-config<<EOF_BUILDKITD"
echo "${config}"
echo "EOF_BUILDKITD"
} >> "$GITHUB_OUTPUT"
echo "layer cache: ${from:-none}"

- name: Set up Buildx
uses: docker/setup-buildx-action@v4
with:
driver: docker-container
buildkitd-config-inline: ${{ steps.layer_cache.outputs.buildkitd-config }}

- name: Login to GitHub Container Registry
if: github.event_name != 'pull_request'
Expand Down Expand Up @@ -128,16 +165,9 @@ jobs:
org.opencontainers.image.vendor=Grounds
org.opencontainers.image.ref.name=${{ github.sha }}

# The GHA layer cache is used on GitHub-hosted runners only. From the
# self-hosted ARC runners it downloads at tens of KB/s: plugin-match
# 0.9.0 (2026-09-29) spent 20+ minutes pulling one 93 MB cached layer at
# ~60 KB/s for a build that takes a few minutes uncached — and the
# 60-minute ARC timeout below exists for the same symptom. On ARC the
# build runs without a layer cache instead. On hosted runners the
# repository-scoped cache still saves time without exposing intermediate
# layers in a potentially public GHCR package. PRs never write cache or
# image tags. Cache mounts (e.g. Gradle dependency mounts) are not
# exported either way.
# Layer cache: see "Resolve layer cache". PRs read it but never write
# cache or image tags. Cache mounts (e.g. Gradle dependency mounts) are
# not exported either way.
- name: Build image (pull requests)
if: github.event_name == 'pull_request'
uses: docker/build-push-action@v7
Expand All @@ -148,7 +178,7 @@ jobs:
platforms: linux/amd64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64', github.repository) || '' }}
cache-from: ${{ steps.layer_cache.outputs.from }}
build-args: |
GITHUB_USER=${{ github.actor }}
secrets: |
Expand All @@ -164,8 +194,8 @@ jobs:
platforms: linux/amd64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64', github.repository) || '' }}
cache-to: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64,mode=max', github.repository) || '' }}
cache-from: ${{ steps.layer_cache.outputs.from }}
cache-to: ${{ steps.layer_cache.outputs.to }}
build-args: |
GITHUB_USER=${{ github.actor }}
secrets: |
Expand Down
63 changes: 52 additions & 11 deletions tests/test_docker_workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ def test_build_and_cache_contract_for_public_and_private_events(self):
'github.event_name': event,
'github.repository': 'groundsgg/plugin-notifications',
'inputs.runner': '',
# Public repos land on GitHub's runners, private ones on their ARC set.
'runner.environment': 'github-hosted' if visibility == 'public' else 'self-hosted',
}
active = [step for step in job['steps'] if resolve('${{ ' + step['if'] + ' }}' if 'if' in step else True, context) not in ['False', False]]
builders = [step for step in active if 'setup-buildx-action' in step.get('uses', '')]
Expand All @@ -64,27 +62,70 @@ def test_build_and_cache_contract_for_public_and_private_events(self):
options = step['with']
self.assertEqual(options['platforms'], 'linux/amd64')
self.assertIn('github_token=', options['secrets'])
# The GHA layer cache downloads at tens of KB/s on ARC; hosted runners only.
expected_from = 'type=gha,scope=groundsgg/plugin-notifications-amd64' if visibility == 'public' else ''
self.assertEqual(resolve(options['cache-from'], context), expected_from)
# Where the cache lives is decided at run time by "Resolve layer cache".
self.assertEqual(options['cache-from'], '${{ steps.layer_cache.outputs.from }}')
pushes = [step for step in builds if step['with'].get('push') is True]
cache_writes = [step for step in builds if resolve(step['with'].get('cache-to', ''), context)]
cache_writes = [step for step in builds if step['with'].get('cache-to')]
if event == 'pull_request':
self.assertEqual(pushes, [])
self.assertEqual(cache_writes, [])
self.assertTrue(any(step['with'].get('load') for step in builds))
else:
self.assertEqual(len(pushes), 1)
if visibility == 'public':
self.assertEqual(len(cache_writes), 1)
self.assertIn('mode=max', resolve(cache_writes[0]['with']['cache-to'], context))
else:
self.assertEqual(cache_writes, [], 'ARC builds must not export to the GHA cache')
self.assertEqual(len(cache_writes), 1)
self.assertEqual(cache_writes[0]['with']['cache-to'], '${{ steps.layer_cache.outputs.to }}')
expected_runner = 'ubuntu-24.04' if visibility == 'public' else 'plugin-notifications'
self.assertEqual(resolve(job['runs-on'], context).strip(), expected_runner)
expected_budget = 30 if visibility == 'public' else 60
self.assertEqual(int(resolve(job['timeout-minutes'], context)), expected_budget)

def run_layer_cache(self, **env):
"""Run the "Resolve layer cache" script the way the runner would and return its outputs."""
import os
import subprocess
import tempfile
job = yaml.safe_load(WORKFLOW.read_text())['jobs']['docker-build-push']
script = next(step for step in job['steps'] if step.get('id') == 'layer_cache')['run']
with tempfile.NamedTemporaryFile('r', suffix='.out') as output:
base = {'PATH': os.environ['PATH'], 'GITHUB_OUTPUT': output.name,
'GITHUB_REPOSITORY': 'groundsgg/plugin-match'}
subprocess.run(['bash', '-c', script], env={**base, **env}, check=True, capture_output=True)
text = output.read()
outputs, lines = {}, iter(text.splitlines())
for line in lines:
if '<<' in line:
key, marker = line.split('<<', 1)
body = []
for inner in lines:
if inner == marker:
break
body.append(inner)
outputs[key] = '\n'.join(body).strip()
else:
key, _, value = line.partition('=')
outputs[key] = value
return outputs

def test_layer_cache_is_gha_on_hosted_runners(self):
out = self.run_layer_cache(RUNNER_ENVIRONMENT='github-hosted', BUILDKIT_CACHE_REGISTRY='ignored:5000')
self.assertEqual(out['from'], 'type=gha,scope=groundsgg/plugin-match-amd64')
self.assertEqual(out['to'], 'type=gha,scope=groundsgg/plugin-match-amd64,mode=max')
self.assertEqual(out['buildkitd-config'], '')

def test_layer_cache_uses_the_registry_a_self_hosted_runner_advertises(self):
registry = 'build-cache-registry.arc-runners.svc.cluster.local:5000'
out = self.run_layer_cache(RUNNER_ENVIRONMENT='self-hosted', BUILDKIT_CACHE_REGISTRY=registry)
ref = registry + '/cache/plugin-match:amd64'
self.assertEqual(out['from'], 'type=registry,ref=' + ref)
self.assertEqual(out['to'], 'type=registry,ref=' + ref + ',mode=max,ignore-error=true')
# BuildKit must be told the registry is plain HTTP, or every cache call fails TLS.
self.assertIn('[registry."' + registry + '"]', out['buildkitd-config'])
self.assertIn('http = true', out['buildkitd-config'])

def test_no_layer_cache_on_a_self_hosted_runner_without_a_registry(self):
out = self.run_layer_cache(RUNNER_ENVIRONMENT='self-hosted')
self.assertEqual((out['from'], out['to'], out['buildkitd-config']), ('', '', ''))

def test_cold_arc_budget_preserves_hosted_override_and_fallback_limits(self):
job = yaml.safe_load(WORKFLOW.read_text())['jobs']['docker-build-push']
cases = [
Expand Down