diff --git a/.github/workflows/docker-gradle-build-push.yml b/.github/workflows/docker-gradle-build-push.yml index f0270cc..40919f0 100644 --- a/.github/workflows/docker-gradle-build-push.yml +++ b/.github/workflows/docker-gradle-build-push.yml @@ -96,10 +96,47 @@ jobs: mkdir -p "$docker_config" echo "DOCKER_CONFIG=${docker_config}" >> "$GITHUB_ENV" + # Where BuildKit keeps its layer cache, decided on the runner: + # - GitHub-hosted: the repository-scoped GHA cache. + # - self-hosted with BUILDKIT_CACHE_REGISTRY: that registry. The hbr1 ARC + # runners advertise one next to them (plain HTTP, cluster-internal), + # the same way they advertise GRADLE_BUILD_CACHE_URL. The GHA cache is + # no option there: pulled from hbr1 it crawled, 2026-09-29 one 93 MB + # layer took 20+ minutes. + # - anything else: no layer cache. + # Exports to the registry use ignore-error so a cache hiccup costs a cold + # build next time rather than failing this one. + - name: Resolve layer cache + id: layer_cache + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY#*/}" + from=""; to=""; config="" + if [ "${RUNNER_ENVIRONMENT:-}" = "github-hosted" ]; then + from="type=gha,scope=${GITHUB_REPOSITORY}-amd64" + to="type=gha,scope=${GITHUB_REPOSITORY}-amd64,mode=max" + elif [ -n "${BUILDKIT_CACHE_REGISTRY:-}" ]; then + ref="${BUILDKIT_CACHE_REGISTRY}/cache/${repo}:amd64" + from="type=registry,ref=${ref}" + to="type=registry,ref=${ref},mode=max,ignore-error=true" + config="[registry.\"${BUILDKIT_CACHE_REGISTRY}\"] + http = true + insecure = true" + fi + { + echo "from=${from}" + echo "to=${to}" + echo "buildkitd-config<> "$GITHUB_OUTPUT" + echo "layer cache: ${from:-none}" + - name: Set up Buildx uses: docker/setup-buildx-action@v4 with: driver: docker-container + buildkitd-config-inline: ${{ steps.layer_cache.outputs.buildkitd-config }} - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' @@ -128,16 +165,9 @@ jobs: org.opencontainers.image.vendor=Grounds org.opencontainers.image.ref.name=${{ github.sha }} - # The GHA layer cache is used on GitHub-hosted runners only. From the - # self-hosted ARC runners it downloads at tens of KB/s: plugin-match - # 0.9.0 (2026-09-29) spent 20+ minutes pulling one 93 MB cached layer at - # ~60 KB/s for a build that takes a few minutes uncached — and the - # 60-minute ARC timeout below exists for the same symptom. On ARC the - # build runs without a layer cache instead. On hosted runners the - # repository-scoped cache still saves time without exposing intermediate - # layers in a potentially public GHCR package. PRs never write cache or - # image tags. Cache mounts (e.g. Gradle dependency mounts) are not - # exported either way. + # Layer cache: see "Resolve layer cache". PRs read it but never write + # cache or image tags. Cache mounts (e.g. Gradle dependency mounts) are + # not exported either way. - name: Build image (pull requests) if: github.event_name == 'pull_request' uses: docker/build-push-action@v7 @@ -148,7 +178,7 @@ jobs: platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64', github.repository) || '' }} + cache-from: ${{ steps.layer_cache.outputs.from }} build-args: | GITHUB_USER=${{ github.actor }} secrets: | @@ -164,8 +194,8 @@ jobs: platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64', github.repository) || '' }} - cache-to: ${{ runner.environment == 'github-hosted' && format('type=gha,scope={0}-amd64,mode=max', github.repository) || '' }} + cache-from: ${{ steps.layer_cache.outputs.from }} + cache-to: ${{ steps.layer_cache.outputs.to }} build-args: | GITHUB_USER=${{ github.actor }} secrets: | diff --git a/tests/test_docker_workflow.py b/tests/test_docker_workflow.py index 85cc646..6de4510 100644 --- a/tests/test_docker_workflow.py +++ b/tests/test_docker_workflow.py @@ -51,8 +51,6 @@ def test_build_and_cache_contract_for_public_and_private_events(self): 'github.event_name': event, 'github.repository': 'groundsgg/plugin-notifications', 'inputs.runner': '', - # Public repos land on GitHub's runners, private ones on their ARC set. - 'runner.environment': 'github-hosted' if visibility == 'public' else 'self-hosted', } active = [step for step in job['steps'] if resolve('${{ ' + step['if'] + ' }}' if 'if' in step else True, context) not in ['False', False]] builders = [step for step in active if 'setup-buildx-action' in step.get('uses', '')] @@ -64,27 +62,70 @@ def test_build_and_cache_contract_for_public_and_private_events(self): options = step['with'] self.assertEqual(options['platforms'], 'linux/amd64') self.assertIn('github_token=', options['secrets']) - # The GHA layer cache downloads at tens of KB/s on ARC; hosted runners only. - expected_from = 'type=gha,scope=groundsgg/plugin-notifications-amd64' if visibility == 'public' else '' - self.assertEqual(resolve(options['cache-from'], context), expected_from) + # Where the cache lives is decided at run time by "Resolve layer cache". + self.assertEqual(options['cache-from'], '${{ steps.layer_cache.outputs.from }}') pushes = [step for step in builds if step['with'].get('push') is True] - cache_writes = [step for step in builds if resolve(step['with'].get('cache-to', ''), context)] + cache_writes = [step for step in builds if step['with'].get('cache-to')] if event == 'pull_request': self.assertEqual(pushes, []) self.assertEqual(cache_writes, []) self.assertTrue(any(step['with'].get('load') for step in builds)) else: self.assertEqual(len(pushes), 1) - if visibility == 'public': - self.assertEqual(len(cache_writes), 1) - self.assertIn('mode=max', resolve(cache_writes[0]['with']['cache-to'], context)) - else: - self.assertEqual(cache_writes, [], 'ARC builds must not export to the GHA cache') + self.assertEqual(len(cache_writes), 1) + self.assertEqual(cache_writes[0]['with']['cache-to'], '${{ steps.layer_cache.outputs.to }}') expected_runner = 'ubuntu-24.04' if visibility == 'public' else 'plugin-notifications' self.assertEqual(resolve(job['runs-on'], context).strip(), expected_runner) expected_budget = 30 if visibility == 'public' else 60 self.assertEqual(int(resolve(job['timeout-minutes'], context)), expected_budget) + def run_layer_cache(self, **env): + """Run the "Resolve layer cache" script the way the runner would and return its outputs.""" + import os + import subprocess + import tempfile + job = yaml.safe_load(WORKFLOW.read_text())['jobs']['docker-build-push'] + script = next(step for step in job['steps'] if step.get('id') == 'layer_cache')['run'] + with tempfile.NamedTemporaryFile('r', suffix='.out') as output: + base = {'PATH': os.environ['PATH'], 'GITHUB_OUTPUT': output.name, + 'GITHUB_REPOSITORY': 'groundsgg/plugin-match'} + subprocess.run(['bash', '-c', script], env={**base, **env}, check=True, capture_output=True) + text = output.read() + outputs, lines = {}, iter(text.splitlines()) + for line in lines: + if '<<' in line: + key, marker = line.split('<<', 1) + body = [] + for inner in lines: + if inner == marker: + break + body.append(inner) + outputs[key] = '\n'.join(body).strip() + else: + key, _, value = line.partition('=') + outputs[key] = value + return outputs + + def test_layer_cache_is_gha_on_hosted_runners(self): + out = self.run_layer_cache(RUNNER_ENVIRONMENT='github-hosted', BUILDKIT_CACHE_REGISTRY='ignored:5000') + self.assertEqual(out['from'], 'type=gha,scope=groundsgg/plugin-match-amd64') + self.assertEqual(out['to'], 'type=gha,scope=groundsgg/plugin-match-amd64,mode=max') + self.assertEqual(out['buildkitd-config'], '') + + def test_layer_cache_uses_the_registry_a_self_hosted_runner_advertises(self): + registry = 'build-cache-registry.arc-runners.svc.cluster.local:5000' + out = self.run_layer_cache(RUNNER_ENVIRONMENT='self-hosted', BUILDKIT_CACHE_REGISTRY=registry) + ref = registry + '/cache/plugin-match:amd64' + self.assertEqual(out['from'], 'type=registry,ref=' + ref) + self.assertEqual(out['to'], 'type=registry,ref=' + ref + ',mode=max,ignore-error=true') + # BuildKit must be told the registry is plain HTTP, or every cache call fails TLS. + self.assertIn('[registry."' + registry + '"]', out['buildkitd-config']) + self.assertIn('http = true', out['buildkitd-config']) + + def test_no_layer_cache_on_a_self_hosted_runner_without_a_registry(self): + out = self.run_layer_cache(RUNNER_ENVIRONMENT='self-hosted') + self.assertEqual((out['from'], out['to'], out['buildkitd-config']), ('', '', '')) + def test_cold_arc_budget_preserves_hosted_override_and_fallback_limits(self): job = yaml.safe_load(WORKFLOW.read_text())['jobs']['docker-build-push'] cases = [