From fc9bb9174b1e76220801b26b1c1842e49627b281 Mon Sep 17 00:00:00 2001 From: Wikid82 Date: Tue, 8 Sep 2026 13:57:03 -0400 Subject: [PATCH 1/3] ci: pin regctl release version in the toolchain-pin check The verify-toolchain-pin job installed regctl via regctl-installer with the action's default 'latest' release, so a new regctl could change this required check's behavior without any change in the repo. Pin it to an explicit released version (v0.11.6) for reproducibility. Claude-Session: https://claude.ai/code/session_01KXA4x9LrA2AsnLrvdHMZbS --- .github/workflows/quality-checks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index a4dffff15..c3b180502 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -109,6 +109,11 @@ jobs: - name: Install regctl uses: iarekylew00t/regctl-installer@c2202c17a65fe59371c71ecc169c9e58c3710a15 # v4.0.16 + with: + # Pin the regctl binary to an explicit released version (not the + # action's `latest` default) so this required check is reproducible + # and can't shift underneath us when a new regctl ships. + regctl-release: v0.11.6 - name: Verify the pinned toolchain image matches the recipe env: From 09a984c2b44538b3aae080e67dd4c30809bb15da Mon Sep 17 00:00:00 2001 From: Wikid82 Date: Tue, 8 Sep 2026 13:57:16 -0400 Subject: [PATCH 2/3] test: widen toolchain freshness-guard bats coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify-toolchain-pin.bats: add the branches flagged as uncovered in the prebuilt-toolchain-image review — * same-repo run where the pinned :$KEY tag does not resolve in GHCR -> hard exit 1 (failure-closed) * CHARON_TOOLCHAIN_DIGEST blank / ARG line absent -> hard exit 1 toolchain-key.bats: add per-input sensitivity cases for the tonistiigi/xx pin and the ALPINE_IMAGE pin (each change flips the recomputed key; an unchanged re-emit keeps it stable). Helper gains TF_XX_PIN / TF_ALPINE_IMAGE knobs and a tf_stub_regctl_unresolvable mock (image digest exits non-zero), reusing the existing PATH-stub pattern. Also harden verify-toolchain-pin.sh's arg_value(): a missing ARG line now yields empty output instead of aborting the script under 'set -e' before the actionable CHARON_TOOLCHAIN_DIGEST error can print. Claude-Session: https://claude.ai/code/session_01KXA4x9LrA2AsnLrvdHMZbS --- scripts/tests/helpers/toolchain_fixture.bash | 21 ++++++++++-- scripts/tests/toolchain-key.bats | 28 +++++++++++++++ scripts/tests/verify-toolchain-pin.bats | 36 ++++++++++++++++++++ scripts/verify-toolchain-pin.sh | 4 +-- 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/scripts/tests/helpers/toolchain_fixture.bash b/scripts/tests/helpers/toolchain_fixture.bash index d13d3a4b9..ca9cdb12e 100644 --- a/scripts/tests/helpers/toolchain_fixture.bash +++ b/scripts/tests/helpers/toolchain_fixture.bash @@ -44,11 +44,13 @@ tf_write_dockerfile() { local geoip2_version="${TF_GEOIP2_VERSION:-v0.0.0-20260623062220-3675c6e7e63d}" local caddy_get_line="${TF_CADDY_GET_LINE:- _retry go get golang.org/x/net@v0.58.0; \\}" local golang_digest="${TF_GOLANG_DIGEST:-sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125}" + local alpine_image="${TF_ALPINE_IMAGE:-alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b}" + local xx_pin="${TF_XX_PIN:-tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707}" cat > "$TF_DF" < "$TF_BIN/regctl" <<'EOF' +#!/usr/bin/env bash +case "$1 $2" in + "registry login") exit 0 ;; + "image digest") echo "regctl: manifest unknown" >&2; exit 1 ;; + *) exit 0 ;; +esac +EOF + chmod +x "$TF_BIN/regctl" +} + # A minimal PATH that contains coreutils + the fixture stub dir but is # guaranteed NOT to contain a system `regctl`. Use for "regctl absent" tests: # PATH="$(tf_min_path)" run bash "$TF_ROOT/scripts/verify-toolchain-pin.sh" diff --git a/scripts/tests/toolchain-key.bats b/scripts/tests/toolchain-key.bats index 66e5fd86c..28a2947de 100644 --- a/scripts/tests/toolchain-key.bats +++ b/scripts/tests/toolchain-key.bats @@ -6,6 +6,8 @@ # * changes when a `go get` line INSIDE caddy-inline changes # * changes when a tracked ARG default (CADDY_VERSION / CADDY_GEOIP2_VERSION) moves # * changes when the digest-pinned golang base moves +# * changes when the tonistiigi/xx pin moves +# * changes when the ALPINE_IMAGE pin moves # * changes when .trivyignore changes # * fails loudly if stage extraction breaks @@ -70,6 +72,32 @@ key_of() { bash "$TF_ROOT/scripts/toolchain-key.sh" "$TF_DF"; } [ "$before" != "$after" ] } +@test "changes when the tonistiigi/xx pin moves (N4)" { + before="$(key_of)" + export TF_XX_PIN='tonistiigi/xx:1.9.1@sha256:1111111111111111111111111111111111111111111111111111111111111111' + tf_write_dockerfile + after="$(key_of)" + [ "$before" != "$after" ] +} + +@test "changes when the ALPINE_IMAGE pin moves" { + before="$(key_of)" + export TF_ALPINE_IMAGE='alpine:3.25.0@sha256:2222222222222222222222222222222222222222222222222222222222222222' + tf_write_dockerfile + after="$(key_of)" + [ "$before" != "$after" ] +} + +@test "stable when the tonistiigi/xx and ALPINE_IMAGE pins are re-emitted unchanged" { + before="$(key_of)" + # Re-write the fixture with the exact same (default) xx / alpine pins. + export TF_XX_PIN='tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707' + export TF_ALPINE_IMAGE='alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b' + tf_write_dockerfile + after="$(key_of)" + [ "$before" = "$after" ] +} + @test "changes when .trivyignore changes" { before="$(key_of)" printf 'another-cve\n' >> "$TF_ROOT/.trivyignore" diff --git a/scripts/tests/verify-toolchain-pin.bats b/scripts/tests/verify-toolchain-pin.bats index 8cee19b85..0092595a6 100644 --- a/scripts/tests/verify-toolchain-pin.bats +++ b/scripts/tests/verify-toolchain-pin.bats @@ -6,6 +6,8 @@ # mismatched tag -> exit 1 (actionable) # same-repo + regctl absent -> exit 1 # same-repo + GHCR_READ_TOKEN unset -> exit 1 +# same-repo + :$KEY tag does not resolve -> exit 1 +# same-repo + CHARON_TOOLCHAIN_DIGEST empty -> exit 1 # same-repo + GHCR digest != pinned digest -> exit 1 # same-repo + GHCR digest == pinned digest -> exit 0 @@ -74,6 +76,40 @@ verify() { bash "$TF_ROOT/scripts/verify-toolchain-pin.sh" "$TF_DF"; } [[ "$output" == *"hand-edited or stale"* ]] } +@test "same-repo PR + pinned :\$KEY tag does not resolve in GHCR: exit 1 (failure-closed)" { + tf_stub_regctl_unresolvable + export GITHUB_EVENT_NAME=pull_request + export GITHUB_REPOSITORY=wikid82/Charon + export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon + export GHCR_READ_TOKEN=tok + run verify + [ "$status" -eq 1 ] + [[ "$output" == *"does not resolve in GHCR"* ]] +} + +@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST value blank: exit 1 (failure-closed)" { + tf_stub_regctl "$GOOD_DIGEST" + sed -i "s|^ARG CHARON_TOOLCHAIN_DIGEST=.*|ARG CHARON_TOOLCHAIN_DIGEST=|" "$TF_DF" + export GITHUB_EVENT_NAME=pull_request + export GITHUB_REPOSITORY=wikid82/Charon + export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon + export GHCR_READ_TOKEN=tok + run verify + [ "$status" -eq 1 ] + [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]] +} + +@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST ARG line absent: exit 1 (failure-closed)" { + tf_stub_regctl "$GOOD_DIGEST" + sed -i "/^ARG CHARON_TOOLCHAIN_DIGEST=/d" "$TF_DF" + export GITHUB_EVENT_NAME=push + export GITHUB_REPOSITORY=wikid82/Charon + export GHCR_READ_TOKEN=tok + run verify + [ "$status" -eq 1 ] + [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]] +} + @test "same-repo PR + GHCR digest matches the pinned digest: exit 0" { tf_stub_regctl "$GOOD_DIGEST" export GITHUB_EVENT_NAME=pull_request diff --git a/scripts/verify-toolchain-pin.sh b/scripts/verify-toolchain-pin.sh index 7a425e316..a9e2398fb 100755 --- a/scripts/verify-toolchain-pin.sh +++ b/scripts/verify-toolchain-pin.sh @@ -43,8 +43,8 @@ if [[ ! -f "$df" ]]; then exit 2 fi -arg_value() { # $1 = ARG name - grep -E "^ARG $1=" "$df" | head -n1 | cut -d= -f2- +arg_value() { # $1 = ARG name; empty output (never a hard error) when absent + { grep -E "^ARG $1=" "$df" || true; } | head -n1 | cut -d= -f2- } KEY="$("$here/toolchain-key.sh" "$df")" From 9056392b3d1a0b6d6ea591bf42ba7f7eff004b52 Mon Sep 17 00:00:00 2001 From: Wikid82 Date: Tue, 8 Sep 2026 14:22:56 -0400 Subject: [PATCH 3/3] ci: add offline (inline-toolchain) build canary workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in CI builds the app image via the caddy-inline / crowdsec-inline path end to end, so the fork / air-gapped / not-logged-in-to-GHCR fallback (from-source Caddy + CrowdSec compile) can rot silently until someone tries to build offline. Add .github/workflows/build-offline.yml: an amd64-only, advisory (NOT required) canary that runs 'make build-offline' with GHA layer caching. * workflow_dispatch + weekly schedule (Wed 08:00 UTC, clear of the existing cron cluster) catch upstream drift. * pull_request with a tight paths filter (Dockerfile, .dockerignore, Makefile, the workflow itself) re-validates only when the inline recipe can actually change — the ~20-min job doesn't run on every PR. * permissions: contents:read only (the inline path pulls nothing from GHCR — that's the point); no registry login, no push, no multi-arch. * smoke step asserts the from-source caddy/crowdsec/cscli/charon binaries landed in the final image and 'caddy version' runs. Makefile: parametrize build-offline with overridable DOCKER_BUILD / BUILD_OFFLINE_ARGS so CI reuses the target (single source of truth for the two builder-src selectors) while adding buildx --platform/--load/ --cache flags. Default local behavior unchanged. docs/ci/toolchain-image.md: note the new canary in the components table. Claude-Session: https://claude.ai/code/session_01KXA4x9LrA2AsnLrvdHMZbS --- .github/workflows/build-offline.yml | 81 +++++++++++++++++++++++++++++ Makefile | 11 +++- docs/ci/toolchain-image.md | 1 + 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-offline.yml diff --git a/.github/workflows/build-offline.yml b/.github/workflows/build-offline.yml new file mode 100644 index 000000000..f87c1b5da --- /dev/null +++ b/.github/workflows/build-offline.yml @@ -0,0 +1,81 @@ +# Offline (inline-toolchain) build canary. +# +# The default app image build (docker-build.yml / nightly-build.yml) pulls the +# prebuilt `ghcr.io/wikid82/charon-toolchain` image and `COPY --from`s the custom +# Caddy + CrowdSec binaries out of it. This workflow exercises the OTHER path: +# `CADDY_BUILDER_SRC=caddy-inline` / `CROWDSEC_BUILDER_SRC=crowdsec-inline`, which +# compiles both binaries from source inside the app Dockerfile. That path is what +# a fork PR, an air-gapped operator, or anyone not logged in to GHCR falls back +# to, and nothing else in CI builds it end to end — so it can rot silently +# (a broken `go get` pin, a stage rename, a base-image bump) until someone +# actually tries to build offline. +# +# Why amd64-only + advisory (NOT a required check): +# * The inline compile is ~14 min on top of the normal build (Makefile notes +# this); a full run is ~20 min warm, more cold. Too slow to gate every PR. +# * arm64 would run the two Go compiles under QEMU emulation (45-90+ min) for +# no extra signal — the recipe is arch-independent in practice. +# * It is a canary: a weekly schedule catches upstream drift, and a tight +# `paths:` filter re-validates it whenever the inline recipe itself can +# change. Failures open a normal red check on matching PRs but do not block +# merge (this context is deliberately absent from the branch ruleset). + +name: Build (offline / inline toolchain) + +on: + workflow_dispatch: + schedule: + # Wednesday 08:00 UTC — a low-traffic slot clear of toolchain-image.yml + # (daily 06:00), security-weekly-rebuild.yml (Tue 12:00) and the Monday + # security/codeql/semgrep cluster. + - cron: '0 8 * * 3' + pull_request: + paths: + # Keep this minimal: only files that can change what the caddy-inline / + # crowdsec-inline stages actually build. Those stages take no repo build + # context (only `COPY --from=xx`), so the recipe lives entirely in the + # Dockerfile plus the build-context shape (.dockerignore). + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - '.github/workflows/build-offline.yml' + +concurrency: + group: build-offline-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-offline: + name: Build offline image (amd64, from-source Caddy + CrowdSec) + runs-on: ubuntu-latest + # ~14 min inline compile + normal build; generous headroom for a cold cache. + timeout-minutes: 35 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - name: Build charon:offline (inline toolchain path) + run: | + set -euo pipefail + make build-offline \ + DOCKER_BUILD="docker buildx build" \ + BUILD_OFFLINE_ARGS="--platform linux/amd64 --load --pull --cache-from type=gha,scope=build-offline --cache-to type=gha,mode=max,scope=build-offline" + + - name: Smoke-check the from-source binaries landed in the image + run: | + set -euo pipefail + docker run --rm --entrypoint sh charon:offline -c ' + set -e + test -x /usr/bin/caddy + test -x /usr/local/bin/crowdsec + test -x /usr/local/bin/cscli + test -x /app/charon + # Functional check on the from-source Caddy (safe, no server start). + /usr/bin/caddy version + ' diff --git a/Makefile b/Makefile index ab443c043..67347ad3b 100644 --- a/Makefile +++ b/Makefile @@ -108,10 +108,19 @@ docker-build-versioned: # Build the image WITHOUT pulling the prebuilt toolchain image — compiles the # custom Caddy + CrowdSec binaries from source (caddy-inline / crowdsec-inline). # Use offline / air-gapped, or when not logged in to GHCR. Slow (~14 min extra). +# +# The two --build-arg selectors below are the single source of truth for "build +# the inline path". Overridable knobs (used by .github/workflows/build-offline.yml): +# DOCKER_BUILD - builder command (default "docker build"; CI passes +# "docker buildx build" for GHA layer caching) +# BUILD_OFFLINE_ARGS - extra flags (e.g. --platform, --load, --cache-from/to) +DOCKER_BUILD ?= docker build +BUILD_OFFLINE_ARGS ?= build-offline: - docker build \ + $(DOCKER_BUILD) \ --build-arg CADDY_BUILDER_SRC=caddy-inline \ --build-arg CROWDSEC_BUILDER_SRC=crowdsec-inline \ + $(BUILD_OFFLINE_ARGS) \ -t charon:offline \ . diff --git a/docs/ci/toolchain-image.md b/docs/ci/toolchain-image.md index bed05d24e..3d6c728ea 100644 --- a/docs/ci/toolchain-image.md +++ b/docs/ci/toolchain-image.md @@ -19,6 +19,7 @@ every CI image build. | `scripts/lib/dockerfile-stage.sh` | Shared `extract_stage` used by both scripts. | | `.github/workflows/toolchain-image.yml` | Builds / publishes / scans. | | `.github/workflows/security-weekly-rebuild.yml` | `workflow_call`s the above for the Tuesday full rebuild + MEDIUM/LOW JSON report. | +| `.github/workflows/build-offline.yml` | Advisory canary (weekly + `Dockerfile`/`Makefile` PR paths, amd64-only). Builds the app image via the `caddy-inline` / `crowdsec-inline` path — i.e. `make build-offline` — so the fork/air-gapped fallback can't rot unnoticed. Not a required check. | ## Determinism