chore: propagate changes from main into development

.github/workflows/build-offline.yml

@@ -0,0 +1,81 @@
+# Offline (inline-toolchain) build canary.
+#
+# The default app image build (docker-build.yml / nightly-build.yml) pulls the
+# prebuilt `ghcr.io/wikid82/charon-toolchain` image and `COPY --from`s the custom
+# Caddy + CrowdSec binaries out of it. This workflow exercises the OTHER path:
+# `CADDY_BUILDER_SRC=caddy-inline` / `CROWDSEC_BUILDER_SRC=crowdsec-inline`, which
+# compiles both binaries from source inside the app Dockerfile. That path is what
+# a fork PR, an air-gapped operator, or anyone not logged in to GHCR falls back
+# to, and nothing else in CI builds it end to end — so it can rot silently
+# (a broken `go get` pin, a stage rename, a base-image bump) until someone
+# actually tries to build offline.
+#
+# Why amd64-only + advisory (NOT a required check):
+#   * The inline compile is ~14 min on top of the normal build (Makefile notes
+#     this); a full run is ~20 min warm, more cold. Too slow to gate every PR.
+#   * arm64 would run the two Go compiles under QEMU emulation (45-90+ min) for
+#     no extra signal — the recipe is arch-independent in practice.
+#   * It is a canary: a weekly schedule catches upstream drift, and a tight
+#     `paths:` filter re-validates it whenever the inline recipe itself can
+#     change. Failures open a normal red check on matching PRs but do not block
+#     merge (this context is deliberately absent from the branch ruleset).
+
+name: Build (offline / inline toolchain)
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Wednesday 08:00 UTC — a low-traffic slot clear of toolchain-image.yml
+    # (daily 06:00), security-weekly-rebuild.yml (Tue 12:00) and the Monday
+    # security/codeql/semgrep cluster.
+    - cron: '0 8 * * 3'
+  pull_request:
+    paths:
+      # Keep this minimal: only files that can change what the caddy-inline /
+      # crowdsec-inline stages actually build. Those stages take no repo build
+      # context (only `COPY --from=xx`), so the recipe lives entirely in the
+      # Dockerfile plus the build-context shape (.dockerignore).
+      - 'Dockerfile'
+      - '.dockerignore'
+      - 'Makefile'
+      - '.github/workflows/build-offline.yml'
+
+concurrency:
+  group: build-offline-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-offline:
+    name: Build offline image (amd64, from-source Caddy + CrowdSec)
+    runs-on: ubuntu-latest
+    # ~14 min inline compile + normal build; generous headroom for a cold cache.
+    timeout-minutes: 35
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Build charon:offline (inline toolchain path)
+        run: |
+          set -euo pipefail
+          make build-offline \
+            DOCKER_BUILD="docker buildx build" \
+            BUILD_OFFLINE_ARGS="--platform linux/amd64 --load --pull --cache-from type=gha,scope=build-offline --cache-to type=gha,mode=max,scope=build-offline"
+
+      - name: Smoke-check the from-source binaries landed in the image
+        run: |
+          set -euo pipefail
+          docker run --rm --entrypoint sh charon:offline -c '
+            set -e
+            test -x /usr/bin/caddy
+            test -x /usr/local/bin/crowdsec
+            test -x /usr/local/bin/cscli
+            test -x /app/charon
+            # Functional check on the from-source Caddy (safe, no server start).
+            /usr/bin/caddy version
+          '

.github/workflows/quality-checks.yml

@@ -109,6 +109,11 @@ jobs:
 
       - name: Install regctl
         uses: iarekylew00t/regctl-installer@c2202c17a65fe59371c71ecc169c9e58c3710a15 # v4.0.16
+        with:
+          # Pin the regctl binary to an explicit released version (not the
+          # action's `latest` default) so this required check is reproducible
+          # and can't shift underneath us when a new regctl ships.
+          regctl-release: v0.11.6
 
       - name: Verify the pinned toolchain image matches the recipe
         env:

Makefile

@@ -108,10 +108,19 @@ docker-build-versioned:
 # Build the image WITHOUT pulling the prebuilt toolchain image — compiles the
 # custom Caddy + CrowdSec binaries from source (caddy-inline / crowdsec-inline).
 # Use offline / air-gapped, or when not logged in to GHCR. Slow (~14 min extra).
+#
+# The two --build-arg selectors below are the single source of truth for "build
+# the inline path". Overridable knobs (used by .github/workflows/build-offline.yml):
+#   DOCKER_BUILD        - builder command (default "docker build"; CI passes
+#                         "docker buildx build" for GHA layer caching)
+#   BUILD_OFFLINE_ARGS  - extra flags (e.g. --platform, --load, --cache-from/to)
+DOCKER_BUILD ?= docker build
+BUILD_OFFLINE_ARGS ?=
 build-offline:
-	docker build \
+	$(DOCKER_BUILD) \
 		--build-arg CADDY_BUILDER_SRC=caddy-inline \
 		--build-arg CROWDSEC_BUILDER_SRC=crowdsec-inline \
+		$(BUILD_OFFLINE_ARGS) \
 		-t charon:offline \
 		.
 

docs/ci/toolchain-image.md

@@ -19,6 +19,7 @@ every CI image build.
 | `scripts/lib/dockerfile-stage.sh` | Shared `extract_stage` used by both scripts. |
 | `.github/workflows/toolchain-image.yml` | Builds / publishes / scans. |
 | `.github/workflows/security-weekly-rebuild.yml` | `workflow_call`s the above for the Tuesday full rebuild + MEDIUM/LOW JSON report. |
+| `.github/workflows/build-offline.yml` | Advisory canary (weekly + `Dockerfile`/`Makefile` PR paths, amd64-only). Builds the app image via the `caddy-inline` / `crowdsec-inline` path — i.e. `make build-offline` — so the fork/air-gapped fallback can't rot unnoticed. Not a required check. |
 
 ## Determinism
 

scripts/tests/helpers/toolchain_fixture.bash

@@ -44,11 +44,13 @@ tf_write_dockerfile() {
   local geoip2_version="${TF_GEOIP2_VERSION:-v0.0.0-20260623062220-3675c6e7e63d}"
   local caddy_get_line="${TF_CADDY_GET_LINE:-    _retry go get golang.org/x/net@v0.58.0; \\}"
   local golang_digest="${TF_GOLANG_DIGEST:-sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125}"
+  local alpine_image="${TF_ALPINE_IMAGE:-alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b}"
+  local xx_pin="${TF_XX_PIN:-tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707}"
 
   cat > "$TF_DF" <<EOF
 # syntax=docker/dockerfile:1
 ARG GO_VERSION=1.27.1
-ARG ALPINE_IMAGE=alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
+ARG ALPINE_IMAGE=${alpine_image}
 ARG CROWDSEC_VERSION=1.8.1
 ARG EXPR_LANG_VERSION=1.17.8
 ARG XNET_VERSION=0.58.0
@@ -69,7 +71,7 @@ ARG CHARON_TOOLCHAIN_IMAGE=ghcr.io/wikid82/charon-toolchain
 ARG CHARON_TOOLCHAIN_TAG=caddy-crowdsec-0000000000000000
 ARG CHARON_TOOLCHAIN_DIGEST=sha256:0000000000000000000000000000000000000000000000000000000000000000
 
-FROM --platform=\$BUILDPLATFORM tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707 AS xx
+FROM --platform=\$BUILDPLATFORM ${xx_pin} AS xx
 
 # renovate: datasource=docker depName=golang
 FROM --platform=\$BUILDPLATFORM golang:\${GO_VERSION}-alpine@${golang_digest} AS caddy-inline
@@ -149,6 +151,21 @@ EOF
   chmod +x "$TF_BIN/regctl"
 }
 
+# Install a fake `regctl` on PATH whose `image digest` call FAILS (non-zero,
+# no stdout) — simulates the pinned `:$KEY` tag not resolving in the registry.
+# `registry login` still succeeds so the script reaches the digest lookup.
+tf_stub_regctl_unresolvable() {
+  cat > "$TF_BIN/regctl" <<'EOF'
+#!/usr/bin/env bash
+case "$1 $2" in
+  "registry login") exit 0 ;;
+  "image digest")   echo "regctl: manifest unknown" >&2; exit 1 ;;
+  *) exit 0 ;;
+esac
+EOF
+  chmod +x "$TF_BIN/regctl"
+}
+
 # A minimal PATH that contains coreutils + the fixture stub dir but is
 # guaranteed NOT to contain a system `regctl`. Use for "regctl absent" tests:
 #   PATH="$(tf_min_path)" run bash "$TF_ROOT/scripts/verify-toolchain-pin.sh"

scripts/tests/toolchain-key.bats

@@ -6,6 +6,8 @@
 #   * changes when a `go get` line INSIDE caddy-inline changes
 #   * changes when a tracked ARG default (CADDY_VERSION / CADDY_GEOIP2_VERSION) moves
 #   * changes when the digest-pinned golang base moves
+#   * changes when the tonistiigi/xx pin moves
+#   * changes when the ALPINE_IMAGE pin moves
 #   * changes when .trivyignore changes
 #   * fails loudly if stage extraction breaks
 
@@ -70,6 +72,32 @@ key_of() { bash "$TF_ROOT/scripts/toolchain-key.sh" "$TF_DF"; }
   [ "$before" != "$after" ]
 }
 
+@test "changes when the tonistiigi/xx pin moves (N4)" {
+  before="$(key_of)"
+  export TF_XX_PIN='tonistiigi/xx:1.9.1@sha256:1111111111111111111111111111111111111111111111111111111111111111'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" != "$after" ]
+}
+
+@test "changes when the ALPINE_IMAGE pin moves" {
+  before="$(key_of)"
+  export TF_ALPINE_IMAGE='alpine:3.25.0@sha256:2222222222222222222222222222222222222222222222222222222222222222'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" != "$after" ]
+}
+
+@test "stable when the tonistiigi/xx and ALPINE_IMAGE pins are re-emitted unchanged" {
+  before="$(key_of)"
+  # Re-write the fixture with the exact same (default) xx / alpine pins.
+  export TF_XX_PIN='tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707'
+  export TF_ALPINE_IMAGE='alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b'
+  tf_write_dockerfile
+  after="$(key_of)"
+  [ "$before" = "$after" ]
+}
+
 @test "changes when .trivyignore changes" {
   before="$(key_of)"
   printf 'another-cve\n' >> "$TF_ROOT/.trivyignore"

scripts/tests/verify-toolchain-pin.bats

@@ -6,6 +6,8 @@
 #   mismatched tag                               -> exit 1 (actionable)
 #   same-repo + regctl absent                    -> exit 1
 #   same-repo + GHCR_READ_TOKEN unset            -> exit 1
+#   same-repo + :$KEY tag does not resolve       -> exit 1
+#   same-repo + CHARON_TOOLCHAIN_DIGEST empty    -> exit 1
 #   same-repo + GHCR digest != pinned digest     -> exit 1
 #   same-repo + GHCR digest == pinned digest     -> exit 0
 
@@ -74,6 +76,40 @@ verify() { bash "$TF_ROOT/scripts/verify-toolchain-pin.sh" "$TF_DF"; }
   [[ "$output" == *"hand-edited or stale"* ]]
 }
 
+@test "same-repo PR + pinned :\$KEY tag does not resolve in GHCR: exit 1 (failure-closed)" {
+  tf_stub_regctl_unresolvable
+  export GITHUB_EVENT_NAME=pull_request
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"does not resolve in GHCR"* ]]
+}
+
+@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST value blank: exit 1 (failure-closed)" {
+  tf_stub_regctl "$GOOD_DIGEST"
+  sed -i "s|^ARG CHARON_TOOLCHAIN_DIGEST=.*|ARG CHARON_TOOLCHAIN_DIGEST=|" "$TF_DF"
+  export GITHUB_EVENT_NAME=pull_request
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]]
+}
+
+@test "same-repo PR + CHARON_TOOLCHAIN_DIGEST ARG line absent: exit 1 (failure-closed)" {
+  tf_stub_regctl "$GOOD_DIGEST"
+  sed -i "/^ARG CHARON_TOOLCHAIN_DIGEST=/d" "$TF_DF"
+  export GITHUB_EVENT_NAME=push
+  export GITHUB_REPOSITORY=wikid82/Charon
+  export GHCR_READ_TOKEN=tok
+  run verify
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"CHARON_TOOLCHAIN_DIGEST not found"* ]]
+}
+
 @test "same-repo PR + GHCR digest matches the pinned digest: exit 0" {
   tf_stub_regctl "$GOOD_DIGEST"
   export GITHUB_EVENT_NAME=pull_request

scripts/verify-toolchain-pin.sh

@@ -43,8 +43,8 @@ if [[ ! -f "$df" ]]; then
   exit 2
 fi
 
-arg_value() { # $1 = ARG name
-  grep -E "^ARG $1=" "$df" | head -n1 | cut -d= -f2-
+arg_value() { # $1 = ARG name; empty output (never a hard error) when absent
+  { grep -E "^ARG $1=" "$df" || true; } | head -n1 | cut -d= -f2-
 }
 
 KEY="$("$here/toolchain-key.sh" "$df")"