From b430f3737c22ceb5f85eb6a7b3dfff6c35c1c94c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 02:16:07 +0100 Subject: [PATCH 1/2] fix(ci): download release assets through the authenticated API first The setup helpers fetched every GitHub release asset anonymously, so an incident that throttles anonymous traffic stopped the whole delivery path. A shared download function now tries the authenticated release-asset API with the job token, falls back to the public URL, and logs which path served each asset with its final HTTP status. Digest verification is unchanged and still happens before anything is installed or run. Part of #5942 Co-Authored-By: Claude Opus 5.5 --- .github/actions/deploy-prod/action.yml | 3 + .../publish-platform-manifests/action.yml | 3 + .github/scripts/release-asset-download.sh | 94 +++++++++++ .github/scripts/setup-ksail.sh | 11 +- .github/scripts/setup-supply-chain-tools.sh | 18 +- .github/scripts/setup-talosctl.sh | 9 +- .github/workflows/ci.yaml | 21 +++ .../workflows/publish-dex-security-image.yaml | 2 + .../publish-longhorn-v1-retirement.yaml | 2 + .github/workflows/validate-main.yaml | 3 + scripts/tests/test-release-asset-download.sh | 156 ++++++++++++++++++ scripts/tests/test-setup-ksail.sh | 32 +++- .../tests/test-setup-supply-chain-tools.sh | 6 +- scripts/tests/test-setup-talosctl.sh | 10 +- 14 files changed, 345 insertions(+), 25 deletions(-) create mode 100644 .github/scripts/release-asset-download.sh create mode 100755 scripts/tests/test-release-asset-download.sh diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 88850147f0..c139208de6 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -93,6 +93,9 @@ runs: # refresh-flux-ghcr-auth.sh uses the host API for a secret-safe, # per-node RegistryAuthConfig patch and exact image-pull proof. shell: bash + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-talosctl.sh diff --git a/.github/actions/deploy-prod/publish-platform-manifests/action.yml b/.github/actions/deploy-prod/publish-platform-manifests/action.yml index 3d0e556ae4..0723a01461 100644 --- a/.github/actions/deploy-prod/publish-platform-manifests/action.yml +++ b/.github/actions/deploy-prod/publish-platform-manifests/action.yml @@ -49,6 +49,9 @@ runs: - name: 🔐 Install verified supply-chain tools id: setup_supply_chain_tools shell: bash + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: .github/scripts/setup-supply-chain-tools.sh - name: 🏷️ Create unique staging reference diff --git a/.github/scripts/release-asset-download.sh b/.github/scripts/release-asset-download.sh new file mode 100644 index 0000000000..2bf3a26654 --- /dev/null +++ b/.github/scripts/release-asset-download.sh @@ -0,0 +1,94 @@ +# shellcheck shell=bash +# Download one GitHub release asset, preferring the authenticated asset API (#5942). +# +# WHY THIS EXISTS +# The setup helpers used to fetch every release asset from the public +# `https://github.com///releases/download/...` URL with no credentials. +# Hosted runners share their outbound addresses, so those requests count as +# anonymous traffic. On 2026-11-09 a GitHub incident that slowed anonymous traffic +# made every one of those downloads return 504 until curl gave up, and the whole +# platform delivery path stopped before any check of our own ran. +# +# When GITHUB_TOKEN is set, this asks the release-asset API for the same bytes +# with the job's token first, and falls back to the anonymous URL if that fails. +# Without a token it goes straight to the anonymous URL, exactly as before. +# +# 🔴 THIS CHOOSES WHERE THE BYTES COME FROM, NEVER WHETHER THEY ARE TRUSTED. +# Every caller still checks the downloaded file against its reviewed SHA-256 pin +# (or its published digests) before installing or running anything. Neither path +# is more trusted than the other: both serve the same release asset. +# +# Each call prints ONE line to stderr per attempt, with fixed fields only: +# release-asset asset= path= status= result= +# The status is curl's final HTTP code (000 when no response arrived); `unlisted` +# means the release metadata was read but did not list the asset exactly once. The token, +# request headers and response bodies are never printed. curl does not forward the +# Authorization header when the API redirects to the storage host. +# +# Every local is `rad_`-prefixed: this file is SOURCED, and a caller's readonly +# variable of the same name would otherwise abort it under `set -e`. + +# Print a validated three-digit HTTP code, or 000 for anything else. +release_asset_status() { # raw + if [[ "${1-}" =~ ^[0-9]{3}$ ]]; then + printf '%s' "$1" + else + printf '000' + fi +} + +release_asset_log() { # asset path status result + printf 'release-asset asset=%s path=%s status=%s result=%s\n' "$1" "$2" "$(release_asset_status "$3")" "$4" >&2 +} + +# release_asset_download OWNER/REPO TAG ASSET OUTPUT +# Write ASSET from release TAG of OWNER/REPO to OUTPUT. Returns non-zero, with +# OUTPUT removed, when neither path served it. +release_asset_download() { + local rad_repo="${1:?owner/repo required}" rad_tag="${2:?tag required}" + local rad_asset="${3:?asset name required}" rad_output="${4:?output path required}" + local rad_status rad_metadata rad_id + # Transient failures (timeouts, 408, 429 and 5xx) are retried a few times on each + # path (#4549), capped so a long server-requested wait cannot hold the job. + local -a rad_curl=(curl --fail --silent --show-error --location + --proto '=https' --proto-redir '=https' + --retry 5 --retry-delay 3 --retry-max-time 60) + + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + local -a rad_auth=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28") + rad_metadata="$(mktemp "${RUNNER_TEMP:-/tmp}/release-asset.XXXXXX")" + rad_id='' + if rad_status="$("${rad_curl[@]}" "${rad_auth[@]}" -H 'Accept: application/vnd.github+json' \ + --write-out '%{http_code}' --output "${rad_metadata}" \ + "https://api.github.com/repos/${rad_repo}/releases/tags/${rad_tag}")"; then + rad_id="$(jq -r --arg asset "${rad_asset}" \ + '[.assets[]? | select(.name == $asset) | .id | select(type == "number")] | if length == 1 then .[0] else empty end' \ + "${rad_metadata}" 2>/dev/null)" || rad_id='' + fi + rm -f -- "${rad_metadata}" + if [[ "${rad_id}" =~ ^[0-9]+$ ]]; then + if rad_status="$("${rad_curl[@]}" "${rad_auth[@]}" -H 'Accept: application/octet-stream' \ + --write-out '%{http_code}' --output "${rad_output}" \ + "https://api.github.com/repos/${rad_repo}/releases/assets/${rad_id}")"; then + release_asset_log "${rad_asset}" authenticated "${rad_status}" served + return 0 + fi + release_asset_log "${rad_asset}" authenticated "${rad_status:-000}" failed + elif [[ "$(release_asset_status "${rad_status:-}")" == 2* ]]; then + # The release was read but does not list this asset exactly once. + release_asset_log "${rad_asset}" authenticated "${rad_status}" unlisted + else + release_asset_log "${rad_asset}" authenticated "${rad_status:-000}" failed + fi + rm -f -- "${rad_output}" + fi + + if rad_status="$("${rad_curl[@]}" --write-out '%{http_code}' --output "${rad_output}" \ + "https://github.com/${rad_repo}/releases/download/${rad_tag}/${rad_asset}")"; then + release_asset_log "${rad_asset}" anonymous "${rad_status}" served + return 0 + fi + release_asset_log "${rad_asset}" anonymous "${rad_status:-000}" failed + rm -f -- "${rad_output}" + return 1 +} diff --git a/.github/scripts/setup-ksail.sh b/.github/scripts/setup-ksail.sh index 114e3715f0..d1930b578c 100755 --- a/.github/scripts/setup-ksail.sh +++ b/.github/scripts/setup-ksail.sh @@ -7,7 +7,6 @@ if [ -z "${KSAIL_VERSION:-}" ]; then fi asset_name="ksail_${KSAIL_VERSION}_linux_amd64.tar.gz" -release_base="https://github.com/devantler-tech/ksail/releases/download/v${KSAIL_VERSION}" api_url="https://api.github.com/repos/devantler-tech/ksail/releases/tags/v${KSAIL_VERSION}" tarball="${RUNNER_TEMP:-/tmp}/${asset_name}" checksums="${RUNNER_TEMP:-/tmp}/ksail_${KSAIL_VERSION}_checksums.txt" @@ -22,13 +21,17 @@ fi # because one momentary error from the release host would otherwise fail the job and, # in a merge-group deploy, evict the pull request (#4549). Every byte is still checked # against both digest sources below, and a persistent failure still stops here. The -# total is capped so a long server-requested wait cannot hold the job. +# total is capped so a long server-requested wait cannot hold the job. The two release +# assets use the authenticated asset API first when GITHUB_TOKEN is set, and fall back +# to the public URL (#5942). download() { curl -fsSL --retry 5 --retry-delay 3 --retry-max-time 60 "$@" } +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" -download "${release_base}/${asset_name}" -o "${tarball}" -download "${release_base}/ksail_${KSAIL_VERSION}_checksums.txt" -o "${checksums}" +release_asset_download devantler-tech/ksail "v${KSAIL_VERSION}" "${asset_name}" "${tarball}" +release_asset_download devantler-tech/ksail "v${KSAIL_VERSION}" "ksail_${KSAIL_VERSION}_checksums.txt" "${checksums}" download "${curl_headers[@]}" "${api_url}" -o "${release_json}" manifest_digest=$(awk -v asset="${asset_name}" '$2 == asset {print $1}' "${checksums}") diff --git a/.github/scripts/setup-supply-chain-tools.sh b/.github/scripts/setup-supply-chain-tools.sh index 378327c429..d43908cb19 100755 --- a/.github/scripts/setup-supply-chain-tools.sh +++ b/.github/scripts/setup-supply-chain-tools.sh @@ -41,17 +41,17 @@ verify_digest() { } # Release downloads retry curl's transient failures (timeouts, 408, 429 and 5xx) a few times, -# because one momentary error from the release host would otherwise fail the job. Every byte -# is still checked against the reviewed pin below, and a persistent failure still stops here. -download() { - curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ - --retry 5 --retry-delay 3 "$1" --output "$2" -} +# because one momentary error from the release host would otherwise fail the job. With +# GITHUB_TOKEN set they use the authenticated asset API first and fall back to the public +# URL (#5942). Every byte is still checked against the reviewed pin below, and a +# persistent failure still stops here. +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" if [[ ${1:-} == trivy-boundary ]]; then : "${RUNNER_TEMP:?RUNNER_TEMP must name the runner temporary directory}" trivy_dir="${RUNNER_TEMP}/platform-validation-tools" - download "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ + release_asset_download aquasecurity/trivy "v${TRIVY_VERSION}" "trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ "${download_dir}/trivy.tar.gz" verify_digest trivy "${TRIVY_SHA256}" "${download_dir}/trivy.tar.gz" tar -xzf "${download_dir}/trivy.tar.gz" -C "${download_dir}" trivy @@ -62,11 +62,11 @@ if [[ ${1:-} == trivy-boundary ]]; then exit 0 fi -download "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-amd64" \ +release_asset_download sigstore/cosign "v${COSIGN_VERSION}" cosign-linux-amd64 \ "${download_dir}/cosign" verify_digest cosign "${COSIGN_SHA256}" "${download_dir}/cosign" -download "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" \ +release_asset_download anchore/syft "v${SYFT_VERSION}" "syft_${SYFT_VERSION}_linux_amd64.tar.gz" \ "${download_dir}/syft.tar.gz" verify_digest syft "${SYFT_SHA256}" "${download_dir}/syft.tar.gz" diff --git a/.github/scripts/setup-talosctl.sh b/.github/scripts/setup-talosctl.sh index 7fd03d4095..ce96474f3e 100755 --- a/.github/scripts/setup-talosctl.sh +++ b/.github/scripts/setup-talosctl.sh @@ -35,9 +35,12 @@ trap 'rm -f "${target}"' EXIT # The download retries curl's transient failures (timeouts, 408, 429 and 5xx) a few # times, because one momentary error from the release host would otherwise fail the job -# (#4549). The bytes are still checked against the pin above, and a persistent failure -# still stops here. -curl -fsSL --retry 5 --retry-delay 3 --retry-max-time 60 "${release_base}/${asset_name}" -o "${target}" +# (#4549). With GITHUB_TOKEN set it uses the authenticated asset API first and falls +# back to the public URL (#5942). The bytes are still checked against the pin above, +# and a persistent failure still stops here. +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" +release_asset_download siderolabs/talos "v${TALOS_VERSION}" "${asset_name}" "${target}" actual_digest=$(sha256sum "${target}" | cut -d' ' -f1) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2583626c4..02ed7c1191 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,6 +52,9 @@ jobs: shellcheck scripts/tests/test-webapp-compiled-scan.sh bash scripts/tests/test-webapp-compiled-scan.sh - name: 🔏 Install reviewed public-image verification tools + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh - name: Validate the hosted runtime script run: | @@ -245,6 +248,8 @@ jobs: - 'scripts/tests/test-setup-talosctl.sh' - '.github/scripts/setup-supply-chain-tools.sh' - 'scripts/tests/test-setup-supply-chain-tools.sh' + - '.github/scripts/release-asset-download.sh' + - 'scripts/tests/test-release-asset-download.sh' - '.github/actions/deploy-prod/action.yml' - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/validate-image-verifier-liveness.yaml' @@ -844,6 +849,7 @@ jobs: - 'ksail.prod.yaml' - 'ksail.yaml' - '.github/scripts/setup-talosctl.sh' + - '.github/scripts/release-asset-download.sh' - 'scripts/validate-talos-kubernetes-compatibility/**' - '.github/actions/deploy-prod/**' - 'go.mod' @@ -2273,8 +2279,17 @@ jobs: shellcheck .github/scripts/setup-supply-chain-tools.sh scripts/tests/test-setup-supply-chain-tools.sh bash scripts/tests/test-setup-supply-chain-tools.sh + - name: 🔐 Validate the shared release-asset download + if: needs.changes.outputs.k8s == 'true' + run: | + shellcheck .github/scripts/release-asset-download.sh scripts/tests/test-release-asset-download.sh + bash scripts/tests/test-release-asset-download.sh + - name: 🔐 Exercise verified supply-chain tools if: needs.changes.outputs.k8s == 'true' + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-supply-chain-tools.sh /usr/local/bin/syft scan dir:. --output cyclonedx-json=tool-integrity.cdx.json @@ -2325,6 +2340,9 @@ jobs: - name: Install checksum-verified Trivy for disposition boundaries if: needs.changes.outputs.k8s == 'true' + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh trivy-boundary - name: 🧱 Validate vendored-operator disposition boundary @@ -3870,6 +3888,9 @@ jobs: persist-credentials: false - name: ⚙️ Setup talosctl + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-talosctl.sh diff --git a/.github/workflows/publish-dex-security-image.yaml b/.github/workflows/publish-dex-security-image.yaml index b5948bd18d..7bbc435150 100644 --- a/.github/workflows/publish-dex-security-image.yaml +++ b/.github/workflows/publish-dex-security-image.yaml @@ -10,6 +10,7 @@ on: - go.mod - go.sum - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/bases/infrastructure/controllers/dex/DexSecurity.Dockerfile - k8s/bases/infrastructure/controllers/dex/dex-v2.46.1-network.patch - k8s/bases/infrastructure/controllers/dex/IMAGE-NOTICE @@ -22,6 +23,7 @@ on: - go.mod - go.sum - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/bases/infrastructure/controllers/dex/DexSecurity.Dockerfile - k8s/bases/infrastructure/controllers/dex/dex-v2.46.1-network.patch - k8s/bases/infrastructure/controllers/dex/IMAGE-NOTICE diff --git a/.github/workflows/publish-longhorn-v1-retirement.yaml b/.github/workflows/publish-longhorn-v1-retirement.yaml index c0f57da6dc..4923ccd161 100644 --- a/.github/workflows/publish-longhorn-v1-retirement.yaml +++ b/.github/workflows/publish-longhorn-v1-retirement.yaml @@ -7,6 +7,7 @@ name: Publish Longhorn V1 Retirement Image paths: - .github/workflows/publish-longhorn-v1-retirement.yaml - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/providers/hetzner/infrastructure/controllers/longhorn/InstanceManagerRetirement.Dockerfile - k8s/providers/hetzner/infrastructure/controllers/longhorn/instance-manager-v1.13.0-retirement.patch push: @@ -14,6 +15,7 @@ name: Publish Longhorn V1 Retirement Image paths: - .github/workflows/publish-longhorn-v1-retirement.yaml - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/providers/hetzner/infrastructure/controllers/longhorn/InstanceManagerRetirement.Dockerfile - k8s/providers/hetzner/infrastructure/controllers/longhorn/instance-manager-v1.13.0-retirement.patch diff --git a/.github/workflows/validate-main.yaml b/.github/workflows/validate-main.yaml index 4273e4c202..86ffb21df9 100644 --- a/.github/workflows/validate-main.yaml +++ b/.github/workflows/validate-main.yaml @@ -76,6 +76,9 @@ jobs: shellcheck scripts/tests/test-webapp-compiled-scan.sh bash scripts/tests/test-webapp-compiled-scan.sh - name: 🔏 Install reviewed public-image verification tools + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh - name: Validate the hosted runtime script run: | diff --git a/scripts/tests/test-release-asset-download.sh b/scripts/tests/test-release-asset-download.sh new file mode 100755 index 0000000000..39e1de7570 --- /dev/null +++ b/scripts/tests/test-release-asset-download.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Offline tests for .github/scripts/release-asset-download.sh (#5942). +# +# curl is replaced by a fake that serves fixtures by URL and records every +# request, so each case asserts which path served the asset, what the log line +# says, and that the token never reaches the log or the anonymous request. +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel) +lib="${repo_root}/.github/scripts/release-asset-download.sh" +test_root=$(mktemp -d "${TMPDIR:-/tmp}/test-release-asset-download.XXXXXX") +trap 'rm -rf "${test_root}"' EXIT +mkdir -p "${test_root}/bin" "${test_root}/fixtures" "${test_root}/runner" +export TEST_ROOT="${test_root}" + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +printf 'release bytes\n' >"${test_root}/fixtures/asset" +printf '{"assets":[{"name":"other.tar.gz","id":11},{"name":"tool-linux-amd64","id":42}]}\n' \ + >"${test_root}/fixtures/release.json" +printf '{"assets":[{"name":"other.tar.gz","id":11}]}\n' >"${test_root}/fixtures/release-unlisted.json" + +# The fake answers per URL. TEST_FAIL names the requests that fail with a 504: +# a space-separated list of `metadata`, `asset-api` and `anonymous`. +cat >"${test_root}/bin/curl" <<'FAKE_CURL' +#!/usr/bin/env bash +set -euo pipefail +url='' output='' write_out='' retry=0 auth=no +while (($#)); do + case "$1" in + --output | -o) output=$2; shift 2 ;; + --write-out | -w) write_out=$2; shift 2 ;; + --retry) retry=$2; shift 2 ;; + --retry-delay | --retry-max-time | --proto | --proto-redir) shift 2 ;; + -H) + [[ $2 != Authorization:* ]] || auth=yes + shift 2 + ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +case "${url}" in + https://api.github.com/repos/example/tool/releases/tags/v1.2.3) kind=metadata ;; + https://api.github.com/repos/example/tool/releases/assets/42) kind=asset-api ;; + https://github.com/example/tool/releases/download/v1.2.3/tool-linux-amd64) kind=anonymous ;; + *) printf 'unexpected URL: %s\n' "${url}" >&2; exit 2 ;; +esac +printf '%s auth=%s retry=%s\n' "${kind}" "${auth}" "${retry}" >>"${TEST_ROOT}/requests" +code() { [[ ${write_out} != '%{http_code}' ]] || printf '%s' "$1"; } +if [[ " ${TEST_FAIL:-} " == *" ${kind} "* ]]; then + # Like curl --fail: a partial body may be left behind, and the code is still written. + printf 'partial' >"${output}" + code 504 + printf 'curl: (22) The requested URL returned error: 504\n' >&2 + exit 22 +fi +case "${kind}" in + metadata) cp "${TEST_ROOT}/fixtures/${TEST_RELEASE_JSON:-release.json}" "${output}" ;; + *) cp "${TEST_ROOT}/fixtures/asset" "${output}" ;; +esac +code 200 +FAKE_CURL +chmod +x "${test_root}/bin/curl" + +token='test-token-value-never-logged' +out="${test_root}/runner/asset" + +# run TOKEN — downloads the fixture asset with the given GITHUB_TOKEN ('' = none). +run() { + rm -f "${test_root}/requests" "${out}" + status=0 + log=$( + PATH="${test_root}/bin:${PATH}" RUNNER_TEMP="${test_root}/runner" GITHUB_TOKEN="$1" \ + bash -c 'set -euo pipefail; source "$1"; release_asset_download example/tool v1.2.3 tool-linux-amd64 "$2"' \ + _ "${lib}" "${out}" 2>&1 + ) || status=$? + [[ ${log} != *"${token}"* ]] || fail "the token reached the log: ${log}" + # The anonymous URL must never carry the token. + if [[ -e ${test_root}/requests ]] && grep -q '^anonymous auth=yes' "${test_root}/requests"; then + fail 'the anonymous request carried the Authorization header' + fi + # Every request retries transient failures (#4549). + if [[ -e ${test_root}/requests ]] && grep -qv 'retry=[1-9]' "${test_root}/requests"; then + fail "a request does not retry transient failures: $(cat "${test_root}/requests")" + fi + [[ -z $(find "${test_root}/runner" -name 'release-asset.*' -print -quit) ]] || + fail 'the metadata scratch file was left behind' +} + +requests() { tr '\n' ' ' <"${test_root}/requests" | sed 's/ $//'; } + +assert_served() { # path + [[ ${status} == 0 ]] || fail "expected success, got ${status}: ${log}" + cmp -s "${test_root}/fixtures/asset" "${out}" || fail 'the served bytes are not the asset' + [[ ${log} == *"path=$1 status=200 result=served"* ]] || fail "expected the $1 path to serve it: ${log}" +} + +# --- 1. Authenticated success: the asset API serves it, the public URL is never asked. +run "${token}" +assert_served authenticated +[[ $(requests) == 'metadata auth=yes retry=5 asset-api auth=yes retry=5' ]] || + fail "unexpected requests for authenticated success: $(requests)" +[[ $(grep -c '^release-asset ' <<<"${log}") == 1 ]] || fail "expected exactly one log line: ${log}" + +# --- 2. Authenticated download fails, anonymous succeeds. +TEST_FAIL='asset-api' run "${token}" +assert_served anonymous +[[ ${log} == *'release-asset asset=tool-linux-amd64 path=authenticated status=504 result=failed'* ]] || + fail "the authenticated failure was not logged with its status: ${log}" +[[ $(requests) == 'metadata auth=yes retry=5 asset-api auth=yes retry=5 anonymous auth=no retry=5' ]] || + fail "unexpected requests for the fallback: $(requests)" + +# --- 3. Authenticated metadata read fails, anonymous succeeds without asking for the asset. +TEST_FAIL='metadata' run "${token}" +assert_served anonymous +[[ ${log} == *'path=authenticated status=504 result=failed'* ]] || fail "metadata failure not logged: ${log}" +[[ $(requests) == 'metadata auth=yes retry=5 anonymous auth=no retry=5' ]] || + fail "unexpected requests after a metadata failure: $(requests)" + +# --- 4. The release does not list the asset: logged as unlisted, then anonymous. +TEST_RELEASE_JSON='release-unlisted.json' run "${token}" +assert_served anonymous +[[ ${log} == *'path=authenticated status=200 result=unlisted'* ]] || fail "unlisted asset not logged: ${log}" + +# --- 5. Both paths fail: the call fails, names both, and leaves no partial file. +TEST_FAIL='asset-api anonymous' run "${token}" +[[ ${status} != 0 ]] || fail 'a download neither path served was reported as success' +[[ ! -e ${out} ]] || fail 'a failed download left a partial file behind' +[[ ${log} == *'path=authenticated status=504 result=failed'* ]] || fail "authenticated failure not logged: ${log}" +[[ ${log} == *'path=anonymous status=504 result=failed'* ]] || fail "anonymous failure not logged: ${log}" + +# --- 6. No token: straight to the public URL, nothing sent to the API. +run '' +assert_served anonymous +[[ $(requests) == 'anonymous auth=no retry=5' ]] || fail "a tokenless download asked the API: $(requests)" + +# --- 7. No token and the public URL fails: the call fails. +TEST_FAIL='anonymous' run '' +[[ ${status} != 0 ]] || fail 'a failed tokenless download was reported as success' +[[ ! -e ${out} ]] || fail 'a failed tokenless download left a partial file behind' + +# --- 8. Every helper that pulls a GitHub release asset sources and calls this +# function. Each helper's own test asserts which URLs it requests. +expected_source="source \"\$(dirname \"\${BASH_SOURCE[0]}\")/release-asset-download.sh\"" +for helper in setup-talosctl.sh setup-supply-chain-tools.sh setup-ksail.sh; do + path="${repo_root}/.github/scripts/${helper}" + grep -Fqx "${expected_source}" "${path}" || + fail "${helper} does not source the shared download function" + grep -q '^release_asset_download ' "${path}" || fail "${helper} does not call the shared download function" +done + +printf 'release-asset download tests passed\n' diff --git a/scripts/tests/test-setup-ksail.sh b/scripts/tests/test-setup-ksail.sh index cc2631cfc6..703d4e6d47 100755 --- a/scripts/tests/test-setup-ksail.sh +++ b/scripts/tests/test-setup-ksail.sh @@ -15,7 +15,7 @@ chmod +x "${fixture_dir}/ksail" tar -czf "${fixture_dir}/ksail.tar.gz" -C "${fixture_dir}" ksail digest=$(sha256sum "${fixture_dir}/ksail.tar.gz" | cut -d' ' -f1) -printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","digest":"sha256:%s"}]}\n' \ +printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","id":1,"digest":"sha256:%s"},{"name":"ksail_7.178.25_checksums.txt","id":2}]}\n' \ "${digest}" >"${fixture_dir}/release.json" printf '%s ksail_7x178x25_linux_amd64.tar.gz\n' "${digest}" >"${fixture_dir}/checksums.txt" @@ -32,10 +32,10 @@ while [ "$#" -gt 0 ]; do retry=$2 shift 2 ;; - --retry-delay | --retry-max-time) + --retry-delay | --retry-max-time | --proto | --proto-redir | --write-out) shift 2 ;; - -o) + -o | --output) output=$2 shift 2 ;; @@ -56,13 +56,13 @@ done printf '%s %s\n' "${retry}" "${url}" >>"${TEST_RETRY_LOG}" case "${url}" in - */ksail_7.178.25_linux_amd64.tar.gz) + */ksail_7.178.25_linux_amd64.tar.gz | */releases/assets/1) cp "${TEST_FIXTURE_DIR}/ksail.tar.gz" "${output}" ;; */releases/tags/v7.178.25) cp "${TEST_FIXTURE_DIR}/release.json" "${output}" ;; - */ksail_7.178.25_checksums.txt) + */ksail_7.178.25_checksums.txt | */releases/assets/2) cp "${TEST_FIXTURE_DIR}/checksums.txt" "${output}" ;; *) @@ -158,12 +158,32 @@ if [ ! -e "${test_root}/authenticated" ]; then exit 1 fi +# With a token, both release assets come from the authenticated asset API (#5942). +for asset in ksail_7.178.25_linux_amd64.tar.gz ksail_7.178.25_checksums.txt; do + case "${output}" in + *"release-asset asset=${asset} path=authenticated status="*" result=served"*) ;; + *) + printf 'setup-ksail.sh did not download %s through the authenticated asset API:\n%s\n' "${asset}" "${output}" >&2 + exit 1 + ;; + esac +done + +# A release that does not list the assets falls back to the public URL, and the +# digest checks still decide the install. printf '%s ksail_7.178.25_linux_amd64.tar.gz\n' "${digest}" >"${fixture_dir}/checksums.txt" printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","digest":"sha256:%064d"}]}\n' \ 0 >"${fixture_dir}/release.json" run_installer assert_rejected_before_install 'release metadata digest mismatch for ksail_7.178.25_linux_amd64.tar.gz' +case "${output}" in + *'release-asset asset=ksail_7.178.25_linux_amd64.tar.gz path=anonymous status='*' result=served'*) ;; + *) + printf 'setup-ksail.sh did not fall back to the public URL for an unlisted asset:\n%s\n' "${output}" >&2 + exit 1 + ;; +esac # One momentary error from the release host must not fail the job (#4549): every # download that decides the install asks curl to retry transient failures. Each @@ -172,6 +192,8 @@ assert_rejected_before_install 'release metadata digest mismatch for ksail_7.178 for url_suffix in \ /ksail_7.178.25_linux_amd64.tar.gz \ /ksail_7.178.25_checksums.txt \ + /releases/assets/1 \ + /releases/assets/2 \ /releases/tags/v7.178.25; do verdict=$(awk -v suffix="${url_suffix}" ' substr($2, length($2) - length(suffix) + 1) == suffix { seen = 1; if ($1 !~ /^[1-9][0-9]*$/) bad = 1 } diff --git a/scripts/tests/test-setup-supply-chain-tools.sh b/scripts/tests/test-setup-supply-chain-tools.sh index 5c812b0fbc..9b63cc77ba 100755 --- a/scripts/tests/test-setup-supply-chain-tools.sh +++ b/scripts/tests/test-setup-supply-chain-tools.sh @@ -39,7 +39,7 @@ while (( $# )); do case "$1" in -o|--output) output=$2; shift 2 ;; --retry) retry=$2; shift 2 ;; - --proto|--proto-redir|--retry-delay|--retry-max-time) shift 2 ;; + --proto|--proto-redir|--retry-delay|--retry-max-time|--write-out) shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac @@ -93,6 +93,8 @@ make_variant() { -e "s|/usr/local/bin/cosign version|${test_root}/installed/cosign version|" \ -e "s|/usr/local/bin/syft version|${test_root}/installed/syft version|" \ "${installer}" >"${test_root}/installer.sh" + # The installer sources its download function from beside itself. + cp "${repo_root}/.github/scripts/release-asset-download.sh" "${test_root}/" for assignment in 'COSIGN_VERSION="9.9.9"' 'SYFT_VERSION="9.9.9"' \ "COSIGN_SHA256=\"$1\"" "SYFT_SHA256=\"$2\""; do grep -Fxq "${assignment}" "${test_root}/installer.sh" || fail "fixture did not substitute ${assignment}" @@ -104,7 +106,7 @@ run_installer() { rm -f "${test_root}/installs" "${test_root}/executed" "${test_root}/downloads" "${test_root}/retries" "${test_root}/github-path" "${test_root}/installed/"* status=0 output=$(PATH="${test_root}/bin:${test_root}/installed:${PATH}" \ - RUNNER_TEMP="${test_root}/runner" GITHUB_PATH="${test_root}/github-path" \ + RUNNER_TEMP="${test_root}/runner" GITHUB_PATH="${test_root}/github-path" GITHUB_TOKEN="" \ bash "${test_root}/installer.sh" "$@" 2>&1) || status=$? [[ -z $(find "${test_root}/runner" -name 'supply-chain-tools.*' -print -quit) ]] || fail 'installer left temporary downloads behind' } diff --git a/scripts/tests/test-setup-talosctl.sh b/scripts/tests/test-setup-talosctl.sh index 2b5d825c96..4ce8e9c447 100755 --- a/scripts/tests/test-setup-talosctl.sh +++ b/scripts/tests/test-setup-talosctl.sh @@ -31,11 +31,13 @@ set -euo pipefail output='' url='' retry=0 +write_out='' while [ "$#" -gt 0 ]; do case "$1" in - -o) output=$2; shift 2 ;; + -o | --output) output=$2; shift 2 ;; --retry) retry=$2; shift 2 ;; - --retry-delay | --retry-max-time) shift 2 ;; + --retry-delay | --retry-max-time | --proto | --proto-redir) shift 2 ;; + --write-out) write_out=$2; shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac @@ -44,6 +46,7 @@ printf '%s %s\n' "${retry}" "${url}" >>"${TEST_RETRY_LOG}" case "${url}" in */talosctl-linux-amd64) cp "${TEST_FIXTURE_DIR}/talosctl" "${output}" + [ -z "${write_out}" ] || printf 200 ;; */sha256sum.txt) if [ "${TEST_MANIFEST_REACHABLE}" != '1' ]; then @@ -79,6 +82,8 @@ chmod +x "${fake_bin}/curl" "${fake_bin}/sudo" "${fake_bin}/talosctl" make_variant() { pinned_digest=$1 variant="${test_root}/setup-talosctl-variant.sh" + # The installer sources its download function from beside itself. + cp "${repo_root}/.github/scripts/release-asset-download.sh" "${test_root}/" sed \ -e "s|^TALOS_VERSION=.*|TALOS_VERSION=\"${test_version}\"|" \ -e "s|^TALOSCTL_SHA256=.*|TALOSCTL_SHA256=\"${pinned_digest}\"|" \ @@ -103,6 +108,7 @@ run_installer() { cd "${repo_root}" && PATH="${fake_bin}:${PATH}" \ RUNNER_TEMP="${runner_temp}" \ + GITHUB_TOKEN="" \ TEST_FIXTURE_DIR="${fixture_dir}" \ TEST_INSTALL_MARKER="${test_root}/installed" \ TEST_MANIFEST_REACHABLE="${manifest_reachable:-1}" \ From a3483a8fbc9936f1aad549d878f0ec8401ce6144 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 03:04:30 +0100 Subject: [PATCH 2/2] fix(ci): let shellcheck follow the helpers' shared download source The three installer-integrity steps ran shellcheck without -x, so the new sourced download function was reported as an unfollowed source. Part of #5942 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02ed7c1191..4970911369 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2258,7 +2258,7 @@ jobs: - name: 🔐 Validate KSail installer integrity guard if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-ksail.sh scripts/tests/test-setup-ksail.sh + shellcheck -x .github/scripts/setup-ksail.sh scripts/tests/test-setup-ksail.sh bash scripts/tests/test-setup-ksail.sh - name: 🔐 Validate Cilium SNI hostname coverage @@ -2270,13 +2270,13 @@ jobs: - name: 🔐 Validate talosctl installer integrity guard if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-talosctl.sh scripts/tests/test-setup-talosctl.sh + shellcheck -x .github/scripts/setup-talosctl.sh scripts/tests/test-setup-talosctl.sh bash scripts/tests/test-setup-talosctl.sh - name: 🔐 Validate supply-chain tool integrity if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-supply-chain-tools.sh scripts/tests/test-setup-supply-chain-tools.sh + shellcheck -x .github/scripts/setup-supply-chain-tools.sh scripts/tests/test-setup-supply-chain-tools.sh bash scripts/tests/test-setup-supply-chain-tools.sh - name: 🔐 Validate the shared release-asset download