diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 88850147f0..c139208de6 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -93,6 +93,9 @@ runs: # refresh-flux-ghcr-auth.sh uses the host API for a secret-safe, # per-node RegistryAuthConfig patch and exact image-pull proof. shell: bash + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-talosctl.sh diff --git a/.github/actions/deploy-prod/publish-platform-manifests/action.yml b/.github/actions/deploy-prod/publish-platform-manifests/action.yml index 3d0e556ae4..0723a01461 100644 --- a/.github/actions/deploy-prod/publish-platform-manifests/action.yml +++ b/.github/actions/deploy-prod/publish-platform-manifests/action.yml @@ -49,6 +49,9 @@ runs: - name: 🔐 Install verified supply-chain tools id: setup_supply_chain_tools shell: bash + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: .github/scripts/setup-supply-chain-tools.sh - name: 🏷️ Create unique staging reference diff --git a/.github/scripts/release-asset-download.sh b/.github/scripts/release-asset-download.sh new file mode 100644 index 0000000000..2bf3a26654 --- /dev/null +++ b/.github/scripts/release-asset-download.sh @@ -0,0 +1,94 @@ +# shellcheck shell=bash +# Download one GitHub release asset, preferring the authenticated asset API (#5942). +# +# WHY THIS EXISTS +# The setup helpers used to fetch every release asset from the public +# `https://github.com///releases/download/...` URL with no credentials. +# Hosted runners share their outbound addresses, so those requests count as +# anonymous traffic. On 2026-11-09 a GitHub incident that slowed anonymous traffic +# made every one of those downloads return 504 until curl gave up, and the whole +# platform delivery path stopped before any check of our own ran. +# +# When GITHUB_TOKEN is set, this asks the release-asset API for the same bytes +# with the job's token first, and falls back to the anonymous URL if that fails. +# Without a token it goes straight to the anonymous URL, exactly as before. +# +# 🔴 THIS CHOOSES WHERE THE BYTES COME FROM, NEVER WHETHER THEY ARE TRUSTED. +# Every caller still checks the downloaded file against its reviewed SHA-256 pin +# (or its published digests) before installing or running anything. Neither path +# is more trusted than the other: both serve the same release asset. +# +# Each call prints ONE line to stderr per attempt, with fixed fields only: +# release-asset asset= path= status= result= +# The status is curl's final HTTP code (000 when no response arrived); `unlisted` +# means the release metadata was read but did not list the asset exactly once. The token, +# request headers and response bodies are never printed. curl does not forward the +# Authorization header when the API redirects to the storage host. +# +# Every local is `rad_`-prefixed: this file is SOURCED, and a caller's readonly +# variable of the same name would otherwise abort it under `set -e`. + +# Print a validated three-digit HTTP code, or 000 for anything else. +release_asset_status() { # raw + if [[ "${1-}" =~ ^[0-9]{3}$ ]]; then + printf '%s' "$1" + else + printf '000' + fi +} + +release_asset_log() { # asset path status result + printf 'release-asset asset=%s path=%s status=%s result=%s\n' "$1" "$2" "$(release_asset_status "$3")" "$4" >&2 +} + +# release_asset_download OWNER/REPO TAG ASSET OUTPUT +# Write ASSET from release TAG of OWNER/REPO to OUTPUT. Returns non-zero, with +# OUTPUT removed, when neither path served it. +release_asset_download() { + local rad_repo="${1:?owner/repo required}" rad_tag="${2:?tag required}" + local rad_asset="${3:?asset name required}" rad_output="${4:?output path required}" + local rad_status rad_metadata rad_id + # Transient failures (timeouts, 408, 429 and 5xx) are retried a few times on each + # path (#4549), capped so a long server-requested wait cannot hold the job. + local -a rad_curl=(curl --fail --silent --show-error --location + --proto '=https' --proto-redir '=https' + --retry 5 --retry-delay 3 --retry-max-time 60) + + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + local -a rad_auth=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28") + rad_metadata="$(mktemp "${RUNNER_TEMP:-/tmp}/release-asset.XXXXXX")" + rad_id='' + if rad_status="$("${rad_curl[@]}" "${rad_auth[@]}" -H 'Accept: application/vnd.github+json' \ + --write-out '%{http_code}' --output "${rad_metadata}" \ + "https://api.github.com/repos/${rad_repo}/releases/tags/${rad_tag}")"; then + rad_id="$(jq -r --arg asset "${rad_asset}" \ + '[.assets[]? | select(.name == $asset) | .id | select(type == "number")] | if length == 1 then .[0] else empty end' \ + "${rad_metadata}" 2>/dev/null)" || rad_id='' + fi + rm -f -- "${rad_metadata}" + if [[ "${rad_id}" =~ ^[0-9]+$ ]]; then + if rad_status="$("${rad_curl[@]}" "${rad_auth[@]}" -H 'Accept: application/octet-stream' \ + --write-out '%{http_code}' --output "${rad_output}" \ + "https://api.github.com/repos/${rad_repo}/releases/assets/${rad_id}")"; then + release_asset_log "${rad_asset}" authenticated "${rad_status}" served + return 0 + fi + release_asset_log "${rad_asset}" authenticated "${rad_status:-000}" failed + elif [[ "$(release_asset_status "${rad_status:-}")" == 2* ]]; then + # The release was read but does not list this asset exactly once. + release_asset_log "${rad_asset}" authenticated "${rad_status}" unlisted + else + release_asset_log "${rad_asset}" authenticated "${rad_status:-000}" failed + fi + rm -f -- "${rad_output}" + fi + + if rad_status="$("${rad_curl[@]}" --write-out '%{http_code}' --output "${rad_output}" \ + "https://github.com/${rad_repo}/releases/download/${rad_tag}/${rad_asset}")"; then + release_asset_log "${rad_asset}" anonymous "${rad_status}" served + return 0 + fi + release_asset_log "${rad_asset}" anonymous "${rad_status:-000}" failed + rm -f -- "${rad_output}" + return 1 +} diff --git a/.github/scripts/setup-ksail.sh b/.github/scripts/setup-ksail.sh index 114e3715f0..d1930b578c 100755 --- a/.github/scripts/setup-ksail.sh +++ b/.github/scripts/setup-ksail.sh @@ -7,7 +7,6 @@ if [ -z "${KSAIL_VERSION:-}" ]; then fi asset_name="ksail_${KSAIL_VERSION}_linux_amd64.tar.gz" -release_base="https://github.com/devantler-tech/ksail/releases/download/v${KSAIL_VERSION}" api_url="https://api.github.com/repos/devantler-tech/ksail/releases/tags/v${KSAIL_VERSION}" tarball="${RUNNER_TEMP:-/tmp}/${asset_name}" checksums="${RUNNER_TEMP:-/tmp}/ksail_${KSAIL_VERSION}_checksums.txt" @@ -22,13 +21,17 @@ fi # because one momentary error from the release host would otherwise fail the job and, # in a merge-group deploy, evict the pull request (#4549). Every byte is still checked # against both digest sources below, and a persistent failure still stops here. The -# total is capped so a long server-requested wait cannot hold the job. +# total is capped so a long server-requested wait cannot hold the job. The two release +# assets use the authenticated asset API first when GITHUB_TOKEN is set, and fall back +# to the public URL (#5942). download() { curl -fsSL --retry 5 --retry-delay 3 --retry-max-time 60 "$@" } +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" -download "${release_base}/${asset_name}" -o "${tarball}" -download "${release_base}/ksail_${KSAIL_VERSION}_checksums.txt" -o "${checksums}" +release_asset_download devantler-tech/ksail "v${KSAIL_VERSION}" "${asset_name}" "${tarball}" +release_asset_download devantler-tech/ksail "v${KSAIL_VERSION}" "ksail_${KSAIL_VERSION}_checksums.txt" "${checksums}" download "${curl_headers[@]}" "${api_url}" -o "${release_json}" manifest_digest=$(awk -v asset="${asset_name}" '$2 == asset {print $1}' "${checksums}") diff --git a/.github/scripts/setup-supply-chain-tools.sh b/.github/scripts/setup-supply-chain-tools.sh index 378327c429..d43908cb19 100755 --- a/.github/scripts/setup-supply-chain-tools.sh +++ b/.github/scripts/setup-supply-chain-tools.sh @@ -41,17 +41,17 @@ verify_digest() { } # Release downloads retry curl's transient failures (timeouts, 408, 429 and 5xx) a few times, -# because one momentary error from the release host would otherwise fail the job. Every byte -# is still checked against the reviewed pin below, and a persistent failure still stops here. -download() { - curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ - --retry 5 --retry-delay 3 "$1" --output "$2" -} +# because one momentary error from the release host would otherwise fail the job. With +# GITHUB_TOKEN set they use the authenticated asset API first and fall back to the public +# URL (#5942). Every byte is still checked against the reviewed pin below, and a +# persistent failure still stops here. +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" if [[ ${1:-} == trivy-boundary ]]; then : "${RUNNER_TEMP:?RUNNER_TEMP must name the runner temporary directory}" trivy_dir="${RUNNER_TEMP}/platform-validation-tools" - download "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ + release_asset_download aquasecurity/trivy "v${TRIVY_VERSION}" "trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ "${download_dir}/trivy.tar.gz" verify_digest trivy "${TRIVY_SHA256}" "${download_dir}/trivy.tar.gz" tar -xzf "${download_dir}/trivy.tar.gz" -C "${download_dir}" trivy @@ -62,11 +62,11 @@ if [[ ${1:-} == trivy-boundary ]]; then exit 0 fi -download "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-amd64" \ +release_asset_download sigstore/cosign "v${COSIGN_VERSION}" cosign-linux-amd64 \ "${download_dir}/cosign" verify_digest cosign "${COSIGN_SHA256}" "${download_dir}/cosign" -download "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/syft_${SYFT_VERSION}_linux_amd64.tar.gz" \ +release_asset_download anchore/syft "v${SYFT_VERSION}" "syft_${SYFT_VERSION}_linux_amd64.tar.gz" \ "${download_dir}/syft.tar.gz" verify_digest syft "${SYFT_SHA256}" "${download_dir}/syft.tar.gz" diff --git a/.github/scripts/setup-talosctl.sh b/.github/scripts/setup-talosctl.sh index 7fd03d4095..ce96474f3e 100755 --- a/.github/scripts/setup-talosctl.sh +++ b/.github/scripts/setup-talosctl.sh @@ -35,9 +35,12 @@ trap 'rm -f "${target}"' EXIT # The download retries curl's transient failures (timeouts, 408, 429 and 5xx) a few # times, because one momentary error from the release host would otherwise fail the job -# (#4549). The bytes are still checked against the pin above, and a persistent failure -# still stops here. -curl -fsSL --retry 5 --retry-delay 3 --retry-max-time 60 "${release_base}/${asset_name}" -o "${target}" +# (#4549). With GITHUB_TOKEN set it uses the authenticated asset API first and falls +# back to the public URL (#5942). The bytes are still checked against the pin above, +# and a persistent failure still stops here. +# shellcheck source=.github/scripts/release-asset-download.sh +source "$(dirname "${BASH_SOURCE[0]}")/release-asset-download.sh" +release_asset_download siderolabs/talos "v${TALOS_VERSION}" "${asset_name}" "${target}" actual_digest=$(sha256sum "${target}" | cut -d' ' -f1) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2583626c4..4970911369 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,6 +52,9 @@ jobs: shellcheck scripts/tests/test-webapp-compiled-scan.sh bash scripts/tests/test-webapp-compiled-scan.sh - name: 🔏 Install reviewed public-image verification tools + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh - name: Validate the hosted runtime script run: | @@ -245,6 +248,8 @@ jobs: - 'scripts/tests/test-setup-talosctl.sh' - '.github/scripts/setup-supply-chain-tools.sh' - 'scripts/tests/test-setup-supply-chain-tools.sh' + - '.github/scripts/release-asset-download.sh' + - 'scripts/tests/test-release-asset-download.sh' - '.github/actions/deploy-prod/action.yml' - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/validate-image-verifier-liveness.yaml' @@ -844,6 +849,7 @@ jobs: - 'ksail.prod.yaml' - 'ksail.yaml' - '.github/scripts/setup-talosctl.sh' + - '.github/scripts/release-asset-download.sh' - 'scripts/validate-talos-kubernetes-compatibility/**' - '.github/actions/deploy-prod/**' - 'go.mod' @@ -2252,7 +2258,7 @@ jobs: - name: 🔐 Validate KSail installer integrity guard if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-ksail.sh scripts/tests/test-setup-ksail.sh + shellcheck -x .github/scripts/setup-ksail.sh scripts/tests/test-setup-ksail.sh bash scripts/tests/test-setup-ksail.sh - name: 🔐 Validate Cilium SNI hostname coverage @@ -2264,17 +2270,26 @@ jobs: - name: 🔐 Validate talosctl installer integrity guard if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-talosctl.sh scripts/tests/test-setup-talosctl.sh + shellcheck -x .github/scripts/setup-talosctl.sh scripts/tests/test-setup-talosctl.sh bash scripts/tests/test-setup-talosctl.sh - name: 🔐 Validate supply-chain tool integrity if: needs.changes.outputs.k8s == 'true' run: | - shellcheck .github/scripts/setup-supply-chain-tools.sh scripts/tests/test-setup-supply-chain-tools.sh + shellcheck -x .github/scripts/setup-supply-chain-tools.sh scripts/tests/test-setup-supply-chain-tools.sh bash scripts/tests/test-setup-supply-chain-tools.sh + - name: 🔐 Validate the shared release-asset download + if: needs.changes.outputs.k8s == 'true' + run: | + shellcheck .github/scripts/release-asset-download.sh scripts/tests/test-release-asset-download.sh + bash scripts/tests/test-release-asset-download.sh + - name: 🔐 Exercise verified supply-chain tools if: needs.changes.outputs.k8s == 'true' + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-supply-chain-tools.sh /usr/local/bin/syft scan dir:. --output cyclonedx-json=tool-integrity.cdx.json @@ -2325,6 +2340,9 @@ jobs: - name: Install checksum-verified Trivy for disposition boundaries if: needs.changes.outputs.k8s == 'true' + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh trivy-boundary - name: 🧱 Validate vendored-operator disposition boundary @@ -3870,6 +3888,9 @@ jobs: persist-credentials: false - name: ⚙️ Setup talosctl + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: | .github/scripts/setup-talosctl.sh diff --git a/.github/workflows/publish-dex-security-image.yaml b/.github/workflows/publish-dex-security-image.yaml index b5948bd18d..7bbc435150 100644 --- a/.github/workflows/publish-dex-security-image.yaml +++ b/.github/workflows/publish-dex-security-image.yaml @@ -10,6 +10,7 @@ on: - go.mod - go.sum - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/bases/infrastructure/controllers/dex/DexSecurity.Dockerfile - k8s/bases/infrastructure/controllers/dex/dex-v2.46.1-network.patch - k8s/bases/infrastructure/controllers/dex/IMAGE-NOTICE @@ -22,6 +23,7 @@ on: - go.mod - go.sum - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/bases/infrastructure/controllers/dex/DexSecurity.Dockerfile - k8s/bases/infrastructure/controllers/dex/dex-v2.46.1-network.patch - k8s/bases/infrastructure/controllers/dex/IMAGE-NOTICE diff --git a/.github/workflows/publish-longhorn-v1-retirement.yaml b/.github/workflows/publish-longhorn-v1-retirement.yaml index c0f57da6dc..4923ccd161 100644 --- a/.github/workflows/publish-longhorn-v1-retirement.yaml +++ b/.github/workflows/publish-longhorn-v1-retirement.yaml @@ -7,6 +7,7 @@ name: Publish Longhorn V1 Retirement Image paths: - .github/workflows/publish-longhorn-v1-retirement.yaml - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/providers/hetzner/infrastructure/controllers/longhorn/InstanceManagerRetirement.Dockerfile - k8s/providers/hetzner/infrastructure/controllers/longhorn/instance-manager-v1.13.0-retirement.patch push: @@ -14,6 +15,7 @@ name: Publish Longhorn V1 Retirement Image paths: - .github/workflows/publish-longhorn-v1-retirement.yaml - .github/scripts/setup-supply-chain-tools.sh + - .github/scripts/release-asset-download.sh - k8s/providers/hetzner/infrastructure/controllers/longhorn/InstanceManagerRetirement.Dockerfile - k8s/providers/hetzner/infrastructure/controllers/longhorn/instance-manager-v1.13.0-retirement.patch diff --git a/.github/workflows/validate-main.yaml b/.github/workflows/validate-main.yaml index 4273e4c202..86ffb21df9 100644 --- a/.github/workflows/validate-main.yaml +++ b/.github/workflows/validate-main.yaml @@ -76,6 +76,9 @@ jobs: shellcheck scripts/tests/test-webapp-compiled-scan.sh bash scripts/tests/test-webapp-compiled-scan.sh - name: 🔏 Install reviewed public-image verification tools + env: + # Download through the authenticated asset API first (#5942). + GITHUB_TOKEN: ${{ github.token }} run: bash .github/scripts/setup-supply-chain-tools.sh - name: Validate the hosted runtime script run: | diff --git a/scripts/tests/test-release-asset-download.sh b/scripts/tests/test-release-asset-download.sh new file mode 100755 index 0000000000..39e1de7570 --- /dev/null +++ b/scripts/tests/test-release-asset-download.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Offline tests for .github/scripts/release-asset-download.sh (#5942). +# +# curl is replaced by a fake that serves fixtures by URL and records every +# request, so each case asserts which path served the asset, what the log line +# says, and that the token never reaches the log or the anonymous request. +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel) +lib="${repo_root}/.github/scripts/release-asset-download.sh" +test_root=$(mktemp -d "${TMPDIR:-/tmp}/test-release-asset-download.XXXXXX") +trap 'rm -rf "${test_root}"' EXIT +mkdir -p "${test_root}/bin" "${test_root}/fixtures" "${test_root}/runner" +export TEST_ROOT="${test_root}" + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +printf 'release bytes\n' >"${test_root}/fixtures/asset" +printf '{"assets":[{"name":"other.tar.gz","id":11},{"name":"tool-linux-amd64","id":42}]}\n' \ + >"${test_root}/fixtures/release.json" +printf '{"assets":[{"name":"other.tar.gz","id":11}]}\n' >"${test_root}/fixtures/release-unlisted.json" + +# The fake answers per URL. TEST_FAIL names the requests that fail with a 504: +# a space-separated list of `metadata`, `asset-api` and `anonymous`. +cat >"${test_root}/bin/curl" <<'FAKE_CURL' +#!/usr/bin/env bash +set -euo pipefail +url='' output='' write_out='' retry=0 auth=no +while (($#)); do + case "$1" in + --output | -o) output=$2; shift 2 ;; + --write-out | -w) write_out=$2; shift 2 ;; + --retry) retry=$2; shift 2 ;; + --retry-delay | --retry-max-time | --proto | --proto-redir) shift 2 ;; + -H) + [[ $2 != Authorization:* ]] || auth=yes + shift 2 + ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +case "${url}" in + https://api.github.com/repos/example/tool/releases/tags/v1.2.3) kind=metadata ;; + https://api.github.com/repos/example/tool/releases/assets/42) kind=asset-api ;; + https://github.com/example/tool/releases/download/v1.2.3/tool-linux-amd64) kind=anonymous ;; + *) printf 'unexpected URL: %s\n' "${url}" >&2; exit 2 ;; +esac +printf '%s auth=%s retry=%s\n' "${kind}" "${auth}" "${retry}" >>"${TEST_ROOT}/requests" +code() { [[ ${write_out} != '%{http_code}' ]] || printf '%s' "$1"; } +if [[ " ${TEST_FAIL:-} " == *" ${kind} "* ]]; then + # Like curl --fail: a partial body may be left behind, and the code is still written. + printf 'partial' >"${output}" + code 504 + printf 'curl: (22) The requested URL returned error: 504\n' >&2 + exit 22 +fi +case "${kind}" in + metadata) cp "${TEST_ROOT}/fixtures/${TEST_RELEASE_JSON:-release.json}" "${output}" ;; + *) cp "${TEST_ROOT}/fixtures/asset" "${output}" ;; +esac +code 200 +FAKE_CURL +chmod +x "${test_root}/bin/curl" + +token='test-token-value-never-logged' +out="${test_root}/runner/asset" + +# run TOKEN — downloads the fixture asset with the given GITHUB_TOKEN ('' = none). +run() { + rm -f "${test_root}/requests" "${out}" + status=0 + log=$( + PATH="${test_root}/bin:${PATH}" RUNNER_TEMP="${test_root}/runner" GITHUB_TOKEN="$1" \ + bash -c 'set -euo pipefail; source "$1"; release_asset_download example/tool v1.2.3 tool-linux-amd64 "$2"' \ + _ "${lib}" "${out}" 2>&1 + ) || status=$? + [[ ${log} != *"${token}"* ]] || fail "the token reached the log: ${log}" + # The anonymous URL must never carry the token. + if [[ -e ${test_root}/requests ]] && grep -q '^anonymous auth=yes' "${test_root}/requests"; then + fail 'the anonymous request carried the Authorization header' + fi + # Every request retries transient failures (#4549). + if [[ -e ${test_root}/requests ]] && grep -qv 'retry=[1-9]' "${test_root}/requests"; then + fail "a request does not retry transient failures: $(cat "${test_root}/requests")" + fi + [[ -z $(find "${test_root}/runner" -name 'release-asset.*' -print -quit) ]] || + fail 'the metadata scratch file was left behind' +} + +requests() { tr '\n' ' ' <"${test_root}/requests" | sed 's/ $//'; } + +assert_served() { # path + [[ ${status} == 0 ]] || fail "expected success, got ${status}: ${log}" + cmp -s "${test_root}/fixtures/asset" "${out}" || fail 'the served bytes are not the asset' + [[ ${log} == *"path=$1 status=200 result=served"* ]] || fail "expected the $1 path to serve it: ${log}" +} + +# --- 1. Authenticated success: the asset API serves it, the public URL is never asked. +run "${token}" +assert_served authenticated +[[ $(requests) == 'metadata auth=yes retry=5 asset-api auth=yes retry=5' ]] || + fail "unexpected requests for authenticated success: $(requests)" +[[ $(grep -c '^release-asset ' <<<"${log}") == 1 ]] || fail "expected exactly one log line: ${log}" + +# --- 2. Authenticated download fails, anonymous succeeds. +TEST_FAIL='asset-api' run "${token}" +assert_served anonymous +[[ ${log} == *'release-asset asset=tool-linux-amd64 path=authenticated status=504 result=failed'* ]] || + fail "the authenticated failure was not logged with its status: ${log}" +[[ $(requests) == 'metadata auth=yes retry=5 asset-api auth=yes retry=5 anonymous auth=no retry=5' ]] || + fail "unexpected requests for the fallback: $(requests)" + +# --- 3. Authenticated metadata read fails, anonymous succeeds without asking for the asset. +TEST_FAIL='metadata' run "${token}" +assert_served anonymous +[[ ${log} == *'path=authenticated status=504 result=failed'* ]] || fail "metadata failure not logged: ${log}" +[[ $(requests) == 'metadata auth=yes retry=5 anonymous auth=no retry=5' ]] || + fail "unexpected requests after a metadata failure: $(requests)" + +# --- 4. The release does not list the asset: logged as unlisted, then anonymous. +TEST_RELEASE_JSON='release-unlisted.json' run "${token}" +assert_served anonymous +[[ ${log} == *'path=authenticated status=200 result=unlisted'* ]] || fail "unlisted asset not logged: ${log}" + +# --- 5. Both paths fail: the call fails, names both, and leaves no partial file. +TEST_FAIL='asset-api anonymous' run "${token}" +[[ ${status} != 0 ]] || fail 'a download neither path served was reported as success' +[[ ! -e ${out} ]] || fail 'a failed download left a partial file behind' +[[ ${log} == *'path=authenticated status=504 result=failed'* ]] || fail "authenticated failure not logged: ${log}" +[[ ${log} == *'path=anonymous status=504 result=failed'* ]] || fail "anonymous failure not logged: ${log}" + +# --- 6. No token: straight to the public URL, nothing sent to the API. +run '' +assert_served anonymous +[[ $(requests) == 'anonymous auth=no retry=5' ]] || fail "a tokenless download asked the API: $(requests)" + +# --- 7. No token and the public URL fails: the call fails. +TEST_FAIL='anonymous' run '' +[[ ${status} != 0 ]] || fail 'a failed tokenless download was reported as success' +[[ ! -e ${out} ]] || fail 'a failed tokenless download left a partial file behind' + +# --- 8. Every helper that pulls a GitHub release asset sources and calls this +# function. Each helper's own test asserts which URLs it requests. +expected_source="source \"\$(dirname \"\${BASH_SOURCE[0]}\")/release-asset-download.sh\"" +for helper in setup-talosctl.sh setup-supply-chain-tools.sh setup-ksail.sh; do + path="${repo_root}/.github/scripts/${helper}" + grep -Fqx "${expected_source}" "${path}" || + fail "${helper} does not source the shared download function" + grep -q '^release_asset_download ' "${path}" || fail "${helper} does not call the shared download function" +done + +printf 'release-asset download tests passed\n' diff --git a/scripts/tests/test-setup-ksail.sh b/scripts/tests/test-setup-ksail.sh index cc2631cfc6..703d4e6d47 100755 --- a/scripts/tests/test-setup-ksail.sh +++ b/scripts/tests/test-setup-ksail.sh @@ -15,7 +15,7 @@ chmod +x "${fixture_dir}/ksail" tar -czf "${fixture_dir}/ksail.tar.gz" -C "${fixture_dir}" ksail digest=$(sha256sum "${fixture_dir}/ksail.tar.gz" | cut -d' ' -f1) -printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","digest":"sha256:%s"}]}\n' \ +printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","id":1,"digest":"sha256:%s"},{"name":"ksail_7.178.25_checksums.txt","id":2}]}\n' \ "${digest}" >"${fixture_dir}/release.json" printf '%s ksail_7x178x25_linux_amd64.tar.gz\n' "${digest}" >"${fixture_dir}/checksums.txt" @@ -32,10 +32,10 @@ while [ "$#" -gt 0 ]; do retry=$2 shift 2 ;; - --retry-delay | --retry-max-time) + --retry-delay | --retry-max-time | --proto | --proto-redir | --write-out) shift 2 ;; - -o) + -o | --output) output=$2 shift 2 ;; @@ -56,13 +56,13 @@ done printf '%s %s\n' "${retry}" "${url}" >>"${TEST_RETRY_LOG}" case "${url}" in - */ksail_7.178.25_linux_amd64.tar.gz) + */ksail_7.178.25_linux_amd64.tar.gz | */releases/assets/1) cp "${TEST_FIXTURE_DIR}/ksail.tar.gz" "${output}" ;; */releases/tags/v7.178.25) cp "${TEST_FIXTURE_DIR}/release.json" "${output}" ;; - */ksail_7.178.25_checksums.txt) + */ksail_7.178.25_checksums.txt | */releases/assets/2) cp "${TEST_FIXTURE_DIR}/checksums.txt" "${output}" ;; *) @@ -158,12 +158,32 @@ if [ ! -e "${test_root}/authenticated" ]; then exit 1 fi +# With a token, both release assets come from the authenticated asset API (#5942). +for asset in ksail_7.178.25_linux_amd64.tar.gz ksail_7.178.25_checksums.txt; do + case "${output}" in + *"release-asset asset=${asset} path=authenticated status="*" result=served"*) ;; + *) + printf 'setup-ksail.sh did not download %s through the authenticated asset API:\n%s\n' "${asset}" "${output}" >&2 + exit 1 + ;; + esac +done + +# A release that does not list the assets falls back to the public URL, and the +# digest checks still decide the install. printf '%s ksail_7.178.25_linux_amd64.tar.gz\n' "${digest}" >"${fixture_dir}/checksums.txt" printf '{"assets":[{"name":"ksail_7.178.25_linux_amd64.tar.gz","digest":"sha256:%064d"}]}\n' \ 0 >"${fixture_dir}/release.json" run_installer assert_rejected_before_install 'release metadata digest mismatch for ksail_7.178.25_linux_amd64.tar.gz' +case "${output}" in + *'release-asset asset=ksail_7.178.25_linux_amd64.tar.gz path=anonymous status='*' result=served'*) ;; + *) + printf 'setup-ksail.sh did not fall back to the public URL for an unlisted asset:\n%s\n' "${output}" >&2 + exit 1 + ;; +esac # One momentary error from the release host must not fail the job (#4549): every # download that decides the install asks curl to retry transient failures. Each @@ -172,6 +192,8 @@ assert_rejected_before_install 'release metadata digest mismatch for ksail_7.178 for url_suffix in \ /ksail_7.178.25_linux_amd64.tar.gz \ /ksail_7.178.25_checksums.txt \ + /releases/assets/1 \ + /releases/assets/2 \ /releases/tags/v7.178.25; do verdict=$(awk -v suffix="${url_suffix}" ' substr($2, length($2) - length(suffix) + 1) == suffix { seen = 1; if ($1 !~ /^[1-9][0-9]*$/) bad = 1 } diff --git a/scripts/tests/test-setup-supply-chain-tools.sh b/scripts/tests/test-setup-supply-chain-tools.sh index 5c812b0fbc..9b63cc77ba 100755 --- a/scripts/tests/test-setup-supply-chain-tools.sh +++ b/scripts/tests/test-setup-supply-chain-tools.sh @@ -39,7 +39,7 @@ while (( $# )); do case "$1" in -o|--output) output=$2; shift 2 ;; --retry) retry=$2; shift 2 ;; - --proto|--proto-redir|--retry-delay|--retry-max-time) shift 2 ;; + --proto|--proto-redir|--retry-delay|--retry-max-time|--write-out) shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac @@ -93,6 +93,8 @@ make_variant() { -e "s|/usr/local/bin/cosign version|${test_root}/installed/cosign version|" \ -e "s|/usr/local/bin/syft version|${test_root}/installed/syft version|" \ "${installer}" >"${test_root}/installer.sh" + # The installer sources its download function from beside itself. + cp "${repo_root}/.github/scripts/release-asset-download.sh" "${test_root}/" for assignment in 'COSIGN_VERSION="9.9.9"' 'SYFT_VERSION="9.9.9"' \ "COSIGN_SHA256=\"$1\"" "SYFT_SHA256=\"$2\""; do grep -Fxq "${assignment}" "${test_root}/installer.sh" || fail "fixture did not substitute ${assignment}" @@ -104,7 +106,7 @@ run_installer() { rm -f "${test_root}/installs" "${test_root}/executed" "${test_root}/downloads" "${test_root}/retries" "${test_root}/github-path" "${test_root}/installed/"* status=0 output=$(PATH="${test_root}/bin:${test_root}/installed:${PATH}" \ - RUNNER_TEMP="${test_root}/runner" GITHUB_PATH="${test_root}/github-path" \ + RUNNER_TEMP="${test_root}/runner" GITHUB_PATH="${test_root}/github-path" GITHUB_TOKEN="" \ bash "${test_root}/installer.sh" "$@" 2>&1) || status=$? [[ -z $(find "${test_root}/runner" -name 'supply-chain-tools.*' -print -quit) ]] || fail 'installer left temporary downloads behind' } diff --git a/scripts/tests/test-setup-talosctl.sh b/scripts/tests/test-setup-talosctl.sh index 2b5d825c96..4ce8e9c447 100755 --- a/scripts/tests/test-setup-talosctl.sh +++ b/scripts/tests/test-setup-talosctl.sh @@ -31,11 +31,13 @@ set -euo pipefail output='' url='' retry=0 +write_out='' while [ "$#" -gt 0 ]; do case "$1" in - -o) output=$2; shift 2 ;; + -o | --output) output=$2; shift 2 ;; --retry) retry=$2; shift 2 ;; - --retry-delay | --retry-max-time) shift 2 ;; + --retry-delay | --retry-max-time | --proto | --proto-redir) shift 2 ;; + --write-out) write_out=$2; shift 2 ;; -*) shift ;; *) url=$1; shift ;; esac @@ -44,6 +46,7 @@ printf '%s %s\n' "${retry}" "${url}" >>"${TEST_RETRY_LOG}" case "${url}" in */talosctl-linux-amd64) cp "${TEST_FIXTURE_DIR}/talosctl" "${output}" + [ -z "${write_out}" ] || printf 200 ;; */sha256sum.txt) if [ "${TEST_MANIFEST_REACHABLE}" != '1' ]; then @@ -79,6 +82,8 @@ chmod +x "${fake_bin}/curl" "${fake_bin}/sudo" "${fake_bin}/talosctl" make_variant() { pinned_digest=$1 variant="${test_root}/setup-talosctl-variant.sh" + # The installer sources its download function from beside itself. + cp "${repo_root}/.github/scripts/release-asset-download.sh" "${test_root}/" sed \ -e "s|^TALOS_VERSION=.*|TALOS_VERSION=\"${test_version}\"|" \ -e "s|^TALOSCTL_SHA256=.*|TALOSCTL_SHA256=\"${pinned_digest}\"|" \ @@ -103,6 +108,7 @@ run_installer() { cd "${repo_root}" && PATH="${fake_bin}:${PATH}" \ RUNNER_TEMP="${runner_temp}" \ + GITHUB_TOKEN="" \ TEST_FIXTURE_DIR="${fixture_dir}" \ TEST_INSTALL_MARKER="${test_root}/installed" \ TEST_MANIFEST_REACHABLE="${manifest_reachable:-1}" \