From 9c9981e1e51df0efba9607407ba8189ad19af250 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:00:33 +0100 Subject: [PATCH] =?UTF-8?q?feat(ci):=20detect=20lockfile=20version=20drift?= =?UTF-8?q?=20=E2=80=94=20the=20estate's=20recurring=20CI=20killer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE FAULT. Dependabot bumps an action version in a workflow file. Nobody regenerates .github/workflows/actions.lock. The workflow then requests a version the lockfile does not record, and GitHub rejects the run at startup_failure with ZERO jobs -- no log, no annotation, nothing in REST or GraphQL. It is SELF-REINFLICTING: any repo with dependabot AND a lockfile AND no regeneration step re-acquires the fault on every bump. Measured 2026-08-25: 60 of 60 sampled repos carry both. That is why repos appear to "go bad again" after being fixed -- nobody broke them, the clock did. PROVEN on hypatia#723. Three drifted entries (docker/setup-buildx-action, github/codeql-action, taiki-e/install-action) discriminated PERFECTLY: every workflow using one was dead, every workflow using none was alive. Regeneration took all four dead workflows from startup_failure to running, and CodeQL to green. Ships: * scripts/check-lockfile-drift.sh -- per-repo checker, TSV output * .github/workflows/lockfile-drift-detect.yml -- weekly estate sweep, Tuesday 07:20 UTC (after dependabot's window, so a bump and its drift are caught the same week) REPORT-ONLY by design. It opens/updates one tracking issue and never mutates another repository, per the estate guardrail that unattended cross-repo mutation is a human decision. It follows the one findings-> issue model in the estate that actually works (farm/drift-detect.yml). Verified both ways, because a check that cannot fail is not a check: * against hypatia POST-fix -> reports only a genuine residual SHA/tag mismatch in tests.yml (confirmed real: workflow pins 34e11487, lockfile records v7.0.1 -> 3d3c42e5; different commits) * against hypatia PRE-fix -> finds all three known drifted actions across the correct six workflows * against a lockfile-free repo -> exits 0, out of scope (absence of a lockfile is failure mode 1, not drift) * dogfooded against standards itself -> clean, 41 workflows Calibrated honestly: a hit is a genuine inconsistency but NOT proof the workflow is dead -- tests.yml starts despite its residual mismatch. The reliable direction is the converse: workflows that were dead had drifted entries. Two self-inflicted defects caught in review and fixed here, both worth noting because the estate has been bitten by each before: * the issue-body heredoc sat at column 0, terminating the enclosing `run: |` block scalar -- the exact defect that made .git-private-farm/inbox-steward-propagate.yml unparseable and blocked gh actions-lock from running at all * `[ test ] && cmd` under `set -e` exits the run when the test is false, and false was the DEFAULT path Lockfile entry added for the new workflow; without it, it would die at 0s -- the very fault it exists to detect. Co-Authored-By: Claude Opus 5 --- .github/workflows/actions.lock | 3 + .github/workflows/lockfile-drift-detect.yml | 179 ++++++++++++++++++++ scripts/check-lockfile-drift.sh | 125 ++++++++++++++ 3 files changed, 307 insertions(+) create mode 100644 .github/workflows/lockfile-drift-detect.yml create mode 100755 scripts/check-lockfile-drift.sh diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 8e1be585..fc6e0913 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -61,6 +61,9 @@ workflows: - 'peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697' '.github/workflows/launcher-standard-lockstep.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + '.github/workflows/lockfile-drift-detect.yml': + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' '.github/workflows/makefile-blocker.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' '.github/workflows/mirror-reusable.yml': diff --git a/.github/workflows/lockfile-drift-detect.yml b/.github/workflows/lockfile-drift-detect.yml new file mode 100644 index 00000000..0785ccbc --- /dev/null +++ b/.github/workflows/lockfile-drift-detect.yml @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell +# +# lockfile-drift-detect.yml — estate-wide sweep for the recurring CI killer. +# +# THE FAULT: Dependabot bumps an action version in a workflow; nobody +# regenerates `.github/workflows/actions.lock`; the workflow then requests a +# version the lockfile does not record, and GitHub rejects the run at +# `startup_failure` with ZERO jobs — no log, no annotation, nothing in REST. +# +# It is SELF-REINFLICTING. Measured 2026-08-25: 60 of 60 sampled repos carry +# both a lockfile and a Dependabot config, so the whole estate re-acquires this +# on every action bump. That is why repos appear to "go bad again" after being +# fixed — nobody broke them, the clock did. +# +# Proven on hypatia#723: three drifted entries discriminated perfectly (every +# workflow using one was dead, every workflow using none was alive), and +# regeneration took all four dead workflows from `startup_failure` to running. +# +# This workflow REPORTS ONLY. It opens/updates one tracking issue and never +# mutates another repository — consistent with the estate guardrail that +# unattended cross-repo mutation is a human decision. +name: Lockfile Drift Detect + +on: + schedule: + # Tuesday 07:20 UTC — after Dependabot's usual weekly window, so a bump + # and its resulting drift are caught in the same week rather than the next. + - cron: '20 7 * * 2' + workflow_dispatch: + inputs: + limit: + description: 'Max repos to scan (0 = all)' + type: string + default: '0' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + issues: write + +jobs: + sweep: + name: Sweep the estate for lockfile drift + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout standards (for the checker script) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - name: Sweep + id: sweep + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LIMIT: ${{ inputs.limit || '0' }} + run: | + set -euo pipefail + : > drift-report.tsv + printf 'repo\tworkflow\trequested\tlocked\n' >> drift-report.tsv + + # Enumerate, then filter. `gh repo list` has a denominator we can + # state; `gh search` does not. A count landing on a round number is a + # page size, not a measurement. + gh repo list hyperpolymath --limit 1000 --no-archived \ + --json name --jq '.[].name' > repos.txt + TOTAL=$(wc -l < repos.txt) + echo "enumerated $TOTAL non-archived repos" + # NOT `[ test ] && cmd` — under `set -e` a false test makes the whole + # compound non-zero and kills the run, and 0 is the DEFAULT. + if [ "$LIMIT" != "0" ]; then + head -n "$LIMIT" repos.txt > r.tmp + mv r.tmp repos.txt + fi + + scanned=0; withlock=0; drifted=0 + while read -r r; do + scanned=$((scanned+1)) + # Cheap probe first: skip repos with no lockfile (that is failure + # mode 1, not drift, and a lockfile-free repo may be legitimately + # outside the enforcement cohort). + gh api "repos/hyperpolymath/$r/contents/.github/workflows/actions.lock" \ + --jq .size >/dev/null 2>&1 || continue + withlock=$((withlock+1)) + + rm -rf _w + if ! git clone -q --depth 1 \ + "https://x-access-token:${GH_TOKEN}@github.com/hyperpolymath/$r.git" _w 2>/dev/null; then + continue + fi + if ! bash scripts/check-lockfile-drift.sh _w >> drift-report.tsv 2>/dev/null; then + drifted=$((drifted+1)) + fi + rm -rf _w + done < repos.txt + + rows=$(( $(wc -l < drift-report.tsv) - 1 )) + { + echo "total=$TOTAL" + echo "scanned=$scanned" + echo "withlock=$withlock" + echo "drifted=$drifted" + echo "rows=$rows" + } >> "$GITHUB_OUTPUT" + echo "scanned=$scanned withlock=$withlock drifted=$drifted rows=$rows" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: lockfile-drift-report + path: drift-report.tsv + retention-days: 90 + + - name: Step summary + if: always() + run: | + { + echo "## Lockfile Drift Detect" + echo + echo "| | |" + echo "|---|---:|" + echo "| non-archived repos | ${{ steps.sweep.outputs.total }} |" + echo "| scanned | ${{ steps.sweep.outputs.scanned }} |" + echo "| carrying a lockfile | ${{ steps.sweep.outputs.withlock }} |" + echo "| **with drift** | **${{ steps.sweep.outputs.drifted }}** |" + echo "| drifted entries | ${{ steps.sweep.outputs.rows }} |" + echo + echo '```' + head -40 drift-report.tsv + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Open / update the tracking issue + if: steps.sweep.outputs.drifted != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + title="lockfile drift: ${{ steps.sweep.outputs.drifted }} repo(s) as of $(date -u +%Y-%m-%d)" + + # NOTE: deliberately NOT a heredoc. A heredoc body sits at column 0, + # which terminates the enclosing `run: |` block scalar and makes the + # workflow unparseable — the exact defect that took + # .git-private-farm/inbox-steward-propagate.yml out of service and + # blocked `gh actions-lock` from running at all. Build the body with + # indented appends instead. + { + echo "Automated sweep for **lockfile version drift** — the estate's recurring CI killer." + echo + echo "**The fault.** Dependabot bumps an action version in a workflow; \`actions.lock\` is not regenerated; the workflow then requests a version the lockfile does not record, and GitHub rejects the run at \`startup_failure\` with **zero jobs** — no log, no annotation. It is self-reinflicting, so affected repos recur." + echo + echo "**To fix a listed repo:** run \`gh actions-lock\`, then restore the SPDX header to line 1 (the tool restamps its banner above it and the workflow-security linter greps \`head -1\` only)." + echo + echo "**Calibration:** a listed row is a genuine inconsistency but is *not* proof that workflow is dead — some drift is survivable. The reliable direction is the converse: workflows that were dead had drifted entries." + echo + echo "Full list in the run's \`lockfile-drift-report.tsv\` artifact." + echo + echo "| | |" + echo "|---|---:|" + echo "| scanned | ${{ steps.sweep.outputs.scanned }} |" + echo "| carrying a lockfile | ${{ steps.sweep.outputs.withlock }} |" + echo "| **with drift** | **${{ steps.sweep.outputs.drifted }}** |" + echo "| drifted entries | ${{ steps.sweep.outputs.rows }} |" + } > issue-body.md + body="$(cat issue-body.md)" + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --label lockfile-drift \ + --state open --limit 1 --json number --jq '.[0].number // empty') + if [ -n "$existing" ]; then + gh issue edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" \ + --body "Re-swept $(date -u +%Y-%m-%d): **${{ steps.sweep.outputs.drifted }}** repo(s) drifted." + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" \ + --label lockfile-drift || \ + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" + fi diff --git a/scripts/check-lockfile-drift.sh b/scripts/check-lockfile-drift.sh new file mode 100755 index 00000000..83c95c6f --- /dev/null +++ b/scripts/check-lockfile-drift.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell +set -eo pipefail + +# check-lockfile-drift.sh — detect the estate's recurring CI killer. +# +# ── The fault this catches ───────────────────────────────────────────────── +# Dependabot bumps an action version *in a workflow file*. Nobody regenerates +# `.github/workflows/actions.lock`. The workflow now requests a version the +# lockfile does not record, and GitHub rejects the run at `startup_failure` +# with ZERO jobs — no log, no annotation, nothing in REST or GraphQL. +# +# It is self-reinflicting: any repo with Dependabot AND a lockfile AND no +# regeneration step re-acquires the fault on every action bump. Measured +# 2026-08-25: 60 of 60 sampled repos have both. That is why repos appear to +# "go bad again" after being fixed — nobody broke them, the clock did. +# +# Proven on hypatia (standards session 2026-08-25): three drifted entries +# (docker/setup-buildx-action, github/codeql-action, taiki-e/install-action) +# discriminated PERFECTLY — every workflow using one was dead, every workflow +# using none was alive. Regeneration revived all four. See hypatia#723. +# +# ── What this is NOT ─────────────────────────────────────────────────────── +# Three other lockfile failure modes exist and this script does not cover them +# (it reports mode 4 only, because that is the one that recurs on its own): +# 1. no actions.lock at all in an enforced repo → whole repo dead +# 2. lockfile present but a workflow has NO entry → that workflow dead +# 3. entry exists but UNDER-declares (unresolvable action → false `[]`) +# 4. entry exists but the VERSION has drifted ← THIS SCRIPT +# +# ── Calibration: what a hit does and does not prove ──────────────────────── +# A reported line is a genuine inconsistency — the workflow requests a version +# the lockfile does not record for that action. It is NOT proof the workflow is +# dead. Measured on hypatia after regeneration: `tests.yml` still carried an +# inline `actions/checkout@34e11487…` against a lockfile recording `v7.0.1` +# (→ 3d3c42e5…), and it *started* anyway. So treat output as "reconcile this", +# not "this is why CI is down". The strong claim is the converse and it holds: +# every workflow that WAS dead had a drifted entry. +# +# Usage: check-lockfile-drift.sh [REPO_DIR] (default: .) +# Output: TSV — repo workflow requested locked +# Exit: 0 = no drift (or no lockfile — not this script's business) +# 1 = drift found +# 2 = usage / environment error + +REPO_DIR="${1:-.}" +LOCK="$REPO_DIR/.github/workflows/actions.lock" +WFDIR="$REPO_DIR/.github/workflows" + +[ -d "$WFDIR" ] || { echo "[drift] no .github/workflows in $REPO_DIR — nothing to check"; exit 0; } + +# No lockfile is failure mode 1, not mode 4. Report and leave it alone: a repo +# outside the enforcement cohort is legitimately lockfile-free (proof-burrower +# has none and runs fine), so absence is NOT evidence of a fault. +[ -f "$LOCK" ] || { echo "[drift] no actions.lock in $REPO_DIR — out of scope (see mode 1)"; exit 0; } + +drift=0 +checked=0 + +for wf in "$WFDIR"/*.yml "$WFDIR"/*.yaml; do + [ -f "$wf" ] || continue + base="$(basename "$wf")" + checked=$((checked + 1)) + + # Actions the workflow actually requests. Two normalisations matter: + # * drop reusable-workflow calls (owner/repo/.github/workflows/x.yml@ref) — + # those legitimately carry a bare `[]` entry and are not drift + # * strip sub-action paths: `github/codeql-action/init@v1` is recorded in + # the lockfile as `github/codeql-action@v1` + grep -oE "uses:[[:space:]]*[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+@[A-Za-z0-9._-]+" "$wf" 2>/dev/null \ + | sed -E 's/uses:[[:space:]]*//' \ + | grep -v '/\.github/workflows/' \ + | sed -E 's#^([^/]+/[^/@]+)(/[^@]*)?@#\1@#' \ + | sort -u > /tmp/_drift_want.$$ || true + + [ -s /tmp/_drift_want.$$ ] || { rm -f /tmp/_drift_want.$$; continue; } + + # Versions the lockfile records for THIS workflow. + awk -v key=" '.github/workflows/$base':" ' + $0 == key { on = 1; next } + on && /^ - / { gsub(/^ - .|.$/, ""); print; next } + on && NF && $0 !~ /^ / { exit } + ' "$LOCK" | sort -u > /tmp/_drift_have.$$ || true + + while read -r want; do + [ -n "$want" ] || continue + grep -qxF "$want" /tmp/_drift_have.$$ && continue + + name="${want%@*}" + ref="${want#*@}" + + # Only DRIFT if the lockfile knows this action at a *different* version. + # Absent entirely is mode 2/3, or a verified-creator action that + # legitimately needs no entry (e.g. Swatinem/rust-cache) — not drift. + have="$(grep -m1 -F "$name@" /tmp/_drift_have.$$)" || continue + + # A workflow may pin by 40-char SHA while the lockfile records a TAG. + # That is the same action in two notations, NOT drift. The lockfile + # resolves every entry to `commit: 'sha1-<40hex>'`, so compare against + # that rather than against the tag string. + if printf '%s' "$ref" | grep -qE '^[0-9a-f]{40}$'; then + resolved="$(awk -v k=" '$have':" ' + $0 == k { on = 1; next } + on && /commit:/ { gsub(/.*sha1-|.$/, ""); print; exit } + on && NF && $0 !~ /^ / { exit } + ' "$LOCK")" + [ "$resolved" = "$ref" ] && continue # same commit, different notation + fi + + printf '%s\t%s\t%s\t%s\n' "$(basename "$REPO_DIR")" "$base" "$want" "$have" + drift=$((drift + 1)) + done < /tmp/_drift_want.$$ + + rm -f /tmp/_drift_want.$$ /tmp/_drift_have.$$ +done + +if [ "$drift" -gt 0 ]; then + echo "[drift] $drift drifted entry/entries across $checked workflow(s) in $REPO_DIR" >&2 + echo "[drift] fix: run \`gh actions-lock\` then restore SPDX to line 1" >&2 + exit 1 +fi + +echo "[drift] clean — $checked workflow(s) checked in $REPO_DIR" +exit 0