feat(ci): detect lockfile version drift — the estate's recurring CI killer

.github/workflows/actions.lock

@@ -61,6 +61,9 @@ workflows:
         - 'peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697'
     '.github/workflows/launcher-standard-lockstep.yml':
         - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1'
+    '.github/workflows/lockfile-drift-detect.yml':
+        - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1'
+        - 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a'
     '.github/workflows/makefile-blocker.yml':
         - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1'
     '.github/workflows/mirror-reusable.yml':

.github/workflows/lockfile-drift-detect.yml

@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (c) 2026 Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
+#
+# lockfile-drift-detect.yml — estate-wide sweep for the recurring CI killer.
+#
+# THE FAULT: Dependabot bumps an action version in a workflow; nobody
+# regenerates `.github/workflows/actions.lock`; the workflow then requests a
+# version the lockfile does not record, and GitHub rejects the run at
+# `startup_failure` with ZERO jobs — no log, no annotation, nothing in REST.
+#
+# It is SELF-REINFLICTING. Measured 2026-08-25: 60 of 60 sampled repos carry
+# both a lockfile and a Dependabot config, so the whole estate re-acquires this
+# on every action bump. That is why repos appear to "go bad again" after being
+# fixed — nobody broke them, the clock did.
+#
+# Proven on hypatia#723: three drifted entries discriminated perfectly (every
+# workflow using one was dead, every workflow using none was alive), and
+# regeneration took all four dead workflows from `startup_failure` to running.
+#
+# This workflow REPORTS ONLY. It opens/updates one tracking issue and never
+# mutates another repository — consistent with the estate guardrail that
+# unattended cross-repo mutation is a human decision.
+name: Lockfile Drift Detect
+
+on:
+  schedule:
+    # Tuesday 07:20 UTC — after Dependabot's usual weekly window, so a bump
+    # and its resulting drift are caught in the same week rather than the next.
+    - cron: '20 7 * * 2'
+  workflow_dispatch:
+    inputs:
+      limit:
+        description: 'Max repos to scan (0 = all)'
+        type: string
+        default: '0'
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  sweep:
+    name: Sweep the estate for lockfile drift
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout standards (for the checker script)
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Sweep
+        id: sweep
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LIMIT: ${{ inputs.limit || '0' }}
+        run: |
+          set -euo pipefail
+          : > drift-report.tsv
+          printf 'repo\tworkflow\trequested\tlocked\n' >> drift-report.tsv
+
+          # Enumerate, then filter. `gh repo list` has a denominator we can
+          # state; `gh search` does not. A count landing on a round number is a
+          # page size, not a measurement.
+          gh repo list hyperpolymath --limit 1000 --no-archived \
+            --json name --jq '.[].name' > repos.txt
+          TOTAL=$(wc -l < repos.txt)
+          echo "enumerated $TOTAL non-archived repos"
+          # NOT `[ test ] && cmd` — under `set -e` a false test makes the whole
+          # compound non-zero and kills the run, and 0 is the DEFAULT.
+          if [ "$LIMIT" != "0" ]; then
+            head -n "$LIMIT" repos.txt > r.tmp
+            mv r.tmp repos.txt
+          fi
+
+          scanned=0; withlock=0; drifted=0
+          while read -r r; do
+            scanned=$((scanned+1))
+            # Cheap probe first: skip repos with no lockfile (that is failure
+            # mode 1, not drift, and a lockfile-free repo may be legitimately
+            # outside the enforcement cohort).
+            gh api "repos/hyperpolymath/$r/contents/.github/workflows/actions.lock" \
+              --jq .size >/dev/null 2>&1 || continue
+            withlock=$((withlock+1))
+
+            rm -rf _w
+            if ! git clone -q --depth 1 \
+                 "https://x-access-token:${GH_TOKEN}@github.com/hyperpolymath/$r.git" _w 2>/dev/null; then
+              continue
+            fi
+            if ! bash scripts/check-lockfile-drift.sh _w >> drift-report.tsv 2>/dev/null; then
+              drifted=$((drifted+1))
+            fi
+            rm -rf _w
+          done < repos.txt
+
+          rows=$(( $(wc -l < drift-report.tsv) - 1 ))
+          {
+            echo "total=$TOTAL"
+            echo "scanned=$scanned"
+            echo "withlock=$withlock"
+            echo "drifted=$drifted"
+            echo "rows=$rows"
+          } >> "$GITHUB_OUTPUT"
+          echo "scanned=$scanned withlock=$withlock drifted=$drifted rows=$rows"
+
+      - name: Upload report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: lockfile-drift-report
+          path: drift-report.tsv
+          retention-days: 90
+
+      - name: Step summary
+        if: always()
+        run: |
+          {
+            echo "## Lockfile Drift Detect"
+            echo
+            echo "| | |"
+            echo "|---|---:|"
+            echo "| non-archived repos | ${{ steps.sweep.outputs.total }} |"
+            echo "| scanned | ${{ steps.sweep.outputs.scanned }} |"
+            echo "| carrying a lockfile | ${{ steps.sweep.outputs.withlock }} |"
+            echo "| **with drift** | **${{ steps.sweep.outputs.drifted }}** |"
+            echo "| drifted entries | ${{ steps.sweep.outputs.rows }} |"
+            echo
+            echo '```'
+            head -40 drift-report.tsv
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Open / update the tracking issue
+        if: steps.sweep.outputs.drifted != '0'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          title="lockfile drift: ${{ steps.sweep.outputs.drifted }} repo(s) as of $(date -u +%Y-%m-%d)"
+
+          # NOTE: deliberately NOT a heredoc. A heredoc body sits at column 0,
+          # which terminates the enclosing `run: |` block scalar and makes the
+          # workflow unparseable — the exact defect that took
+          # .git-private-farm/inbox-steward-propagate.yml out of service and
+          # blocked `gh actions-lock` from running at all. Build the body with
+          # indented appends instead.
+          {
+            echo "Automated sweep for **lockfile version drift** — the estate's recurring CI killer."
+            echo
+            echo "**The fault.** Dependabot bumps an action version in a workflow; \`actions.lock\` is not regenerated; the workflow then requests a version the lockfile does not record, and GitHub rejects the run at \`startup_failure\` with **zero jobs** — no log, no annotation. It is self-reinflicting, so affected repos recur."
+            echo
+            echo "**To fix a listed repo:** run \`gh actions-lock\`, then restore the SPDX header to line 1 (the tool restamps its banner above it and the workflow-security linter greps \`head -1\` only)."
+            echo
+            echo "**Calibration:** a listed row is a genuine inconsistency but is *not* proof that workflow is dead — some drift is survivable. The reliable direction is the converse: workflows that were dead had drifted entries."
+            echo
+            echo "Full list in the run's \`lockfile-drift-report.tsv\` artifact."
+            echo
+            echo "| | |"
+            echo "|---|---:|"
+            echo "| scanned | ${{ steps.sweep.outputs.scanned }} |"
+            echo "| carrying a lockfile | ${{ steps.sweep.outputs.withlock }} |"
+            echo "| **with drift** | **${{ steps.sweep.outputs.drifted }}** |"
+            echo "| drifted entries | ${{ steps.sweep.outputs.rows }} |"
+          } > issue-body.md
+          body="$(cat issue-body.md)"
+          existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --label lockfile-drift \
+                       --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$existing" ]; then
+            gh issue edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body"
+            gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" \
+              --body "Re-swept $(date -u +%Y-%m-%d): **${{ steps.sweep.outputs.drifted }}** repo(s) drifted."
+          else
+            gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" \
+              --label lockfile-drift || \
+            gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body"
+          fi

scripts/check-lockfile-drift.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (c) 2026 Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
+set -eo pipefail
+
+# check-lockfile-drift.sh — detect the estate's recurring CI killer.
+#
+# ── The fault this catches ─────────────────────────────────────────────────
+# Dependabot bumps an action version *in a workflow file*. Nobody regenerates
+# `.github/workflows/actions.lock`. The workflow now requests a version the
+# lockfile does not record, and GitHub rejects the run at `startup_failure`
+# with ZERO jobs — no log, no annotation, nothing in REST or GraphQL.
+#
+# It is self-reinflicting: any repo with Dependabot AND a lockfile AND no
+# regeneration step re-acquires the fault on every action bump. Measured
+# 2026-08-25: 60 of 60 sampled repos have both. That is why repos appear to
+# "go bad again" after being fixed — nobody broke them, the clock did.
+#
+# Proven on hypatia (standards session 2026-08-25): three drifted entries
+# (docker/setup-buildx-action, github/codeql-action, taiki-e/install-action)
+# discriminated PERFECTLY — every workflow using one was dead, every workflow
+# using none was alive. Regeneration revived all four. See hypatia#723.
+#
+# ── What this is NOT ───────────────────────────────────────────────────────
+# Three other lockfile failure modes exist and this script does not cover them
+# (it reports mode 4 only, because that is the one that recurs on its own):
+#   1. no actions.lock at all in an enforced repo → whole repo dead
+#   2. lockfile present but a workflow has NO entry → that workflow dead
+#   3. entry exists but UNDER-declares (unresolvable action → false `[]`)
+#   4. entry exists but the VERSION has drifted    ← THIS SCRIPT
+#
+# ── Calibration: what a hit does and does not prove ────────────────────────
+# A reported line is a genuine inconsistency — the workflow requests a version
+# the lockfile does not record for that action. It is NOT proof the workflow is
+# dead. Measured on hypatia after regeneration: `tests.yml` still carried an
+# inline `actions/checkout@34e11487…` against a lockfile recording `v7.0.1`
+# (→ 3d3c42e5…), and it *started* anyway. So treat output as "reconcile this",
+# not "this is why CI is down". The strong claim is the converse and it holds:
+# every workflow that WAS dead had a drifted entry.
+#
+# Usage:  check-lockfile-drift.sh [REPO_DIR]     (default: .)
+# Output: TSV — repo <TAB> workflow <TAB> requested <TAB> locked
+# Exit:   0 = no drift (or no lockfile — not this script's business)
+#         1 = drift found
+#         2 = usage / environment error
+
+REPO_DIR="${1:-.}"
+LOCK="$REPO_DIR/.github/workflows/actions.lock"
+WFDIR="$REPO_DIR/.github/workflows"
+
+[ -d "$WFDIR" ] || { echo "[drift] no .github/workflows in $REPO_DIR — nothing to check"; exit 0; }
+
+# No lockfile is failure mode 1, not mode 4. Report and leave it alone: a repo
+# outside the enforcement cohort is legitimately lockfile-free (proof-burrower
+# has none and runs fine), so absence is NOT evidence of a fault.
+[ -f "$LOCK" ] || { echo "[drift] no actions.lock in $REPO_DIR — out of scope (see mode 1)"; exit 0; }
+
+drift=0
+checked=0
+
+for wf in "$WFDIR"/*.yml "$WFDIR"/*.yaml; do
+  [ -f "$wf" ] || continue
+  base="$(basename "$wf")"
+  checked=$((checked + 1))
+
+  # Actions the workflow actually requests. Two normalisations matter:
+  #   * drop reusable-workflow calls (owner/repo/.github/workflows/x.yml@ref) —
+  #     those legitimately carry a bare `[]` entry and are not drift
+  #   * strip sub-action paths: `github/codeql-action/init@v1` is recorded in
+  #     the lockfile as `github/codeql-action@v1`
+  grep -oE "uses:[[:space:]]*[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+@[A-Za-z0-9._-]+" "$wf" 2>/dev/null \
+    | sed -E 's/uses:[[:space:]]*//' \
+    | grep -v '/\.github/workflows/' \
+    | sed -E 's#^([^/]+/[^/@]+)(/[^@]*)?@#\1@#' \
+    | sort -u > /tmp/_drift_want.$$ || true
+
+  [ -s /tmp/_drift_want.$$ ] || { rm -f /tmp/_drift_want.$$; continue; }
+
+  # Versions the lockfile records for THIS workflow.
+  awk -v key="    '.github/workflows/$base':" '
+    $0 == key            { on = 1; next }
+    on && /^        - / { gsub(/^        - .|.$/, ""); print; next }
+    on && NF && $0 !~ /^        / { exit }
+  ' "$LOCK" | sort -u > /tmp/_drift_have.$$ || true
+
+  while read -r want; do
+    [ -n "$want" ] || continue
+    grep -qxF "$want" /tmp/_drift_have.$$ && continue
+
+    name="${want%@*}"
+    ref="${want#*@}"
+
+    # Only DRIFT if the lockfile knows this action at a *different* version.
+    # Absent entirely is mode 2/3, or a verified-creator action that
+    # legitimately needs no entry (e.g. Swatinem/rust-cache) — not drift.
+    have="$(grep -m1 -F "$name@" /tmp/_drift_have.$$)" || continue
+
+    # A workflow may pin by 40-char SHA while the lockfile records a TAG.
+    # That is the same action in two notations, NOT drift. The lockfile
+    # resolves every entry to `commit: 'sha1-<40hex>'`, so compare against
+    # that rather than against the tag string.
+    if printf '%s' "$ref" | grep -qE '^[0-9a-f]{40}$'; then
+      resolved="$(awk -v k="    '$have':" '
+        $0 == k { on = 1; next }
+        on && /commit:/ { gsub(/.*sha1-|.$/, ""); print; exit }
+        on && NF && $0 !~ /^        / { exit }
+      ' "$LOCK")"
+      [ "$resolved" = "$ref" ] && continue   # same commit, different notation
+    fi
+
+    printf '%s\t%s\t%s\t%s\n' "$(basename "$REPO_DIR")" "$base" "$want" "$have"
+    drift=$((drift + 1))
+  done < /tmp/_drift_want.$$
+
+  rm -f /tmp/_drift_want.$$ /tmp/_drift_have.$$
+done
+
+if [ "$drift" -gt 0 ]; then
+  echo "[drift] $drift drifted entry/entries across $checked workflow(s) in $REPO_DIR" >&2
+  echo "[drift] fix: run \`gh actions-lock\` then restore SPDX to line 1" >&2
+  exit 1
+fi
+
+echo "[drift] clean — $checked workflow(s) checked in $REPO_DIR"
+exit 0