diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d7ad20..2603f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ permissions: jobs: empty-diff: if: github.event_name == 'pull_request' - uses: Glyndor/.github/.github/workflows/empty-diff.yml@c958978687af37dc2d826a967d9c549589afb39f # v1.14.1 + uses: Glyndor/.github/.github/workflows/empty-diff.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 pin-policy: - uses: Glyndor/.github/.github/workflows/pin-policy-reusable.yml@8517d94fde00466f59b95c5d1bd838e2ad24a8bf # v1.15.2 + uses: Glyndor/.github/.github/workflows/pin-policy-reusable.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 # Lint and test the Python scripts that this repository ships. The # job name is `tests`; python-ci.yml's inner job name is `Lint & test`; @@ -44,7 +44,7 @@ jobs: # rename that breaks the four consumer repositories. The 90% gate is # a follow-up after the rename. tests: - uses: Glyndor/.github/.github/workflows/python-ci.yml@7099f8a9c8be91fa1732fe2402268b338771233f # v1.15.0 + uses: Glyndor/.github/.github/workflows/python-ci.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 with: coverage-threshold: 0 diff --git a/.github/workflows/pin-watch.yml b/.github/workflows/pin-watch.yml new file mode 100644 index 0000000..8ebe2bf --- /dev/null +++ b/.github/workflows/pin-watch.yml @@ -0,0 +1,95 @@ +name: Pin watch + +# Self-CI for this repository: opens a single issue when any pinned tool +# version here has drifted from its upstream release. Glyndor/podup#1526 +# motivated this — the reusables each carry their own pins, and the +# hand survey on 2026-08-24 found eight of nineteen drifted, one of +# them a real defect. A hand survey is exactly what the issue says is +# not good enough. +# +# The Python script lives at `scripts/pin-watch.py` rather than as a +# heredoc inside this file, mirroring how `pin-policy-reusable.yml` +# invokes `scripts/pin-policy.py`. A heredoc here would force every +# `actionlint` run on this workflow through shellcheck against the full +# script, and the script's size (well over a hundred lines) hits a +# pathological case in shellcheck v0.11.0 that hangs the linter +# indefinitely. The separate-file form is also the more reviewable +# shape: pin-watch.py can be diffed and tested like any other Python +# module, and this workflow file stays focused on the schedule, the +# permissions and the issue it opens. +# +# Pin values are READ FROM the workflow files at runtime, never +# hardcoded. Hardcoding the pin in the watcher would mean the watcher +# can disagree with reality, which is the bug it exists to prevent. +# Where a pin appears in more than one file (rust toolchain, +# actionlint, shellcheck), the first file listed in scripts/pin-watch.py +# is the canonical source — every other file is asserted equal to it +# by tests/test_workflow_pins.py, so the watcher reads one and trusts +# the test to enforce the rest. +# +# A held pin (the four entries in HOLDS in scripts/pin-watch.py) is +# reported ONLY when the pinned value stops matching its hold version — +# meaning somebody changed the pin and the hold note is now stale. While +# the pin still matches the hold, the watcher is silent about it. +# Reporting the upstream delta every Tuesday would train the reader to +# ignore the issue, which is the failure mode the hold list exists to +# prevent. Do not "fix" the silence by deleting the entry. +# +# Schedule: Tuesday 06:00 UTC. Deliberately not Monday: podup's own +# pin-watch runs Monday 05:00, and stacking this repository's other +# scheduled work on a single minute is the failure mode +# schedule-freshness.yml exists to detect (see 2026-06-29 in the header +# of that reusable). +# +# Reporting shape, after podup's pin-watch: collect all drifts, open +# ONE issue titled exactly `Pinned tool versions have drifted`, skip if +# an issue with that exact title is already open. Labels: `type:ci`, +# `prio:P2`, `status:ready`, `area:ci`, `effort:S`. Exit 0 with a short +# confirmation when nothing drifted. +# +# PSScriptAnalyzer's upstream endpoint +# (powershellgallery.com/api/v2/FindPackagesById()) returns an Atom XML +# feed and is awkward to parse reliably; pssa-version is therefore read +# (so the pin is still reviewed) but its upstream is not queried, and +# the entry sits in SKIP_UPSTREAM in scripts/pin-watch.py. If a future +# maintainer wires a cleaner endpoint, drop the skip and add a fetcher. +# +# Default shell is pinned to bash: the inner `sort -V` style +# comparators the upstream fetchers use are GNU semantics, and a future +# Windows caller would otherwise silently mis-compare and fail closed +# on an empty string. + +on: + schedule: + - cron: '0 6 * * 2' + workflow_dispatch: + +permissions: + contents: read + issues: write + +defaults: + run: + shell: bash + +concurrency: + group: pin-watch-${{ github.ref }} + cancel-in-progress: true + +jobs: + pin-watch: + name: pin watch + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Detect drift and open a tracking issue + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + python3 -u scripts/pin-watch.py diff --git a/.github/workflows/rust-supply-chain.yml b/.github/workflows/rust-supply-chain.yml index 9abca66..f003e4b 100644 --- a/.github/workflows/rust-supply-chain.yml +++ b/.github/workflows/rust-supply-chain.yml @@ -15,12 +15,13 @@ on: # Pinned, not the runner's floating preinstalled Rust — see rust-ci.yml's # toolchain input for why an unpinned toolchain is a problem. # - # Keep this equal to rust-ci.yml's `toolchain`. It was 1.97 while rust-ci - # was already on 1.98, because the bump raised one and not the other and - # nothing compared them. A repository whose SBOM is produced by a compiler - # its test suite never ran is the defect podup#1487 recorded, and podup - # now has a test asserting its own two pins agree — nothing asserts it - # across these two reusables, so it is written here instead. + # Keep this equal to rust-ci.yml's and rust-audit.yml's `toolchain`. It + # was 1.97 while rust-ci was already on 1.98, because the bump raised + # one and not the other and nothing compared them. A repository whose + # SBOM is produced by a compiler its test suite never ran is the defect + # Glyndor/podup#1487 recorded. tests/test_workflow_pins.py asserts the + # three defaults agree; keep them in step on the bump PR so the test + # stays green. toolchain: description: Rust toolchain to pin the SBOM and license job to type: string diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e29dc63 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# Test dependencies for this repository's own suite. +# +# python-ci installs this file when it exists, which is how tests/ gets PyYAML. +# tests/test_workflow_pins.py parses the reusables as YAML rather than by regex, +# because the values it compares are quoted inconsistently across the files and a +# regex that tolerates every form is harder to trust than a parser. +# +# This is a hand-typed pin like the nineteen in the workflows, so pin-watch +# watches it too — the alternative is a watcher that misses the pin its own +# repository added. +pyyaml==6.0.3 diff --git a/scripts/pin-watch.py b/scripts/pin-watch.py new file mode 100644 index 0000000..e5c1781 --- /dev/null +++ b/scripts/pin-watch.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""Read every pinned tool version in this repository from its workflow file +and report any that have drifted from their upstream release. + +Pins are READ FROM the workflow files at runtime, never hardcoded. Where a +pin appears in more than one file (rust toolchain, actionlint, shellcheck), +the first file in PIN_SOURCES below is the canonical source — every other +file is asserted equal to it by tests/test_workflow_pins.py, so this script +reads one and trusts the test to enforce the rest. Hardcoding the pin in +the watcher would mean the watcher can disagree with reality, which is the +bug the watcher exists to prevent. + +A held pin (the four entries in HOLDS below) is reported ONLY when the +pinned value stops matching its hold version — meaning somebody changed +the pin and the hold note is now stale. While the pin still matches the +hold, the watcher is silent about it. Reporting the upstream delta every +Tuesday would train the reader to ignore the issue, which is the failure +mode the hold list exists to prevent. + +Upstream sources: + +- crates.io https://crates.io/api/v1/crates/ -> .crate.max_stable_version +- PyPI https://pypi.org/pypi//json -> .info.version +- GitHub gh release list --repo --limit 1 --json tagName --jq '.[0].tagName' +- rust https://static.rust-lang.org/dist/channel-rust-stable.toml, + the version under [pkg.rust]; compared only at MAJOR.MINOR +- python https://endoflife.date/api/python.json -> [0].cycle +- PSScriptAnalyzer — endpoint returns Atom XML; awkward to parse reliably, + so pssa-version is read but its upstream is not queried. + +Reporting: collect all drifts, open ONE issue titled exactly "Pinned tool +versions have drifted", skip if an issue with that exact title is already +open. Labels: type:ci, prio:P2, status:ready, area:ci, effort:S. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +WORKFLOWS_DIR = Path(".github/workflows") + +# Held pins: silent while the pinned value matches the hold version +# recorded here. The only time a held pin shows up in the report is when +# somebody changed the pin and forgot to update this list — the hold note +# is then stale and the next person needs to know. Do not "fix" the +# silence by deleting the entry; that trains the reader to ignore the +# report and is the failure mode the hold list exists to prevent. +HOLDS: dict[str, tuple[str, str]] = { + "ruff": ( + "0.12.0", + "0.16.4 reports 15 findings that are code changes in five files; PLW1510 needs check=False, not check=True", + ), + "bun": ( + "1.3.14", + "consumers epistle-panel and specio cannot be exercised from here", + ), + "gosec": ("v2.27.1", "consumer authcore cannot be exercised from here"), + "cargo-llvm-cov": ( + "0.8.7", + "five consumers; a coverage tool's bump can move numbers into a threshold", + ), +} + +# The single canonical source for each pin. Where a name appears in more +# than one file, tests/test_workflow_pins.py already asserts equality, so +# the FIRST file in the list is the one this watcher reads. Adding a new +# file to a list should be paired with a pin-agreement test, or two +# workflows will silently drift. +# +# kind=env reads a literal `KEY: value` line in a job's env: +# kind=input reads an on.workflow_call.inputs..default. +# kind=requirements reads a `name==version` line in requirements.txt, which +# python-ci installs when the file exists. It is a hand-typed pin like the +# others, so it is watched like the others — a watcher that skips the pin its +# own repository added is the failure this file exists to prevent. +PIN_SOURCES: list[tuple[str, str, str, str]] = [ + ("actionlint", "actionlint.yml", "env", "ACTIONLINT_VERSION"), + ("pyyaml-test-dep", "requirements.txt", "requirements", "pyyaml"), + ("shellcheck", "actionlint.yml", "env", "SHELLCHECK_VERSION"), + ("PyYAML", "docs-current.yml", "env", "PYYAML_VERSION"), + ("bun", "bun-ci.yml", "input", "bun-version"), + ("govulncheck", "go-audit.yml", "input", "govulncheck-version"), + ("gosec", "go-audit.yml", "input", "gosec-version"), + ("pssa-version", "powershell-ci.yml", "input", "pssa-version"), + ("python", "python-ci.yml", "input", "python-version"), + ("ruff", "python-ci.yml", "input", "ruff-version"), + ("pytest", "python-ci.yml", "input", "pytest-version"), + ("rust", "rust-ci.yml", "input", "toolchain"), + ("cargo-audit", "rust-audit.yml", "input", "cargo-audit-version"), + ("cargo-deny", "rust-audit.yml", "input", "cargo-deny-version"), + ("cargo-llvm-cov", "rust-ci.yml", "input", "llvm-cov-version"), + ("cargo-semver-checks", "rust-ci.yml", "input", "semver-checks-version"), + ("cargo-fuzz", "rust-fuzz.yml", "input", "cargo-fuzz-version"), + ("cargo-cyclonedx", "rust-supply-chain.yml", "input", "cyclonedx-version"), + ("cargo-about", "rust-supply-chain.yml", "input", "cargo-about-version"), +] + +# Read the pin, but do not query upstream. See the module docstring for +# why PSScriptAnalyzer is on this list. +SKIP_UPSTREAM: set[str] = {"pssa-version"} + + +def _read_env_value(text: str, env_key: str) -> str | None: + """Read the literal `KEY: value` env entry from a workflow file. + + Accepts both bare (`KEY: v0.11.0`) and quoted (`KEY: "v0.11.0"`) forms. + Returns the value, or None if no such line is found. The regex matches + one line at a time; the alternative captures let it accept either + quoting style without dragging in a YAML parser. + """ + pat = re.compile( + r"^\s*" + re.escape(env_key) + r":\s*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'{][^\s\"']*))" + r"\s*(?:#.*)?$", + re.MULTILINE, + ) + for m in pat.finditer(text): + v = ( + m.group(1) + if m.group(1) is not None + else (m.group(2) if m.group(2) is not None else m.group(3)) + ) + if v is not None: + return v + return None + + +def _read_input_default(text: str, input_name: str) -> str | None: + """Read the `default:` value of a named input from a workflow file. + + The input block is `input_name:` followed by an indented series of + keys. The `default:` value can be bare (`default: 1.2.3`) or quoted + (`default: "1.2.3"`); both are accepted. Returns the value, or None + if not found. + + This walks the file line-by-line rather than YAML-parsing the whole + document, so the watcher does not depend on having a particular + PyYAML version installed — the pin PyYAML would itself need to be + read first, and that is a bootstrap problem the workflow does not + need. + """ + lines = text.splitlines() + in_block = False + block_indent: int | None = None + for line in lines: + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + indent = len(line) - len(stripped) + if not in_block: + if stripped == f"{input_name}:" or stripped.startswith(f"{input_name}:"): + # `with:` blocks also use `:` at column zero; + # distinguish by checking that the following real line is + # MORE indented (a block) rather than less-or-equal (a + # scalar or end of an outer block). + idx = lines.index(line) + nxt = next( + ( + nxt_line + for nxt_line in lines[idx + 1 :] + if nxt_line.strip() and not nxt_line.lstrip().startswith("#") + ), + None, + ) + if nxt is None: + return None + if (len(nxt) - len(nxt.lstrip())) <= indent: + return None + in_block = True + block_indent = indent + else: + continue + else: + assert block_indent is not None + if indent <= block_indent: + return None + if stripped.startswith("default:"): + value_part = stripped[len("default:") :].strip() + if (value_part.startswith('"') and value_part.endswith('"')) or ( + value_part.startswith("'") and value_part.endswith("'") + ): + value_part = value_part[1:-1] + return value_part + return None + + +def _read_requirement_pin(text: str, name: str) -> str | None: + """Read `name==version` out of a requirements file. + + Only the `==` form is accepted. A range or a bare name is not a pin, and + reporting drift against something unpinned would be noise. + """ + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + left, sep, right = line.partition("==") + if sep and left.strip().lower() == name.lower(): + return right.strip() + return None + + +def read_pins() -> dict[str, tuple[str, str]]: + """Read every pinned value from its workflow file. + + Returns {pin_name: (source_file, value)}. Fails the run if a pin + cannot be read — the watcher must not silently skip a pin it was + asked to read. + """ + cache: dict[str, str] = {} + pins: dict[str, tuple[str, str]] = {} + for pin_name, source_file, kind, key in PIN_SOURCES: + if source_file not in cache: + # Most pins live under .github/workflows/; requirements.txt is at the + # repository root, so resolve relative to the root when the name is + # not a workflow. + path = ( + Path(source_file) + if not source_file.endswith((".yml", ".yaml")) + else WORKFLOWS_DIR / source_file + ) + cache[source_file] = path.read_text() + text = cache[source_file] + if kind == "env": + v = _read_env_value(text, key) + elif kind == "input": + v = _read_input_default(text, key) + elif kind == "requirements": + v = _read_requirement_pin(text, key) + else: + raise AssertionError(f"unknown kind: {kind!r}") + if v is None: + print( + f"::error file={source_file}::could not read " + f"{kind} {key} (pin: {pin_name})", + file=sys.stderr, + ) + sys.exit(1) + pins[pin_name] = (source_file, v) + return pins + + +# --- Upstream fetchers --------------------------------------------------- + +UA = "glyndor-pin-watch/1.0 (+https://github.com/Glyndor/.github)" + + +def _http_get_json(url: str) -> object: + req = urllib.request.Request( + url, + headers={"User-Agent": UA, "Accept": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def _http_get_text(url: str) -> str: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8") + + +def upstream_crates_io(crate: str) -> str: + body = _http_get_json(f"https://crates.io/api/v1/crates/{crate}") + v = body.get("crate", {}).get("max_stable_version") # type: ignore[union-attr] + if not v: + raise RuntimeError(f"no max_stable_version for {crate}") + return v + + +def upstream_pypi(project: str) -> str: + body = _http_get_json(f"https://pypi.org/pypi/{project}/json") + v = body.get("info", {}).get("version") # type: ignore[union-attr] + if not v: + raise RuntimeError(f"no info.version for {project}") + return v + + +def upstream_gh_release(repo: str) -> str: + out = subprocess.check_output( + [ + "gh", + "release", + "list", + "--repo", + repo, + "--limit", + "1", + "--json", + "tagName", + "--jq", + ".[0].tagName", + ], + text=True, + ).strip() + if not out: + raise RuntimeError(f"no releases found for {repo}") + return out + + +def upstream_rust_stable() -> str: + body = _http_get_text( + "https://static.rust-lang.org/dist/channel-rust-stable.toml", + ) + # Locate the [pkg.rust] section; the version= line immediately after + # it is the current stable. Everything else in the file is per-target + # metadata and not relevant here. + m = re.search( + r"^\[pkg\.rust\]\s*$(.*?)(?=^\[|\Z)", + body, + re.MULTILINE | re.DOTALL, + ) + if not m: + raise RuntimeError("no [pkg.rust] section in channel-rust-stable.toml") + # The version= line has the shape `version = "1.98.0 (88d9e12ae 2026-08-18)"` + # — the MAJOR.MINOR.PATCH is the first whitespace-separated token + # inside the quotes, the rest is the channel's git describe metadata. + # Match just the first token; the strict MAJOR.MINOR.PATCH regex + # below rejects anything else, so a future change to the file's + # shape surfaces as a hard error rather than a silent miscompare. + v_m = re.search(r'version\s*=\s*"(\d+\.\d+\.\d+)', m.group(1)) + if not v_m: + raise RuntimeError("no version= under [pkg.rust]") + return v_m.group(1) + + +def upstream_python_eol() -> str: + body = _http_get_json("https://endoflife.date/api/python.json") + cycle = body[0].get("cycle") # type: ignore[index] + if not cycle: + raise RuntimeError("no cycle[0] in endoflife.date response") + return cycle + + +FETCHERS: dict = { + "cargo-audit": lambda: upstream_crates_io("cargo-audit"), + "cargo-deny": lambda: upstream_crates_io("cargo-deny"), + "cargo-llvm-cov": lambda: upstream_crates_io("cargo-llvm-cov"), + "cargo-semver-checks": lambda: upstream_crates_io("cargo-semver-checks"), + "cargo-fuzz": lambda: upstream_crates_io("cargo-fuzz"), + "cargo-cyclonedx": lambda: upstream_crates_io("cargo-cyclonedx"), + "cargo-about": lambda: upstream_crates_io("cargo-about"), + "PyYAML": lambda: upstream_pypi("PyYAML"), + "pyyaml-test-dep": lambda: upstream_pypi("PyYAML"), + "ruff": lambda: upstream_pypi("ruff"), + "pytest": lambda: upstream_pypi("pytest"), + "actionlint": lambda: upstream_gh_release("rhysd/actionlint"), + "shellcheck": lambda: upstream_gh_release("koalaman/shellcheck"), + "bun": lambda: upstream_gh_release("oven-sh/bun"), + "gosec": lambda: upstream_gh_release("securego/gosec"), + "govulncheck": lambda: upstream_gh_release("golang/vuln"), + "rust": upstream_rust_stable, + "python": upstream_python_eol, +} + + +def fetch_upstream(pin_name: str) -> str: + fetcher = FETCHERS.get(pin_name) + if fetcher is None: + raise RuntimeError(f"no fetcher for {pin_name}") + return fetcher() + + +# --- Reporting ----------------------------------------------------------- + +ISSUE_TITLE = "Pinned tool versions have drifted" +ISSUE_LABELS = ("type:ci", "prio:P2", "status:ready", "area:ci", "effort:S") + + +def compare(pin_name: str, pin_value: str, upstream: str) -> bool: + """Return True if pin and upstream are equal for this pin's contract. + + Most pins compare as exact strings. The Rust toolchain pin is + MAJOR.MINOR (e.g. "1.98"); upstream reports MAJOR.MINOR.PATCH (e.g. + "1.98.0"). Compare only the two-component prefix — the third + component is a patch that the pin's contract deliberately does not + name. + """ + if pin_name == "rust": + pin_mm = ".".join(pin_value.split(".")[:2]) + up_mm = ".".join(upstream.split(".")[:2]) + return pin_mm == up_mm + return pin_value == upstream + + +def render_body( + drift: list[tuple[str, str, str, str]], + held_stale: list[tuple[str, str, str, str, str]], + skipped: list[tuple[str, str, str, str]], +) -> str: + lines: list[str] = [] + lines.append( + "The following tool versions pinned in this repository " + "have drifted from their upstream releases." + ) + lines.append("") + lines.append("| Pin | File | Pinned | Upstream |") + lines.append("|---|---|---|---|") + for name, src, pin, up in drift: + lines.append(f"| `{name}` | `{src}` | `{pin}` | `{up}` |") + lines.append("") + if held_stale: + lines.append("## Hold notes needing update") + lines.append("") + lines.append( + "The following pins are deliberately held below upstream, but " + "the pinned value no longer matches the hold version listed " + "in `.github/workflows/pin-watch.yml`'s `HOLDS` table — " + "somebody changed the pin and the hold note is now stale. " + "Update `HOLDS` to match the new pin (or restore the " + "original pin if the change was unintentional)." + ) + lines.append("") + lines.append("| Pin | File | Pinned (now) | Hold says | Reason |") + lines.append("|---|---|---|---|---|") + for name, src, pin, hold, reason in held_stale: + lines.append(f"| `{name}` | `{src}` | `{pin}` | `{hold}` | {reason} |") + lines.append("") + if skipped: + lines.append("## Upstream check skipped") + lines.append("") + lines.append( + "The following pins were read but their upstream was not " + "queried this run (transient or by design). They are NOT in " + "the drift table above and are listed here only for " + "visibility." + ) + lines.append("") + lines.append("| Pin | File | Pinned | Reason |") + lines.append("|---|---|---|---|") + for name, src, pin, reason in skipped: + lines.append(f"| `{name}` | `{src}` | `{pin}` | {reason} |") + lines.append("") + lines.append("---") + lines.append("") + lines.append( + "_Opened by the `pin-watch` self-CI workflow. Closing this " + "issue without acting is the wrong response — bump the pinned " + "versions in the named workflow files (or update the hold note " + "in `.github/workflows/pin-watch.yml`), then close the issue " + "once the next scheduled run reports nothing._" + ) + return "\n".join(lines) + "\n" + + +def issue_already_open(title: str) -> bool: + """Return True if an open issue with this exact title exists.""" + out = subprocess.check_output( + [ + "gh", + "issue", + "list", + "--state", + "open", + "--search", + f"in:title {title}", + "--json", + "title", + "--jq", + ".[] | .title", + ], + text=True, + ) + titles = [line.strip() for line in out.splitlines() if line.strip()] + return title in titles + + +def open_issue(title: str, body: str) -> str: + cmd = ["gh", "issue", "create", "--title", title, "--body", body] + for label in ISSUE_LABELS: + cmd.extend(["--label", label]) + out = subprocess.check_output(cmd, text=True).strip() + return out + + +# --- Main flow ----------------------------------------------------------- + + +def main() -> int: + pins = read_pins() + print("Pins read from workflow files:") + for name in sorted(pins): + src, v = pins[name] + print(f" {name:22s} {v!r:18s} ({src})") + print() + + drift: list[tuple[str, str, str, str]] = [] + held_stale: list[tuple[str, str, str, str, str]] = [] + fetch_errors: list[tuple[str, str, str, str]] = [] + skipped: list[tuple[str, str, str, str]] = [] + + for name, (src, pin) in pins.items(): + if name in SKIP_UPSTREAM: + skipped.append( + ( + name, + src, + pin, + "upstream endpoint not wired (see SKIP_UPSTREAM in pin-watch.yml)", + ) + ) + continue + if name in HOLDS: + hold_v, hold_reason = HOLDS[name] + if pin != hold_v: + held_stale.append((name, src, pin, hold_v, hold_reason)) + continue + try: + upstream = fetch_upstream(name) + except ( + subprocess.CalledProcessError, + RuntimeError, + urllib.error.URLError, + json.JSONDecodeError, + OSError, + ) as e: + fetch_errors.append((name, src, pin, str(e))) + continue + if not compare(name, pin, upstream): + drift.append((name, src, pin, upstream)) + + print("Upstream results (non-held, non-skipped pins):") + for name in sorted(pins): + if name in SKIP_UPSTREAM or name in HOLDS: + continue + src, pin = pins[name] + if any(d[0] == name for d in drift): + up = next(d[3] for d in drift if d[0] == name) + print(f" {name:22s} pin={pin!r:18s} upstream={up!r:18s} DRIFT") + elif any(e[0] == name for e in fetch_errors): + err = next(e[3] for e in fetch_errors if e[0] == name) + print(f" {name:22s} pin={pin!r:18s} upstream=") + else: + print(f" {name:22s} pin={pin!r:18s} upstream=") + print() + + print("Held pins (silent while pin matches hold):") + for name in sorted(HOLDS): + if name not in pins: + continue + src, pin = pins[name] + hold_v, _ = HOLDS[name] + if pin == hold_v: + print( + f" {name:22s} pin={pin!r:14s} hold={hold_v!r:14s} " + "SILENT (pin matches hold)" + ) + else: + print( + f" {name:22s} pin={pin!r:14s} hold={hold_v!r:14s} " + "STALE HOLD (expected to be reported)" + ) + print() + + if fetch_errors: + print("Upstream fetch errors:") + for name, src, pin, err in fetch_errors: + print(f" {name}: {err}") + print() + + if not drift and not held_stale: + print("No drift detected. pin-watch is current.") + return 0 + + body = render_body(drift, held_stale, skipped) + print("---") + print("Issue body:") + print(body) + print("---") + + if issue_already_open(ISSUE_TITLE): + print(f"Issue {ISSUE_TITLE!r} is already open — not creating a duplicate.") + return 0 + + url = open_issue(ISSUE_TITLE, body) + print(f"Opened: {url}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_workflow_pins.py b/tests/test_workflow_pins.py new file mode 100644 index 0000000..9ec5069 --- /dev/null +++ b/tests/test_workflow_pins.py @@ -0,0 +1,298 @@ +"""Tests that pin agreement across reusables is not just a comment. + +Three reusables in `.github/workflows/` declare a Rust `toolchain` input, +two env values for shellcheck/actionlint ride on three workflows, and a +shellcheck input default travels on two. Each value is pinned on its own, +but no comment in any of the files compared them until rust-supply-chain +drifted to 1.97 while rust-ci and rust-audit were already on 1.98. A +release's SBOM and license attribution were then produced by a compiler +the test suite never ran — the defect Glyndor/podup#1487 recorded. + +A comment about "keep these equal" does not fail when one of them stops +being equal. These tests parse the workflow YAML directly (no subprocess, +no network, no `gh`) and fail the run when the pins diverge, so a future +drift trips CI on the bump PR instead of three months later on a release +artifact. + +The four invariants: + +1. The Rust `toolchain` defaults in `rust-ci.yml`, `rust-audit.yml`, and + `rust-supply-chain.yml` all agree. +2. The shared Rust toolchain value is a two-component version (`1.98`, + not `1.98.0` and not `stable`) — `rustup toolchain install` accepts + all three and they mean different things, so the shape is part of the + contract. +3. The `SHELLCHECK_VERSION` env value in `actionlint.yml` matches the + `shellcheck-version` input defaults in `workflow-lint.yml` and + `shell-ci.yml`. The shape that drifts is "two of three bumped" — + one workflow's author updates one pin, the others stay. +4. The `ACTIONLINT_VERSION` env value in `actionlint.yml` matches the + one in `workflow-lint.yml`. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + + +def _default_for(workflow: str, input_name: str) -> str: + """Return the declared default of `input_name` in `workflow`. + + Reads the workflow file with `yaml.safe_load`, walks down to + `on.workflow_call.inputs..default`, and returns the + string. Fails the test with a clear, named message if any link in + that chain is missing — never returns None silently, because a silent + None would let a missing default pass an equality check. + + YAML 1.1 reads the bare `on:` key in a GitHub Actions workflow as the + Python boolean `True`. PyYAML 6.x keeps that behaviour here, so the + parsed key is `True`; older or YAML-1.2-only resolvers can surface + it as the string `"on"` instead. Both are handled. + """ + path = WORKFLOWS_DIR / workflow + if not path.is_file(): + pytest.fail(f"{workflow}: file not found at {path}") + try: + with path.open() as fh: + doc = yaml.safe_load(fh) + except yaml.YAMLError as e: + pytest.fail(f"{workflow}: YAML parse error: {e}") + if not isinstance(doc, dict): + pytest.fail(f"{workflow}: top-level YAML is not a mapping") + # Both keys are possible across YAML resolvers; try both. + on_node = doc.get(True) if True in doc else doc.get("on") + if on_node is None: + pytest.fail(f"{workflow}: missing `on:` section") + if not isinstance(on_node, dict): + pytest.fail( + f"{workflow}: `on:` is not a mapping (got {type(on_node).__name__})" + ) + workflow_call = on_node.get("workflow_call") + if not isinstance(workflow_call, dict): + pytest.fail(f"{workflow}: missing `on.workflow_call` section") + inputs = workflow_call.get("inputs") + if not isinstance(inputs, dict): + pytest.fail(f"{workflow}: missing `on.workflow_call.inputs` section") + input_spec = inputs.get(input_name) + if not isinstance(input_spec, dict): + pytest.fail(f"{workflow}: missing input `{input_name}`") + if "default" not in input_spec: + pytest.fail(f"{workflow}: input `{input_name}` has no `default`") + default = input_spec["default"] + if not isinstance(default, str): + pytest.fail( + f"{workflow}: input `{input_name}` default is {type(default).__name__}, " + f"expected string" + ) + return default + + +# --- 1. The three Rust toolchain pins agree ------------------------------ + + +def test_every_rust_toolchain_default_agrees() -> None: + """The three Rust reusables ship the same pinned toolchain. + + Real event: rust-supply-chain.yml sat on Rust 1.97 for an extended + period while rust-ci.yml and rust-audit.yml had already moved to + 1.98, because a toolchain bump raised two and not the third and + nothing compared them. The release's SBOM and license attribution + were then generated by a compiler the test suite never ran — the + defect Glyndor/podup#1487 recorded. A test asserting agreement is + the only shape that catches that class of drift on the bump PR + rather than at release time. + + Assertion message lists every file with its value so the failure + says which one moved, not just that they differ. The expected + version is intentionally NOT hardcoded — the test is about + agreement, not about a number, so it survives the next bump and + only fails on divergence. + """ + files = ("rust-ci.yml", "rust-audit.yml", "rust-supply-chain.yml") + values = {wf: _default_for(wf, "toolchain") for wf in files} + distinct = set(values.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{wf}={v!r}" for wf, v in values.items()) + pytest.fail( + f"Rust toolchain defaults disagree: {breakdown}. " + f"All three reusables must pin the same toolchain — see " + f"rust-supply-chain.yml's toolchain comment and the regression " + f"recorded in Glyndor/podup#1487." + ) + + +def test_rust_toolchain_is_a_two_component_version() -> None: + """The shared Rust toolchain value matches `MAJOR.MINOR`. + + `rustup toolchain install` accepts three shapes that mean different + things: a floating channel like `stable` (always the latest stable + release — which is why every other input here pins an exact version), + a fully-qualified triple like `1.98.0` (a single patch release, not a + channel), and the two-component channel like `1.98` (the stable + release on that minor — what the three reusables use). The shape is + part of the contract: a bump from `1.98` to `1.98.0` silently changes + caching and reproducibility without surfacing a difference in any + input name, and a reversion to `stable` undoes the whole pinning + story. + + Every file is checked — not just one — because the failure mode is + a single-file drift while the other two still say `1.98`: reading + only the first value would let that drift pass. + + Real event: no single drift has happened here, but the equivalent + drift would be a future PR that flips one of the three from `1.98` + to `stable` and slips through review because the test the audit + called out didn't exist. + """ + files = ("rust-ci.yml", "rust-audit.yml", "rust-supply-chain.yml") + values = {wf: _default_for(wf, "toolchain") for wf in files} + bad = {wf: v for wf, v in values.items() if not re.fullmatch(r"[0-9]+\.[0-9]+", v)} + if bad: + breakdown = ", ".join(f"{wf}={v!r}" for wf, v in sorted(bad.items())) + pytest.fail( + f"Rust toolchain default is not a two-component version: " + f"{breakdown}. Expected MAJOR.MINOR like '1.98'. `stable` " + f"floats; '1.98.0' pins a single patch; only MAJOR.MINOR pins " + f"the stable release on that minor." + ) + + +# --- 2. Shellcheck version agrees across the three workflows ---------- + + +# Matches a literal env value (skips `${{ ... }}` expressions, which are +# references, not values). Captures the first non-whitespace, non-comment +# token after the colon. +_SHELLCHECK_ENV_VALUE = re.compile( + r"^\s*SHELLCHECK_VERSION:\s*([^\s\"'#][^\s\"']*|[^\s\"'{][^\s\"']*)\s*$", + re.MULTILINE, +) +# Matches an input `shellcheck-version:` block and the `default:` line +# inside it. The capture is the default value, without surrounding quotes +# or trailing whitespace. `re.MULTILINE` makes `^` and `$` line-aware. +_SHELLCHECK_INPUT_DEFAULT = re.compile( + r"^\s*shellcheck-version:\s*$\n" + r"(?:[^\n]*\n)*?" + r"^\s*default:\s*[\"']?([^\s\"'#]+)[\"']?\s*$", + re.MULTILINE, +) + + +def _shellcheck_values() -> dict[str, str]: + """Return `{source: value}` for every SHELLCHECK_VERSION literal and + shellcheck-version default across the workflows directory. + + Two patterns cover the two shapes the workflows use: a literal + `SHELLCHECK_VERSION: vX.Y.Z` env entry (actionlint.yml), and a + `shellcheck-version: ... default: "vX.Y.Z"` input block + (workflow-lint.yml, shell-ci.yml). Both are returned with a + `source` key naming the file so a divergence message points at + the actual occurrence. + """ + found: dict[str, str] = {} + for path in sorted(WORKFLOWS_DIR.glob("*.yml")): + text = path.read_text() + for m in _SHELLCHECK_ENV_VALUE.finditer(text): + key = f"{path.name}:SHELLCHECK_VERSION(env)" + found[key] = m.group(1) + for m in _SHELLCHECK_INPUT_DEFAULT.finditer(text): + key = f"{path.name}:shellcheck-version(input.default)" + found[key] = m.group(1) + return found + + +def test_shellcheck_version_agrees_across_workflows() -> None: + """Every SHELLCHECK_VERSION env value and shellcheck-version input + default across the workflows directory agrees. + + The same drift shape as the Rust toolchain: one workflow author + bumps one pin, the others stay. There is no comment that compares + these three and the same defect (two of three bumped) would land + here if any one of them drifted. + + Real event: no shellcheck version drift has been recorded, but the + failure mode this defends against is the same one + `test_every_rust_toolchain_default_agrees` was written for — an + audit missed one bump and three workflows no longer lint each other + to a compatible version. The test exists so the next bump is + blocked at the PR. + """ + found = _shellcheck_values() + if len(found) < 2: + pytest.fail( + f"Found only {len(found)} SHELLCHECK_VERSION/shellcheck-version " + f"occurrences: {found}. Expected at least two." + ) + distinct = set(found.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{k}={v!r}" for k, v in sorted(found.items())) + pytest.fail( + f"Shellcheck version disagrees across workflows: {breakdown}. " + f"All workflows must install the same shellcheck release so " + f"a workflow this repo lints for itself is also the one it " + f"ships to consumers." + ) + + +# --- 3. Actionlint version agrees across the two workflows ---------- + + +_ACTIONLINT_ENV_VALUE = re.compile( + r"^\s*ACTIONLINT_VERSION:\s*([^\s\"'#][^\s\"']*|[^\s\"'{][^\s\"']*)\s*$", + re.MULTILINE, +) + + +def _actionlint_values() -> dict[str, str]: + """Return `{source: value}` for every ACTIONLINT_VERSION literal + across the workflows directory. + + Actionlint is only declared as a literal env value in two files + (actionlint.yml, workflow-lint.yml), so there is no input-default + variant to scan for. + """ + found: dict[str, str] = {} + for path in sorted(WORKFLOWS_DIR.glob("*.yml")): + text = path.read_text() + for m in _ACTIONLINT_ENV_VALUE.finditer(text): + key = f"{path.name}:ACTIONLINT_VERSION(env)" + found[key] = m.group(1) + return found + + +def test_actionlint_version_agrees_across_workflows() -> None: + """Every ACTIONLINT_VERSION env value across the workflows directory + agrees. + + Same shape as the shellcheck test, applied to actionlint. The two + workflows both pin the same release so actionlint reports the same + lints when the repo lints itself as it does when it lints a + consumer's checkout. + + Real event: no actionlint version drift has been recorded, but the + failure mode being defended against is the two-of-three class + described in `test_every_rust_toolchain_default_agrees` — one + workflow's author bumps one pin, the other stays, and the two + linters start disagreeing about what the same workflow means. + """ + found = _actionlint_values() + if len(found) < 2: + pytest.fail( + f"Found only {len(found)} ACTIONLINT_VERSION occurrences: {found}. " + f"Expected at least two." + ) + distinct = set(found.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{k}={v!r}" for k, v in sorted(found.items())) + pytest.fail( + f"Actionlint version disagrees across workflows: {breakdown}. " + f"actionlint.yml and workflow-lint.yml must pin the same release." + )