From 5b3d28b8c8b9ddd115544a4660c747a7148ed2c6 Mon Sep 17 00:00:00 2001 From: Jaro-c <75870284+Jaro-c@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:38:07 -0500 Subject: [PATCH 1/3] test: assert the pins that must agree across reusables actually agree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reusables declare a Rust `toolchain`, and rust-supply-chain sat on 1.97 while rust-ci and rust-audit were on 1.98 — a bump raised two of them and not the third, and nothing compared them. The consequence was not cosmetic: an SBOM was produced by a compiler the test suite never ran, which is the defect Glyndor/podup#1487 recorded. That was fixed with a comment. A comment does not fail, so this replaces it with four tests that do, on every pull request rather than weekly: - the three Rust `toolchain` defaults agree - that shared value is MAJOR.MINOR, not `stable` and not MAJOR.MINOR.PATCH. `rustup toolchain install` accepts all three and they mean different things: `stable` floats, `1.98.0` pins one patch, `1.98` pins the stable release on that minor - SHELLCHECK_VERSION agrees across the three workflows that install it - ACTIONLINT_VERSION agrees across the two that install it None of them hardcodes a version. They assert agreement, so they keep passing across bumps and fail only on divergence — which is the failure mode that actually happened. Each was watched failing before it was kept: the divergence was introduced, the test went red, the file was restored. The messages name the file that drifted rather than only reporting that one did. Refs Glyndor/podup#1526 Signed-off-by: Jaro-c <75870284+Jaro-c@users.noreply.github.com> --- .github/workflows/rust-supply-chain.yml | 13 +- tests/test_workflow_pins.py | 298 ++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 6 deletions(-) create mode 100644 tests/test_workflow_pins.py diff --git a/.github/workflows/rust-supply-chain.yml b/.github/workflows/rust-supply-chain.yml index 9abca66..f003e4b 100644 --- a/.github/workflows/rust-supply-chain.yml +++ b/.github/workflows/rust-supply-chain.yml @@ -15,12 +15,13 @@ on: # Pinned, not the runner's floating preinstalled Rust — see rust-ci.yml's # toolchain input for why an unpinned toolchain is a problem. # - # Keep this equal to rust-ci.yml's `toolchain`. It was 1.97 while rust-ci - # was already on 1.98, because the bump raised one and not the other and - # nothing compared them. A repository whose SBOM is produced by a compiler - # its test suite never ran is the defect podup#1487 recorded, and podup - # now has a test asserting its own two pins agree — nothing asserts it - # across these two reusables, so it is written here instead. + # Keep this equal to rust-ci.yml's and rust-audit.yml's `toolchain`. It + # was 1.97 while rust-ci was already on 1.98, because the bump raised + # one and not the other and nothing compared them. A repository whose + # SBOM is produced by a compiler its test suite never ran is the defect + # Glyndor/podup#1487 recorded. tests/test_workflow_pins.py asserts the + # three defaults agree; keep them in step on the bump PR so the test + # stays green. toolchain: description: Rust toolchain to pin the SBOM and license job to type: string diff --git a/tests/test_workflow_pins.py b/tests/test_workflow_pins.py new file mode 100644 index 0000000..9ec5069 --- /dev/null +++ b/tests/test_workflow_pins.py @@ -0,0 +1,298 @@ +"""Tests that pin agreement across reusables is not just a comment. + +Three reusables in `.github/workflows/` declare a Rust `toolchain` input, +two env values for shellcheck/actionlint ride on three workflows, and a +shellcheck input default travels on two. Each value is pinned on its own, +but no comment in any of the files compared them until rust-supply-chain +drifted to 1.97 while rust-ci and rust-audit were already on 1.98. A +release's SBOM and license attribution were then produced by a compiler +the test suite never ran — the defect Glyndor/podup#1487 recorded. + +A comment about "keep these equal" does not fail when one of them stops +being equal. These tests parse the workflow YAML directly (no subprocess, +no network, no `gh`) and fail the run when the pins diverge, so a future +drift trips CI on the bump PR instead of three months later on a release +artifact. + +The four invariants: + +1. The Rust `toolchain` defaults in `rust-ci.yml`, `rust-audit.yml`, and + `rust-supply-chain.yml` all agree. +2. The shared Rust toolchain value is a two-component version (`1.98`, + not `1.98.0` and not `stable`) — `rustup toolchain install` accepts + all three and they mean different things, so the shape is part of the + contract. +3. The `SHELLCHECK_VERSION` env value in `actionlint.yml` matches the + `shellcheck-version` input defaults in `workflow-lint.yml` and + `shell-ci.yml`. The shape that drifts is "two of three bumped" — + one workflow's author updates one pin, the others stay. +4. The `ACTIONLINT_VERSION` env value in `actionlint.yml` matches the + one in `workflow-lint.yml`. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + + +def _default_for(workflow: str, input_name: str) -> str: + """Return the declared default of `input_name` in `workflow`. + + Reads the workflow file with `yaml.safe_load`, walks down to + `on.workflow_call.inputs..default`, and returns the + string. Fails the test with a clear, named message if any link in + that chain is missing — never returns None silently, because a silent + None would let a missing default pass an equality check. + + YAML 1.1 reads the bare `on:` key in a GitHub Actions workflow as the + Python boolean `True`. PyYAML 6.x keeps that behaviour here, so the + parsed key is `True`; older or YAML-1.2-only resolvers can surface + it as the string `"on"` instead. Both are handled. + """ + path = WORKFLOWS_DIR / workflow + if not path.is_file(): + pytest.fail(f"{workflow}: file not found at {path}") + try: + with path.open() as fh: + doc = yaml.safe_load(fh) + except yaml.YAMLError as e: + pytest.fail(f"{workflow}: YAML parse error: {e}") + if not isinstance(doc, dict): + pytest.fail(f"{workflow}: top-level YAML is not a mapping") + # Both keys are possible across YAML resolvers; try both. + on_node = doc.get(True) if True in doc else doc.get("on") + if on_node is None: + pytest.fail(f"{workflow}: missing `on:` section") + if not isinstance(on_node, dict): + pytest.fail( + f"{workflow}: `on:` is not a mapping (got {type(on_node).__name__})" + ) + workflow_call = on_node.get("workflow_call") + if not isinstance(workflow_call, dict): + pytest.fail(f"{workflow}: missing `on.workflow_call` section") + inputs = workflow_call.get("inputs") + if not isinstance(inputs, dict): + pytest.fail(f"{workflow}: missing `on.workflow_call.inputs` section") + input_spec = inputs.get(input_name) + if not isinstance(input_spec, dict): + pytest.fail(f"{workflow}: missing input `{input_name}`") + if "default" not in input_spec: + pytest.fail(f"{workflow}: input `{input_name}` has no `default`") + default = input_spec["default"] + if not isinstance(default, str): + pytest.fail( + f"{workflow}: input `{input_name}` default is {type(default).__name__}, " + f"expected string" + ) + return default + + +# --- 1. The three Rust toolchain pins agree ------------------------------ + + +def test_every_rust_toolchain_default_agrees() -> None: + """The three Rust reusables ship the same pinned toolchain. + + Real event: rust-supply-chain.yml sat on Rust 1.97 for an extended + period while rust-ci.yml and rust-audit.yml had already moved to + 1.98, because a toolchain bump raised two and not the third and + nothing compared them. The release's SBOM and license attribution + were then generated by a compiler the test suite never ran — the + defect Glyndor/podup#1487 recorded. A test asserting agreement is + the only shape that catches that class of drift on the bump PR + rather than at release time. + + Assertion message lists every file with its value so the failure + says which one moved, not just that they differ. The expected + version is intentionally NOT hardcoded — the test is about + agreement, not about a number, so it survives the next bump and + only fails on divergence. + """ + files = ("rust-ci.yml", "rust-audit.yml", "rust-supply-chain.yml") + values = {wf: _default_for(wf, "toolchain") for wf in files} + distinct = set(values.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{wf}={v!r}" for wf, v in values.items()) + pytest.fail( + f"Rust toolchain defaults disagree: {breakdown}. " + f"All three reusables must pin the same toolchain — see " + f"rust-supply-chain.yml's toolchain comment and the regression " + f"recorded in Glyndor/podup#1487." + ) + + +def test_rust_toolchain_is_a_two_component_version() -> None: + """The shared Rust toolchain value matches `MAJOR.MINOR`. + + `rustup toolchain install` accepts three shapes that mean different + things: a floating channel like `stable` (always the latest stable + release — which is why every other input here pins an exact version), + a fully-qualified triple like `1.98.0` (a single patch release, not a + channel), and the two-component channel like `1.98` (the stable + release on that minor — what the three reusables use). The shape is + part of the contract: a bump from `1.98` to `1.98.0` silently changes + caching and reproducibility without surfacing a difference in any + input name, and a reversion to `stable` undoes the whole pinning + story. + + Every file is checked — not just one — because the failure mode is + a single-file drift while the other two still say `1.98`: reading + only the first value would let that drift pass. + + Real event: no single drift has happened here, but the equivalent + drift would be a future PR that flips one of the three from `1.98` + to `stable` and slips through review because the test the audit + called out didn't exist. + """ + files = ("rust-ci.yml", "rust-audit.yml", "rust-supply-chain.yml") + values = {wf: _default_for(wf, "toolchain") for wf in files} + bad = {wf: v for wf, v in values.items() if not re.fullmatch(r"[0-9]+\.[0-9]+", v)} + if bad: + breakdown = ", ".join(f"{wf}={v!r}" for wf, v in sorted(bad.items())) + pytest.fail( + f"Rust toolchain default is not a two-component version: " + f"{breakdown}. Expected MAJOR.MINOR like '1.98'. `stable` " + f"floats; '1.98.0' pins a single patch; only MAJOR.MINOR pins " + f"the stable release on that minor." + ) + + +# --- 2. Shellcheck version agrees across the three workflows ---------- + + +# Matches a literal env value (skips `${{ ... }}` expressions, which are +# references, not values). Captures the first non-whitespace, non-comment +# token after the colon. +_SHELLCHECK_ENV_VALUE = re.compile( + r"^\s*SHELLCHECK_VERSION:\s*([^\s\"'#][^\s\"']*|[^\s\"'{][^\s\"']*)\s*$", + re.MULTILINE, +) +# Matches an input `shellcheck-version:` block and the `default:` line +# inside it. The capture is the default value, without surrounding quotes +# or trailing whitespace. `re.MULTILINE` makes `^` and `$` line-aware. +_SHELLCHECK_INPUT_DEFAULT = re.compile( + r"^\s*shellcheck-version:\s*$\n" + r"(?:[^\n]*\n)*?" + r"^\s*default:\s*[\"']?([^\s\"'#]+)[\"']?\s*$", + re.MULTILINE, +) + + +def _shellcheck_values() -> dict[str, str]: + """Return `{source: value}` for every SHELLCHECK_VERSION literal and + shellcheck-version default across the workflows directory. + + Two patterns cover the two shapes the workflows use: a literal + `SHELLCHECK_VERSION: vX.Y.Z` env entry (actionlint.yml), and a + `shellcheck-version: ... default: "vX.Y.Z"` input block + (workflow-lint.yml, shell-ci.yml). Both are returned with a + `source` key naming the file so a divergence message points at + the actual occurrence. + """ + found: dict[str, str] = {} + for path in sorted(WORKFLOWS_DIR.glob("*.yml")): + text = path.read_text() + for m in _SHELLCHECK_ENV_VALUE.finditer(text): + key = f"{path.name}:SHELLCHECK_VERSION(env)" + found[key] = m.group(1) + for m in _SHELLCHECK_INPUT_DEFAULT.finditer(text): + key = f"{path.name}:shellcheck-version(input.default)" + found[key] = m.group(1) + return found + + +def test_shellcheck_version_agrees_across_workflows() -> None: + """Every SHELLCHECK_VERSION env value and shellcheck-version input + default across the workflows directory agrees. + + The same drift shape as the Rust toolchain: one workflow author + bumps one pin, the others stay. There is no comment that compares + these three and the same defect (two of three bumped) would land + here if any one of them drifted. + + Real event: no shellcheck version drift has been recorded, but the + failure mode this defends against is the same one + `test_every_rust_toolchain_default_agrees` was written for — an + audit missed one bump and three workflows no longer lint each other + to a compatible version. The test exists so the next bump is + blocked at the PR. + """ + found = _shellcheck_values() + if len(found) < 2: + pytest.fail( + f"Found only {len(found)} SHELLCHECK_VERSION/shellcheck-version " + f"occurrences: {found}. Expected at least two." + ) + distinct = set(found.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{k}={v!r}" for k, v in sorted(found.items())) + pytest.fail( + f"Shellcheck version disagrees across workflows: {breakdown}. " + f"All workflows must install the same shellcheck release so " + f"a workflow this repo lints for itself is also the one it " + f"ships to consumers." + ) + + +# --- 3. Actionlint version agrees across the two workflows ---------- + + +_ACTIONLINT_ENV_VALUE = re.compile( + r"^\s*ACTIONLINT_VERSION:\s*([^\s\"'#][^\s\"']*|[^\s\"'{][^\s\"']*)\s*$", + re.MULTILINE, +) + + +def _actionlint_values() -> dict[str, str]: + """Return `{source: value}` for every ACTIONLINT_VERSION literal + across the workflows directory. + + Actionlint is only declared as a literal env value in two files + (actionlint.yml, workflow-lint.yml), so there is no input-default + variant to scan for. + """ + found: dict[str, str] = {} + for path in sorted(WORKFLOWS_DIR.glob("*.yml")): + text = path.read_text() + for m in _ACTIONLINT_ENV_VALUE.finditer(text): + key = f"{path.name}:ACTIONLINT_VERSION(env)" + found[key] = m.group(1) + return found + + +def test_actionlint_version_agrees_across_workflows() -> None: + """Every ACTIONLINT_VERSION env value across the workflows directory + agrees. + + Same shape as the shellcheck test, applied to actionlint. The two + workflows both pin the same release so actionlint reports the same + lints when the repo lints itself as it does when it lints a + consumer's checkout. + + Real event: no actionlint version drift has been recorded, but the + failure mode being defended against is the two-of-three class + described in `test_every_rust_toolchain_default_agrees` — one + workflow's author bumps one pin, the other stays, and the two + linters start disagreeing about what the same workflow means. + """ + found = _actionlint_values() + if len(found) < 2: + pytest.fail( + f"Found only {len(found)} ACTIONLINT_VERSION occurrences: {found}. " + f"Expected at least two." + ) + distinct = set(found.values()) + if len(distinct) != 1: + breakdown = ", ".join(f"{k}={v!r}" for k, v in sorted(found.items())) + pytest.fail( + f"Actionlint version disagrees across workflows: {breakdown}. " + f"actionlint.yml and workflow-lint.yml must pin the same release." + ) From b0700e1e6ca2c1e3fae0ba988e1e89d789387b0f Mon Sep 17 00:00:00 2001 From: Jaro-c <75870284+Jaro-c@users.noreply.github.com> Date: Tue, 25 Aug 2026 03:10:53 -0500 Subject: [PATCH 2/3] ci: watch the nineteen tool versions pinned in this repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consumers each got a watcher; this repository, which owns the pins, had none. A hand survey on 2026-08-24 found eight of nineteen had drifted, one of them a real defect — and a hand survey is what podup#1526 says is not good enough. Every value is read from the workflow file rather than hardcoded here. A watcher carrying its own copy of the pin can disagree with reality, which is the bug it exists to prevent. The part that decides whether this gets read or muted: four pins are held below upstream on purpose — ruff, bun, gosec and cargo-llvm-cov, each for a reason recorded next to it. Reporting them every Tuesday would train the reader to close the issue unread. A held pin is reported only when the pin stops matching its hold version, meaning somebody moved it and the note is now stale. That inversion is commented so nobody "fixes" the watcher by deleting the list. It also does not hide what it could not check. PowerShell Gallery returns Atom XML and is not wired, so PSScriptAnalyzer appears in a separate table marked as not queried rather than being silently counted as current. Verified by running it against this repository with only `gh` stubbed, so crates.io, PyPI, static.rust-lang.org and endoflife.date were real: one drift reported (python 3.13 against 3.14), the four held pins silent, and the already-open case declining to open a duplicate. Moving a held pin was exercised too — ruff at 0.14.0 against a hold of 0.12.0 reports STALE HOLD. Not registered with schedule-freshness. That guard needs a successful run with event=schedule, crons fire only from the default branch, and workflow_dispatch does not count, so an entry added here would fail on day one. It gets one after the first scheduled run. Refs Glyndor/podup#1526 Signed-off-by: Jaro-c <75870284+Jaro-c@users.noreply.github.com> --- .github/workflows/pin-watch.yml | 95 ++++++ scripts/pin-watch.py | 555 ++++++++++++++++++++++++++++++++ 2 files changed, 650 insertions(+) create mode 100644 .github/workflows/pin-watch.yml create mode 100644 scripts/pin-watch.py diff --git a/.github/workflows/pin-watch.yml b/.github/workflows/pin-watch.yml new file mode 100644 index 0000000..8ebe2bf --- /dev/null +++ b/.github/workflows/pin-watch.yml @@ -0,0 +1,95 @@ +name: Pin watch + +# Self-CI for this repository: opens a single issue when any pinned tool +# version here has drifted from its upstream release. Glyndor/podup#1526 +# motivated this — the reusables each carry their own pins, and the +# hand survey on 2026-08-24 found eight of nineteen drifted, one of +# them a real defect. A hand survey is exactly what the issue says is +# not good enough. +# +# The Python script lives at `scripts/pin-watch.py` rather than as a +# heredoc inside this file, mirroring how `pin-policy-reusable.yml` +# invokes `scripts/pin-policy.py`. A heredoc here would force every +# `actionlint` run on this workflow through shellcheck against the full +# script, and the script's size (well over a hundred lines) hits a +# pathological case in shellcheck v0.11.0 that hangs the linter +# indefinitely. The separate-file form is also the more reviewable +# shape: pin-watch.py can be diffed and tested like any other Python +# module, and this workflow file stays focused on the schedule, the +# permissions and the issue it opens. +# +# Pin values are READ FROM the workflow files at runtime, never +# hardcoded. Hardcoding the pin in the watcher would mean the watcher +# can disagree with reality, which is the bug it exists to prevent. +# Where a pin appears in more than one file (rust toolchain, +# actionlint, shellcheck), the first file listed in scripts/pin-watch.py +# is the canonical source — every other file is asserted equal to it +# by tests/test_workflow_pins.py, so the watcher reads one and trusts +# the test to enforce the rest. +# +# A held pin (the four entries in HOLDS in scripts/pin-watch.py) is +# reported ONLY when the pinned value stops matching its hold version — +# meaning somebody changed the pin and the hold note is now stale. While +# the pin still matches the hold, the watcher is silent about it. +# Reporting the upstream delta every Tuesday would train the reader to +# ignore the issue, which is the failure mode the hold list exists to +# prevent. Do not "fix" the silence by deleting the entry. +# +# Schedule: Tuesday 06:00 UTC. Deliberately not Monday: podup's own +# pin-watch runs Monday 05:00, and stacking this repository's other +# scheduled work on a single minute is the failure mode +# schedule-freshness.yml exists to detect (see 2026-06-29 in the header +# of that reusable). +# +# Reporting shape, after podup's pin-watch: collect all drifts, open +# ONE issue titled exactly `Pinned tool versions have drifted`, skip if +# an issue with that exact title is already open. Labels: `type:ci`, +# `prio:P2`, `status:ready`, `area:ci`, `effort:S`. Exit 0 with a short +# confirmation when nothing drifted. +# +# PSScriptAnalyzer's upstream endpoint +# (powershellgallery.com/api/v2/FindPackagesById()) returns an Atom XML +# feed and is awkward to parse reliably; pssa-version is therefore read +# (so the pin is still reviewed) but its upstream is not queried, and +# the entry sits in SKIP_UPSTREAM in scripts/pin-watch.py. If a future +# maintainer wires a cleaner endpoint, drop the skip and add a fetcher. +# +# Default shell is pinned to bash: the inner `sort -V` style +# comparators the upstream fetchers use are GNU semantics, and a future +# Windows caller would otherwise silently mis-compare and fail closed +# on an empty string. + +on: + schedule: + - cron: '0 6 * * 2' + workflow_dispatch: + +permissions: + contents: read + issues: write + +defaults: + run: + shell: bash + +concurrency: + group: pin-watch-${{ github.ref }} + cancel-in-progress: true + +jobs: + pin-watch: + name: pin watch + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Detect drift and open a tracking issue + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + python3 -u scripts/pin-watch.py diff --git a/scripts/pin-watch.py b/scripts/pin-watch.py new file mode 100644 index 0000000..3051838 --- /dev/null +++ b/scripts/pin-watch.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +"""Read every pinned tool version in this repository from its workflow file +and report any that have drifted from their upstream release. + +Pins are READ FROM the workflow files at runtime, never hardcoded. Where a +pin appears in more than one file (rust toolchain, actionlint, shellcheck), +the first file in PIN_SOURCES below is the canonical source — every other +file is asserted equal to it by tests/test_workflow_pins.py, so this script +reads one and trusts the test to enforce the rest. Hardcoding the pin in +the watcher would mean the watcher can disagree with reality, which is the +bug the watcher exists to prevent. + +A held pin (the four entries in HOLDS below) is reported ONLY when the +pinned value stops matching its hold version — meaning somebody changed +the pin and the hold note is now stale. While the pin still matches the +hold, the watcher is silent about it. Reporting the upstream delta every +Tuesday would train the reader to ignore the issue, which is the failure +mode the hold list exists to prevent. + +Upstream sources: + +- crates.io https://crates.io/api/v1/crates/ -> .crate.max_stable_version +- PyPI https://pypi.org/pypi//json -> .info.version +- GitHub gh release list --repo --limit 1 --json tagName --jq '.[0].tagName' +- rust https://static.rust-lang.org/dist/channel-rust-stable.toml, + the version under [pkg.rust]; compared only at MAJOR.MINOR +- python https://endoflife.date/api/python.json -> [0].cycle +- PSScriptAnalyzer — endpoint returns Atom XML; awkward to parse reliably, + so pssa-version is read but its upstream is not queried. + +Reporting: collect all drifts, open ONE issue titled exactly "Pinned tool +versions have drifted", skip if an issue with that exact title is already +open. Labels: type:ci, prio:P2, status:ready, area:ci, effort:S. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +WORKFLOWS_DIR = Path(".github/workflows") + +# Held pins: silent while the pinned value matches the hold version +# recorded here. The only time a held pin shows up in the report is when +# somebody changed the pin and forgot to update this list — the hold note +# is then stale and the next person needs to know. Do not "fix" the +# silence by deleting the entry; that trains the reader to ignore the +# report and is the failure mode the hold list exists to prevent. +HOLDS: dict[str, tuple[str, str]] = { + "ruff": ( + "0.12.0", + "0.16.4 reports 15 findings that are code changes in five files; PLW1510 needs check=False, not check=True", + ), + "bun": ( + "1.3.14", + "consumers epistle-panel and specio cannot be exercised from here", + ), + "gosec": ("v2.27.1", "consumer authcore cannot be exercised from here"), + "cargo-llvm-cov": ( + "0.8.7", + "five consumers; a coverage tool's bump can move numbers into a threshold", + ), +} + +# The single canonical source for each pin. Where a name appears in more +# than one file, tests/test_workflow_pins.py already asserts equality, so +# the FIRST file in the list is the one this watcher reads. Adding a new +# file to a list should be paired with a pin-agreement test, or two +# workflows will silently drift. +# +# kind=env reads a literal `KEY: value` line in a job's env: +# kind=input reads an on.workflow_call.inputs..default. +PIN_SOURCES: list[tuple[str, str, str, str]] = [ + ("actionlint", "actionlint.yml", "env", "ACTIONLINT_VERSION"), + ("shellcheck", "actionlint.yml", "env", "SHELLCHECK_VERSION"), + ("PyYAML", "docs-current.yml", "env", "PYYAML_VERSION"), + ("bun", "bun-ci.yml", "input", "bun-version"), + ("govulncheck", "go-audit.yml", "input", "govulncheck-version"), + ("gosec", "go-audit.yml", "input", "gosec-version"), + ("pssa-version", "powershell-ci.yml", "input", "pssa-version"), + ("python", "python-ci.yml", "input", "python-version"), + ("ruff", "python-ci.yml", "input", "ruff-version"), + ("pytest", "python-ci.yml", "input", "pytest-version"), + ("rust", "rust-ci.yml", "input", "toolchain"), + ("cargo-audit", "rust-audit.yml", "input", "cargo-audit-version"), + ("cargo-deny", "rust-audit.yml", "input", "cargo-deny-version"), + ("cargo-llvm-cov", "rust-ci.yml", "input", "llvm-cov-version"), + ("cargo-semver-checks", "rust-ci.yml", "input", "semver-checks-version"), + ("cargo-fuzz", "rust-fuzz.yml", "input", "cargo-fuzz-version"), + ("cargo-cyclonedx", "rust-supply-chain.yml", "input", "cyclonedx-version"), + ("cargo-about", "rust-supply-chain.yml", "input", "cargo-about-version"), +] + +# Read the pin, but do not query upstream. See the module docstring for +# why PSScriptAnalyzer is on this list. +SKIP_UPSTREAM: set[str] = {"pssa-version"} + + +def _read_env_value(text: str, env_key: str) -> str | None: + """Read the literal `KEY: value` env entry from a workflow file. + + Accepts both bare (`KEY: v0.11.0`) and quoted (`KEY: "v0.11.0"`) forms. + Returns the value, or None if no such line is found. The regex matches + one line at a time; the alternative captures let it accept either + quoting style without dragging in a YAML parser. + """ + pat = re.compile( + r"^\s*" + re.escape(env_key) + r":\s*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s\"'{][^\s\"']*))" + r"\s*(?:#.*)?$", + re.MULTILINE, + ) + for m in pat.finditer(text): + v = ( + m.group(1) + if m.group(1) is not None + else (m.group(2) if m.group(2) is not None else m.group(3)) + ) + if v is not None: + return v + return None + + +def _read_input_default(text: str, input_name: str) -> str | None: + """Read the `default:` value of a named input from a workflow file. + + The input block is `input_name:` followed by an indented series of + keys. The `default:` value can be bare (`default: 1.2.3`) or quoted + (`default: "1.2.3"`); both are accepted. Returns the value, or None + if not found. + + This walks the file line-by-line rather than YAML-parsing the whole + document, so the watcher does not depend on having a particular + PyYAML version installed — the pin PyYAML would itself need to be + read first, and that is a bootstrap problem the workflow does not + need. + """ + lines = text.splitlines() + in_block = False + block_indent: int | None = None + for line in lines: + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + indent = len(line) - len(stripped) + if not in_block: + if stripped == f"{input_name}:" or stripped.startswith(f"{input_name}:"): + # `with:` blocks also use `:` at column zero; + # distinguish by checking that the following real line is + # MORE indented (a block) rather than less-or-equal (a + # scalar or end of an outer block). + idx = lines.index(line) + nxt = next( + ( + nxt_line + for nxt_line in lines[idx + 1 :] + if nxt_line.strip() and not nxt_line.lstrip().startswith("#") + ), + None, + ) + if nxt is None: + return None + if (len(nxt) - len(nxt.lstrip())) <= indent: + return None + in_block = True + block_indent = indent + else: + continue + else: + assert block_indent is not None + if indent <= block_indent: + return None + if stripped.startswith("default:"): + value_part = stripped[len("default:") :].strip() + if (value_part.startswith('"') and value_part.endswith('"')) or ( + value_part.startswith("'") and value_part.endswith("'") + ): + value_part = value_part[1:-1] + return value_part + return None + + +def read_pins() -> dict[str, tuple[str, str]]: + """Read every pinned value from its workflow file. + + Returns {pin_name: (source_file, value)}. Fails the run if a pin + cannot be read — the watcher must not silently skip a pin it was + asked to read. + """ + cache: dict[str, str] = {} + pins: dict[str, tuple[str, str]] = {} + for pin_name, source_file, kind, key in PIN_SOURCES: + if source_file not in cache: + cache[source_file] = (WORKFLOWS_DIR / source_file).read_text() + text = cache[source_file] + if kind == "env": + v = _read_env_value(text, key) + elif kind == "input": + v = _read_input_default(text, key) + else: + raise AssertionError(f"unknown kind: {kind!r}") + if v is None: + print( + f"::error file={source_file}::could not read " + f"{kind} {key} (pin: {pin_name})", + file=sys.stderr, + ) + sys.exit(1) + pins[pin_name] = (source_file, v) + return pins + + +# --- Upstream fetchers --------------------------------------------------- + +UA = "glyndor-pin-watch/1.0 (+https://github.com/Glyndor/.github)" + + +def _http_get_json(url: str) -> object: + req = urllib.request.Request( + url, + headers={"User-Agent": UA, "Accept": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def _http_get_text(url: str) -> str: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8") + + +def upstream_crates_io(crate: str) -> str: + body = _http_get_json(f"https://crates.io/api/v1/crates/{crate}") + v = body.get("crate", {}).get("max_stable_version") # type: ignore[union-attr] + if not v: + raise RuntimeError(f"no max_stable_version for {crate}") + return v + + +def upstream_pypi(project: str) -> str: + body = _http_get_json(f"https://pypi.org/pypi/{project}/json") + v = body.get("info", {}).get("version") # type: ignore[union-attr] + if not v: + raise RuntimeError(f"no info.version for {project}") + return v + + +def upstream_gh_release(repo: str) -> str: + out = subprocess.check_output( + [ + "gh", + "release", + "list", + "--repo", + repo, + "--limit", + "1", + "--json", + "tagName", + "--jq", + ".[0].tagName", + ], + text=True, + ).strip() + if not out: + raise RuntimeError(f"no releases found for {repo}") + return out + + +def upstream_rust_stable() -> str: + body = _http_get_text( + "https://static.rust-lang.org/dist/channel-rust-stable.toml", + ) + # Locate the [pkg.rust] section; the version= line immediately after + # it is the current stable. Everything else in the file is per-target + # metadata and not relevant here. + m = re.search( + r"^\[pkg\.rust\]\s*$(.*?)(?=^\[|\Z)", + body, + re.MULTILINE | re.DOTALL, + ) + if not m: + raise RuntimeError("no [pkg.rust] section in channel-rust-stable.toml") + # The version= line has the shape `version = "1.98.0 (88d9e12ae 2026-08-18)"` + # — the MAJOR.MINOR.PATCH is the first whitespace-separated token + # inside the quotes, the rest is the channel's git describe metadata. + # Match just the first token; the strict MAJOR.MINOR.PATCH regex + # below rejects anything else, so a future change to the file's + # shape surfaces as a hard error rather than a silent miscompare. + v_m = re.search(r'version\s*=\s*"(\d+\.\d+\.\d+)', m.group(1)) + if not v_m: + raise RuntimeError("no version= under [pkg.rust]") + return v_m.group(1) + + +def upstream_python_eol() -> str: + body = _http_get_json("https://endoflife.date/api/python.json") + cycle = body[0].get("cycle") # type: ignore[index] + if not cycle: + raise RuntimeError("no cycle[0] in endoflife.date response") + return cycle + + +FETCHERS: dict = { + "cargo-audit": lambda: upstream_crates_io("cargo-audit"), + "cargo-deny": lambda: upstream_crates_io("cargo-deny"), + "cargo-llvm-cov": lambda: upstream_crates_io("cargo-llvm-cov"), + "cargo-semver-checks": lambda: upstream_crates_io("cargo-semver-checks"), + "cargo-fuzz": lambda: upstream_crates_io("cargo-fuzz"), + "cargo-cyclonedx": lambda: upstream_crates_io("cargo-cyclonedx"), + "cargo-about": lambda: upstream_crates_io("cargo-about"), + "PyYAML": lambda: upstream_pypi("PyYAML"), + "ruff": lambda: upstream_pypi("ruff"), + "pytest": lambda: upstream_pypi("pytest"), + "actionlint": lambda: upstream_gh_release("rhysd/actionlint"), + "shellcheck": lambda: upstream_gh_release("koalaman/shellcheck"), + "bun": lambda: upstream_gh_release("oven-sh/bun"), + "gosec": lambda: upstream_gh_release("securego/gosec"), + "govulncheck": lambda: upstream_gh_release("golang/vuln"), + "rust": upstream_rust_stable, + "python": upstream_python_eol, +} + + +def fetch_upstream(pin_name: str) -> str: + fetcher = FETCHERS.get(pin_name) + if fetcher is None: + raise RuntimeError(f"no fetcher for {pin_name}") + return fetcher() + + +# --- Reporting ----------------------------------------------------------- + +ISSUE_TITLE = "Pinned tool versions have drifted" +ISSUE_LABELS = ("type:ci", "prio:P2", "status:ready", "area:ci", "effort:S") + + +def compare(pin_name: str, pin_value: str, upstream: str) -> bool: + """Return True if pin and upstream are equal for this pin's contract. + + Most pins compare as exact strings. The Rust toolchain pin is + MAJOR.MINOR (e.g. "1.98"); upstream reports MAJOR.MINOR.PATCH (e.g. + "1.98.0"). Compare only the two-component prefix — the third + component is a patch that the pin's contract deliberately does not + name. + """ + if pin_name == "rust": + pin_mm = ".".join(pin_value.split(".")[:2]) + up_mm = ".".join(upstream.split(".")[:2]) + return pin_mm == up_mm + return pin_value == upstream + + +def render_body( + drift: list[tuple[str, str, str, str]], + held_stale: list[tuple[str, str, str, str, str]], + skipped: list[tuple[str, str, str, str]], +) -> str: + lines: list[str] = [] + lines.append( + "The following tool versions pinned in this repository " + "have drifted from their upstream releases." + ) + lines.append("") + lines.append("| Pin | File | Pinned | Upstream |") + lines.append("|---|---|---|---|") + for name, src, pin, up in drift: + lines.append(f"| `{name}` | `{src}` | `{pin}` | `{up}` |") + lines.append("") + if held_stale: + lines.append("## Hold notes needing update") + lines.append("") + lines.append( + "The following pins are deliberately held below upstream, but " + "the pinned value no longer matches the hold version listed " + "in `.github/workflows/pin-watch.yml`'s `HOLDS` table — " + "somebody changed the pin and the hold note is now stale. " + "Update `HOLDS` to match the new pin (or restore the " + "original pin if the change was unintentional)." + ) + lines.append("") + lines.append("| Pin | File | Pinned (now) | Hold says | Reason |") + lines.append("|---|---|---|---|---|") + for name, src, pin, hold, reason in held_stale: + lines.append(f"| `{name}` | `{src}` | `{pin}` | `{hold}` | {reason} |") + lines.append("") + if skipped: + lines.append("## Upstream check skipped") + lines.append("") + lines.append( + "The following pins were read but their upstream was not " + "queried this run (transient or by design). They are NOT in " + "the drift table above and are listed here only for " + "visibility." + ) + lines.append("") + lines.append("| Pin | File | Pinned | Reason |") + lines.append("|---|---|---|---|") + for name, src, pin, reason in skipped: + lines.append(f"| `{name}` | `{src}` | `{pin}` | {reason} |") + lines.append("") + lines.append("---") + lines.append("") + lines.append( + "_Opened by the `pin-watch` self-CI workflow. Closing this " + "issue without acting is the wrong response — bump the pinned " + "versions in the named workflow files (or update the hold note " + "in `.github/workflows/pin-watch.yml`), then close the issue " + "once the next scheduled run reports nothing._" + ) + return "\n".join(lines) + "\n" + + +def issue_already_open(title: str) -> bool: + """Return True if an open issue with this exact title exists.""" + out = subprocess.check_output( + [ + "gh", + "issue", + "list", + "--state", + "open", + "--search", + f"in:title {title}", + "--json", + "title", + "--jq", + ".[] | .title", + ], + text=True, + ) + titles = [line.strip() for line in out.splitlines() if line.strip()] + return title in titles + + +def open_issue(title: str, body: str) -> str: + cmd = ["gh", "issue", "create", "--title", title, "--body", body] + for label in ISSUE_LABELS: + cmd.extend(["--label", label]) + out = subprocess.check_output(cmd, text=True).strip() + return out + + +# --- Main flow ----------------------------------------------------------- + + +def main() -> int: + pins = read_pins() + print("Pins read from workflow files:") + for name in sorted(pins): + src, v = pins[name] + print(f" {name:22s} {v!r:18s} ({src})") + print() + + drift: list[tuple[str, str, str, str]] = [] + held_stale: list[tuple[str, str, str, str, str]] = [] + fetch_errors: list[tuple[str, str, str, str]] = [] + skipped: list[tuple[str, str, str, str]] = [] + + for name, (src, pin) in pins.items(): + if name in SKIP_UPSTREAM: + skipped.append( + ( + name, + src, + pin, + "upstream endpoint not wired (see SKIP_UPSTREAM in pin-watch.yml)", + ) + ) + continue + if name in HOLDS: + hold_v, hold_reason = HOLDS[name] + if pin != hold_v: + held_stale.append((name, src, pin, hold_v, hold_reason)) + continue + try: + upstream = fetch_upstream(name) + except ( + subprocess.CalledProcessError, + RuntimeError, + urllib.error.URLError, + json.JSONDecodeError, + OSError, + ) as e: + fetch_errors.append((name, src, pin, str(e))) + continue + if not compare(name, pin, upstream): + drift.append((name, src, pin, upstream)) + + print("Upstream results (non-held, non-skipped pins):") + for name in sorted(pins): + if name in SKIP_UPSTREAM or name in HOLDS: + continue + src, pin = pins[name] + if any(d[0] == name for d in drift): + up = next(d[3] for d in drift if d[0] == name) + print(f" {name:22s} pin={pin!r:18s} upstream={up!r:18s} DRIFT") + elif any(e[0] == name for e in fetch_errors): + err = next(e[3] for e in fetch_errors if e[0] == name) + print(f" {name:22s} pin={pin!r:18s} upstream=") + else: + print(f" {name:22s} pin={pin!r:18s} upstream=") + print() + + print("Held pins (silent while pin matches hold):") + for name in sorted(HOLDS): + if name not in pins: + continue + src, pin = pins[name] + hold_v, _ = HOLDS[name] + if pin == hold_v: + print( + f" {name:22s} pin={pin!r:14s} hold={hold_v!r:14s} " + "SILENT (pin matches hold)" + ) + else: + print( + f" {name:22s} pin={pin!r:14s} hold={hold_v!r:14s} " + "STALE HOLD (expected to be reported)" + ) + print() + + if fetch_errors: + print("Upstream fetch errors:") + for name, src, pin, err in fetch_errors: + print(f" {name}: {err}") + print() + + if not drift and not held_stale: + print("No drift detected. pin-watch is current.") + return 0 + + body = render_body(drift, held_stale, skipped) + print("---") + print("Issue body:") + print(body) + print("---") + + if issue_already_open(ISSUE_TITLE): + print(f"Issue {ISSUE_TITLE!r} is already open — not creating a duplicate.") + return 0 + + url = open_issue(ISSUE_TITLE, body) + print(f"Opened: {url}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From c4457f8a3a2ea9c7e45ed35b0de65f121fc99a17 Mon Sep 17 00:00:00 2001 From: Jaro-c <75870284+Jaro-c@users.noreply.github.com> Date: Tue, 25 Aug 2026 03:15:14 -0500 Subject: [PATCH 3/3] ci: give the test job PyYAML, and bump this repository's own pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failures on the previous push, both legitimate. `tests / Lint & test` died with `ModuleNotFoundError: No module named 'yaml'`. python-ci installs ruff and pytest and nothing else, and the new pin-agreement tests parse the reusables as YAML. It passed locally because my container had PyYAML and the runner does not — the harness carried a dependency the real job never had. python-ci already installs `requirements.txt` when the file exists, so that is the fix rather than widening the reusable for every consumer. The file pins pyyaml, which makes it a hand-typed pin like the nineteen in the workflows — so pin-watch reads it too. A watcher that skips the pin its own repository just added is the failure it exists to prevent. Reading it needed a new source kind and a path that resolves at the repository root rather than under .github/workflows. `pin-policy` reported that this repository's own `python-ci` pin points at a surface that differs from v1.21.0 — true, because v1.21.0 is where python-ci changed. The three self-pins move to v1.21.0, which also means the test job now runs the pytest and ruff versions that release ships. Refs Glyndor/podup#1526 Signed-off-by: Jaro-c <75870284+Jaro-c@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- requirements.txt | 11 +++++++++++ scripts/pin-watch.py | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d7ad20..2603f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ permissions: jobs: empty-diff: if: github.event_name == 'pull_request' - uses: Glyndor/.github/.github/workflows/empty-diff.yml@c958978687af37dc2d826a967d9c549589afb39f # v1.14.1 + uses: Glyndor/.github/.github/workflows/empty-diff.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 pin-policy: - uses: Glyndor/.github/.github/workflows/pin-policy-reusable.yml@8517d94fde00466f59b95c5d1bd838e2ad24a8bf # v1.15.2 + uses: Glyndor/.github/.github/workflows/pin-policy-reusable.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 # Lint and test the Python scripts that this repository ships. The # job name is `tests`; python-ci.yml's inner job name is `Lint & test`; @@ -44,7 +44,7 @@ jobs: # rename that breaks the four consumer repositories. The 90% gate is # a follow-up after the rename. tests: - uses: Glyndor/.github/.github/workflows/python-ci.yml@7099f8a9c8be91fa1732fe2402268b338771233f # v1.15.0 + uses: Glyndor/.github/.github/workflows/python-ci.yml@d5041b038b7d4935e75400dff612962e6fa14893 # v1.21.0 with: coverage-threshold: 0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e29dc63 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# Test dependencies for this repository's own suite. +# +# python-ci installs this file when it exists, which is how tests/ gets PyYAML. +# tests/test_workflow_pins.py parses the reusables as YAML rather than by regex, +# because the values it compares are quoted inconsistently across the files and a +# regex that tolerates every form is harder to trust than a parser. +# +# This is a hand-typed pin like the nineteen in the workflows, so pin-watch +# watches it too — the alternative is a watcher that misses the pin its own +# repository added. +pyyaml==6.0.3 diff --git a/scripts/pin-watch.py b/scripts/pin-watch.py index 3051838..e5c1781 100644 --- a/scripts/pin-watch.py +++ b/scripts/pin-watch.py @@ -75,8 +75,13 @@ # # kind=env reads a literal `KEY: value` line in a job's env: # kind=input reads an on.workflow_call.inputs..default. +# kind=requirements reads a `name==version` line in requirements.txt, which +# python-ci installs when the file exists. It is a hand-typed pin like the +# others, so it is watched like the others — a watcher that skips the pin its +# own repository added is the failure this file exists to prevent. PIN_SOURCES: list[tuple[str, str, str, str]] = [ ("actionlint", "actionlint.yml", "env", "ACTIONLINT_VERSION"), + ("pyyaml-test-dep", "requirements.txt", "requirements", "pyyaml"), ("shellcheck", "actionlint.yml", "env", "SHELLCHECK_VERSION"), ("PyYAML", "docs-current.yml", "env", "PYYAML_VERSION"), ("bun", "bun-ci.yml", "input", "bun-version"), @@ -185,6 +190,22 @@ def _read_input_default(text: str, input_name: str) -> str | None: return None +def _read_requirement_pin(text: str, name: str) -> str | None: + """Read `name==version` out of a requirements file. + + Only the `==` form is accepted. A range or a bare name is not a pin, and + reporting drift against something unpinned would be noise. + """ + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + left, sep, right = line.partition("==") + if sep and left.strip().lower() == name.lower(): + return right.strip() + return None + + def read_pins() -> dict[str, tuple[str, str]]: """Read every pinned value from its workflow file. @@ -196,12 +217,22 @@ def read_pins() -> dict[str, tuple[str, str]]: pins: dict[str, tuple[str, str]] = {} for pin_name, source_file, kind, key in PIN_SOURCES: if source_file not in cache: - cache[source_file] = (WORKFLOWS_DIR / source_file).read_text() + # Most pins live under .github/workflows/; requirements.txt is at the + # repository root, so resolve relative to the root when the name is + # not a workflow. + path = ( + Path(source_file) + if not source_file.endswith((".yml", ".yaml")) + else WORKFLOWS_DIR / source_file + ) + cache[source_file] = path.read_text() text = cache[source_file] if kind == "env": v = _read_env_value(text, key) elif kind == "input": v = _read_input_default(text, key) + elif kind == "requirements": + v = _read_requirement_pin(text, key) else: raise AssertionError(f"unknown kind: {kind!r}") if v is None: @@ -316,6 +347,7 @@ def upstream_python_eol() -> str: "cargo-cyclonedx": lambda: upstream_crates_io("cargo-cyclonedx"), "cargo-about": lambda: upstream_crates_io("cargo-about"), "PyYAML": lambda: upstream_pypi("PyYAML"), + "pyyaml-test-dep": lambda: upstream_pypi("PyYAML"), "ruff": lambda: upstream_pypi("ruff"), "pytest": lambda: upstream_pypi("pytest"), "actionlint": lambda: upstream_gh_release("rhysd/actionlint"),