From 975d37ff4b878522cb45d6ae9d2c00c513cb58ad Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 23:05:15 -0400 Subject: [PATCH 01/10] Move publish-db out of the pull-request workflow ci.yml runs on pull requests and installs and runs PyPI packages and each certified variant's pins, yet its push-only publish-db job named the Supabase secrets. A runner receives a secret its job names even in a step that never runs, and GitHub scrubs only the secrets not referenced in the workflow, so those runners may have held the key. publish-db.yml now publishes the database after CI completes on a push to main (workflow_run), downloading the artifact CI's pytest job uploaded. Its one job has actions: read, runs no package code, and runs only for ci.yml's own pushes to main. Unset secrets still skip with a notice, a failed upload is still a warning, and curl stays silent. tests/test_ci_gates.py: no workflow that runs on pull requests names any secret. tests/test_publish_db.py pins the trigger, the guard, the artifact and the step scripts' behaviour under stubs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 58 +---- .github/workflows/publish-db.yml | 131 +++++++++++ tests/test_ci_gates.py | 227 +++++++++++++++--- tests/test_publish_db.py | 391 +++++++++++++++++++++++++++++++ 4 files changed, 727 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/publish-db.yml create mode 100644 tests/test_publish_db.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb71dd5bc..9e807f55d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,11 @@ concurrency: # GITHUB_TOKEN reads the repository and nothing else in every job without # its own block. The repository's default is write, which the pytest job # would otherwise hold (persisted by checkout) while it runs PyPI packages -# on pushes to main. Uploading the built database and publish-db's download -# of it from this run use the Actions runtime token, not GITHUB_TOKEN, so -# neither needs the actions scope (tests/test_ci_gates.py pins all of this). +# on pushes to main. Uploading the built database uses the Actions runtime +# token, not GITHUB_TOKEN, so it needs no actions scope. publish-db.yml +# publishes it after this workflow completes on a push to main: it holds the +# Supabase secrets, and no workflow that runs on pull requests names any +# secret (tests/test_ci_gates.py pins all of this). permissions: contents: read @@ -65,6 +67,7 @@ jobs: PYTHONPATH: . run: bash .github/scripts/uk_replay_regenerate.sh - run: test -z "$(git status --porcelain)" + # publish-db.yml downloads it from this run by this name. - name: Upload built database uses: actions/upload-artifact@v4 with: @@ -235,52 +238,3 @@ jobs: - run: bun run lint - run: bun test src - run: bun run build - - publish-db: - name: publish db to supabase storage - needs: python - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v4 - with: - name: scorecard-db-${{ github.sha }} - # Storage bucket `scorecard-artifacts` on the PE Supabase project: - # every main commit's built DB (gzipped ~6 MB from ~58 MB — the - # project is on the free plan, 50 MB per-object cap) at .db.gz - # plus a moving latest.db.gz, so consumers fetch + gunzip instead - # of rebuilding. Skips with a notice until the - # SUPABASE_SCORECARD_URL/KEY secrets are set. - # - # A failed upload is a warning, not a failed run: nothing in this - # repository reads the published copy, and the built database is - # already attached to the run as an artifact. Every main run from - # 6 October 2026 failed here with curl exit 6 (the storage host no - # longer resolves; the last upload that worked was 28 September), - # which hid whether main was otherwise green. curl stays silent so - # the host name in the secret URL is never printed. - - run: | - if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then - echo "::notice::Supabase secrets not set — skipping publish" - exit 0 - fi - gzip -9 -c scorecard.db > scorecard.db.gz - for name in "${GITHUB_SHA}.db.gz" "latest.db.gz"; do - status=0 - curl -sf -X POST \ - "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ - -H "Authorization: Bearer $SUPABASE_KEY" \ - -H "Content-Type: application/gzip" \ - -H "x-upsert: true" \ - --data-binary @scorecard.db.gz > /dev/null || status=$? - if [ "$status" -ne 0 ]; then - message="curl exit $status uploading scorecard-artifacts/$name: the published database was not updated. This run's built database is its scorecard-db artifact." - echo "::warning title=Supabase publish failed::$message" - echo "**Supabase publish failed.** $message" >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi - echo "uploaded scorecard-artifacts/$name" - done - env: - SUPABASE_URL: ${{ secrets.SUPABASE_SCORECARD_URL }} - SUPABASE_KEY: ${{ secrets.SUPABASE_SCORECARD_KEY }} diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml new file mode 100644 index 000000000..4355886b1 --- /dev/null +++ b/.github/workflows/publish-db.yml @@ -0,0 +1,131 @@ +name: publish db + +# Publishes the database CI built for each push to main to the +# `scorecard-artifacts` bucket on the PE Supabase project, as .db.gz +# (gzipped ~6 MB from ~58 MB: the project is on the free plan, 50 MB +# per-object cap) and a moving latest.db.gz, so consumers fetch and gunzip +# instead of rebuilding. +# +# WHY IT IS NOT IN ci.yml +# +# This is the only workflow that names SUPABASE_SCORECARD_URL and +# SUPABASE_SCORECARD_KEY. ci.yml runs on pull requests and installs and runs +# PyPI packages and each certified variant's pins, as a runner user with +# passwordless sudo. A debug re-run of #215's pull-request CI showed a job's +# runner receiving a secret the job named only in a skipped step, and GitHub +# scrubs only the secrets "not referenced in the workflow". So while +# publish-db was a push-only job of ci.yml, the pull-request runners there may +# have held the key. No workflow that runs on pull requests names a secret +# now (tests/test_ci_gates.py). +# +# WHY workflow_run, NOT A REBUILD +# +# It publishes the bytes CI built and tested, once CI's pytest job has +# passed, as `needs: python` did inside ci.yml. A push workflow that rebuilt +# the database would run the project's PyPI packages in the job that holds +# the key, or in a sibling job of the workflow that names it, and would +# publish a database whose suite had not run. This job runs no package code: +# no checkout, no uv, no Python, only actions/download-artifact and the +# runner image's gh, gzip and curl. +# +# workflow_run runs main's copy of this file with the repository's secrets, +# whatever started the CI run, a fork's pull request included, and the +# `branches` filter names a branch, not an event or a repository. So the +# job runs only for ci.yml on a push to main of this repository, and no +# script interpolates the triggering run's fields: the run id and commit go +# to the scripts as environment variables (tests/test_publish_db.py). +# +# The artifact exists only if CI's pytest job got to its last step, so a +# run whose pytest job failed or was cancelled publishes nothing, with a +# notice. Any other CI conclusion still publishes, as it did: a newer push +# cancels the older run, often after its pytest job has passed. Re-running +# an older main CI run publishes its database again, as latest.db.gz too. +# +# Skips with a notice when the secrets are unset. A failed upload is a +# warning, not a failed run: nothing in this repository reads the published +# copy, and CI's run keeps the built database as an artifact for 30 days. +# Every publish from 6 October 2026 failed with curl exit 6 (the storage host +# no longer resolves; the last upload that worked was 28 September). curl +# stays silent so the host name in the secret URL is never printed. + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + +permissions: + contents: read + +# One publish at a time, so two uploads of latest.db.gz never interleave. A +# publish waiting behind a running one gives way to a newer one. +concurrency: + group: publish-db + cancel-in-progress: false + +jobs: + publish-db: + name: publish db to supabase storage + if: >- + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.path == '.github/workflows/ci.yml' + runs-on: ubuntu-latest + # Listing and downloading another run's artifact takes actions: read, + # and nothing else is needed: there is no checkout. + permissions: + actions: read + env: + CI_RUN_ID: ${{ github.event.workflow_run.id }} + CI_SHA: ${{ github.event.workflow_run.head_sha }} + steps: + - name: Find the database CI built + id: find + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + artifact="scorecard-db-$CI_SHA" + count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/artifacts?name=$artifact" \ + --jq '[.artifacts[] | select(.expired | not)] | length') + if [ "$count" -eq 0 ]; then + echo "::notice::CI run $CI_RUN_ID has no $artifact: its pytest job did not get as far as uploading it, so there is nothing to publish" + echo "found=false" >> "$GITHUB_OUTPUT" + else + echo "found=true" >> "$GITHUB_OUTPUT" + fi + - name: Download it + if: steps.find.outputs.found == 'true' + uses: actions/download-artifact@v4 + with: + name: scorecard-db-${{ github.event.workflow_run.head_sha }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Publish it + if: steps.find.outputs.found == 'true' + env: + SUPABASE_URL: ${{ secrets.SUPABASE_SCORECARD_URL }} + SUPABASE_KEY: ${{ secrets.SUPABASE_SCORECARD_KEY }} + run: | + if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then + echo "::notice::Supabase secrets not set — skipping publish" + exit 0 + fi + gzip -9 -c scorecard.db > scorecard.db.gz + for name in "${CI_SHA}.db.gz" "latest.db.gz"; do + status=0 + curl -sf -X POST \ + "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ + -H "Authorization: Bearer $SUPABASE_KEY" \ + -H "Content-Type: application/gzip" \ + -H "x-upsert: true" \ + --data-binary @scorecard.db.gz > /dev/null || status=$? + if [ "$status" -ne 0 ]; then + message="curl exit $status uploading scorecard-artifacts/$name: the published database was not updated. The database CI built is the scorecard-db-$CI_SHA artifact of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID." + echo "::warning title=Supabase publish failed::$message" + echo "**Supabase publish failed.** $message" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "uploaded scorecard-artifacts/$name" + done diff --git a/tests/test_ci_gates.py b/tests/test_ci_gates.py index b13fe4653..54bd16959 100644 --- a/tests/test_ci_gates.py +++ b/tests/test_ci_gates.py @@ -16,6 +16,11 @@ They also pin the token ci.yml's jobs hold: the repository's default GITHUB_TOKEN is write, so a job the file does not narrow would hold it. + +And they keep every secret out of the workflows that run on pull requests, +which install and run third-party packages: a runner receives a secret its +job names even in a step that never runs, and GitHub scrubs only the secrets +"not referenced in the workflow". """ import re @@ -25,23 +30,91 @@ import yaml ROOT = Path(__file__).resolve().parent.parent -CI = ROOT / ".github" / "workflows" / "ci.yml" -DETERMINISM = ROOT / ".github" / "workflows" / "uk-replay-determinism.yml" -FRESHNESS = ROOT / ".github" / "workflows" / "gate-freshness.yml" +WORKFLOWS = ROOT / ".github" / "workflows" +CI = WORKFLOWS / "ci.yml" +DETERMINISM = WORKFLOWS / "uk-replay-determinism.yml" +FRESHNESS = WORKFLOWS / "gate-freshness.yml" +PUBLISH = WORKFLOWS / "publish-db.yml" READ = {"contents": "read"} LEVEL = {"none": 0, "read": 1, "write": 2} # A ci.yml job that needs more than READ goes here with exactly the scopes it # needs, and why. None does: checkout fetches this repository, setup-uv and -# setup-bun read public releases, and the database artifact moves within one -# run on the Actions runtime token, not GITHUB_TOKEN. +# setup-bun read public releases, and the database artifact is uploaded on +# the Actions runtime token, not GITHUB_TOKEN. NEEDS_MORE: dict[str, dict[str, str]] = {} +# publish-db.yml lists and downloads another run's artifact, CI's, which +# takes actions: read. It has no checkout, so it needs no contents scope. +PUBLISH_NEEDS = {"publish-db": {"actions": "read"}} + +PULL_REQUEST_EVENTS = {"pull_request", "pull_request_target"} +# The secrets a workflow that runs on pull requests may name: none. A debug +# re-run of #215's pull-request CI (job 119587048984) evaluated a secret its +# job named only in a skipped step, so naming one anywhere in such a workflow +# may hand it to runners that run a pull request's packages. +SECRETS_ON_PULL_REQUESTS: frozenset[str] = frozenset() +SUPABASE = {"SUPABASE_SCORECARD_URL", "SUPABASE_SCORECARD_KEY"} def ci() -> dict: return yaml.safe_load(CI.read_text()) +def workflows() -> dict[str, dict]: + return { + path.name: yaml.safe_load(path.read_text()) + for path in sorted(WORKFLOWS.glob("*.y*ml")) + } + + +def triggers(workflow: dict) -> set[str]: + """A workflow's events. PyYAML reads the bare key `on` as True.""" + on = workflow[True] + return {on} if isinstance(on, str) else set(on) + + +EXPRESSION = re.compile(r"\$\{\{(.*?)\}\}", re.S) +# The secrets context itself, not a property named secrets (inputs.secrets), +# in any case, so Secrets.x or SECRETS.X cannot slip past. +CONTEXT = re.compile(r"(? set[str]: + """The secrets ``workflow`` names, upper-cased. A reference that names + no one secret, such as toJSON(secrets), secrets[format(...)] or a + reusable-workflow call's `secrets:`, counts as "*", every secret.""" + names = set() + for expression in expressions(workflow): + matches = NAMED.findall(expression) + names |= {next(n for n in groups if n).upper() for groups in matches} + if len(CONTEXT.findall(expression)) > len(matches): + names.add("*") + if any("secrets" in job for job in workflow["jobs"].values()): + names.add("*") + return names + + def granted(workflow: dict, job: str): """The GITHUB_TOKEN scopes ``job`` gets from the file. GitHub applies the workflow's block, then the job's, and a block that names any scope sets @@ -50,13 +123,13 @@ def granted(workflow: dict, job: str): return workflow["jobs"][job].get("permissions", workflow.get("permissions")) -def overreach(workflow: dict) -> list[str]: - """Every grant beyond READ, or beyond a job's NEEDS_MORE entry.""" +def overreach(workflow: dict, needs_more=NEEDS_MORE) -> list[str]: + """Every grant beyond READ, or beyond a job's ``needs_more`` entry.""" found = [] if workflow.get("permissions") != READ: found.append(f"the workflow level is {workflow.get('permissions')!r}") for job in workflow["jobs"]: - scopes, allowed = granted(workflow, job), NEEDS_MORE.get(job, READ) + scopes, allowed = granted(workflow, job), needs_more.get(job, READ) if not isinstance(scopes, dict): found.append(f"{job} gets {scopes!r}") continue @@ -135,7 +208,8 @@ def test_the_failure_message_says_what_to_do(): def test_every_ci_job_reads_the_repository_and_nothing_more(): """The pytest job runs PyPI packages on pushes to main with checkout's persisted token. Until the workflow-level block, that job, the app job - and publish-db held the repository's write default.""" + and publish-db (in publish-db.yml since) held the repository's write + default.""" workflow = ci() assert workflow["permissions"] == READ assert set(NEEDS_MORE) <= set(workflow["jobs"]), "NEEDS_MORE names a gone job" @@ -160,7 +234,7 @@ def test_every_real_replay_job_reads_the_repository_and_nothing_more(): lambda w: w.update(permissions={**READ, "actions": "read"}), lambda w: w["jobs"]["python"].update(permissions={"contents": "write"}), lambda w: w["jobs"]["app"].update(permissions="write-all"), - lambda w: w["jobs"]["publish-db"].update( + lambda w: w["jobs"]["replay-variants"].update( permissions={**READ, "actions": "read"} ), lambda w: w["jobs"]["replay-engine"].update(permissions={"id-token": "write"}), @@ -173,7 +247,7 @@ def test_every_real_replay_job_reads_the_repository_and_nothing_more(): "workflow-adds-actions", "python-contents-write", "app-write-all", - "publish-db-adds-actions", + "replay-variants-adds-actions", "replay-engine-id-token", ], ) @@ -186,18 +260,17 @@ def test_the_check_catches_every_wider_token(edit): def test_the_check_lets_a_job_narrow_its_token(): workflow = ci() workflow["jobs"]["app"]["permissions"] = {} - workflow["jobs"]["publish-db"]["permissions"] = {"contents": "none"} + workflow["jobs"]["receiver-sandbox"]["permissions"] = {"contents": "none"} assert overreach(workflow) == [] -def test_publish_db_downloads_this_runs_database_without_a_token(): - """actions/download-artifact v4 without `github-token` finds the artifact - in the current run through the Actions runtime token - (ACTIONS_RUNTIME_TOKEN), which is why publish-db needs no actions scope; - a `github-token` would need actions: read. Upload has no token input. - Re-check this when either action changes major version. No step is - handed GITHUB_TOKEN explicitly; setup-uv and setup-bun take it by - default, to read public releases.""" +def test_ci_uploads_the_database_without_a_token(): + """actions/upload-artifact v4 has no token input: it uploads through the + Actions runtime token (ACTIONS_RUNTIME_TOKEN), which is why ci.yml needs + no actions scope. Re-check this when the action changes major version. + No step is handed GITHUB_TOKEN explicitly; setup-uv and setup-bun take + it by default, to read public releases. publish-db.yml downloads the + artifact once the run completes (tests/test_publish_db.py).""" text = CI.read_text() assert "secrets.GITHUB_TOKEN" not in text and "github.token" not in text jobs = ci()["jobs"] @@ -206,15 +279,113 @@ def test_publish_db_downloads_this_runs_database_without_a_token(): for s in jobs["python"]["steps"] if s.get("uses", "").startswith("actions/upload-artifact") ] - (download,) = [ - s - for s in jobs["publish-db"]["steps"] - if s.get("uses", "").startswith("actions/download-artifact") - ] assert upload["uses"] == "actions/upload-artifact@v4" - assert download["uses"] == "actions/download-artifact@v4" - assert download["with"] == {"name": upload["with"]["name"]} - assert jobs["publish-db"]["needs"] == "python" + assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" + assert "publish-db" not in jobs + assert "download-artifact" not in text + + +def test_publish_db_reads_actions_and_nothing_more(): + workflow = yaml.safe_load(PUBLISH.read_text()) + assert granted(workflow, "publish-db") == {"actions": "read"} + assert overreach(workflow, PUBLISH_NEEDS) == [] + + +@pytest.mark.parametrize( + "scopes", + [ + "write-all", + "read-all", + {"actions": "write"}, + {"actions": "read", "contents": "write"}, + {"actions": "read", "id-token": "write"}, + ], +) +def test_the_check_catches_a_wider_publish_token(scopes): + workflow = yaml.safe_load(PUBLISH.read_text()) + workflow["jobs"]["publish-db"]["permissions"] = scopes + assert overreach(workflow, PUBLISH_NEEDS) + + +def test_no_workflow_that_runs_on_pull_requests_names_a_secret(): + """ci.yml runs PyPI packages and each certified variant's pins on pull + requests, as a user with passwordless sudo. No workflow that runs on + pull requests may name a secret, even in a job or step that never runs + for one: publish-db named the Supabase secrets inside ci.yml, push-only, + until it moved to publish-db.yml.""" + found = { + name: named_secrets(workflow) - SECRETS_ON_PULL_REQUESTS + for name, workflow in workflows().items() + if triggers(workflow) & PULL_REQUEST_EVENTS + } + assert {"ci.yml", "gate-freshness.yml"} <= set(found) + assert {name: names for name, names in found.items() if names} == {} + + +def test_only_publish_db_names_the_supabase_secrets(): + """And the scanner sees them where they are.""" + named = {name: named_secrets(w) for name, w in workflows().items()} + assert {name for name, names in named.items() if names & SUPABASE} == { + "publish-db.yml" + } + assert named["publish-db.yml"] == SUPABASE + assert not triggers(workflows()["publish-db.yml"]) & PULL_REQUEST_EVENTS + + +@pytest.mark.parametrize( + "edit", + [ + lambda w: w["jobs"]["app"]["steps"][0].update( + env={"K": "${{ secrets.SUPABASE_SCORECARD_KEY }}"} + ), + lambda w: w["jobs"]["app"]["steps"][0].update( + {"with": {"k": "${{ secrets['ANY_TOKEN'] }}"}} + ), + lambda w: w["jobs"]["app"]["steps"][0].update( + {"with": {"k": '${{ secrets["ANY_TOKEN"] }}'}} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo '${{ toJSON(secrets) }}'"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ secrets[format('{0}_KEY', 'X')] }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"if": "secrets.ANY_TOKEN != ''", "run": "true"} + ), + lambda w: w.update(env={"K": "${{ Secrets . any_token }}"}), + lambda w: w["jobs"].update( + call={"uses": "./.github/workflows/x.yml", "secrets": "inherit"} + ), + ], + ids=[ + "step-env", + "bracket-single", + "bracket-double", + "to-json", + "computed-name", + "bare-if", + "case-and-spaces", + "reusable-inherit", + ], +) +def test_the_scanner_catches_every_way_to_name_a_secret(edit): + workflow = ci() + assert named_secrets(workflow) == set() + edit(workflow) + assert named_secrets(workflow) - SECRETS_ON_PULL_REQUESTS + + +def test_the_scanner_ignores_what_names_no_secret(): + """A property called secrets, the word in a script, and comments.""" + workflow = yaml.safe_load(CI.read_text() + " # ${{ secrets.IN_A_COMMENT }}\n") + workflow["jobs"]["app"]["steps"].append( + { + "if": "github.event.inputs.secrets == 'x'", + "run": "echo 'no secrets here' ${{ github.event.inputs.secrets }}", + } + ) + assert named_secrets(workflow) == set() @pytest.mark.skipif(not (ROOT / ".git").exists(), reason="needs a git checkout") diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py new file mode 100644 index 000000000..8b5fc55c1 --- /dev/null +++ b/tests/test_publish_db.py @@ -0,0 +1,391 @@ +"""publish-db.yml: the one workflow that names the Supabase secrets. + +It publishes the database ci.yml's pytest job built, from that run's +artifact, once CI completes on a push to main. It moved out of ci.yml so that +no workflow that runs on pull requests names a secret +(tests/test_ci_gates.py). These tests pin what the move had to keep: it +publishes only for main's own pushes, only what CI built, and runs no package +code beside the key; unset secrets skip with a notice, a failed upload is a +warning, and curl never prints the host. The step scripts run here under +`bash -e`, the shell GitHub's log shows for them, with `gh` and `curl` stubs. +""" + +import gzip +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS = ROOT / ".github" / "workflows" +CI = WORKFLOWS / "ci.yml" +PUBLISH = WORKFLOWS / "publish-db.yml" + +FIND = "Find the database CI built" +DOWNLOAD = "Download it" +UPLOAD = "Publish it" +SHA = "0123456789abcdef0123456789abcdef01234567" +RUN_ID = "39698300855" +REPOSITORY = "PolicyEngine/policyengine-scorecard" +HOST = "secret-project-ref.supabase.co" +KEY = "sb_secret_key_value" + + +def workflow() -> dict: + return yaml.safe_load(PUBLISH.read_text()) + + +def job() -> dict: + return workflow()["jobs"]["publish-db"] + + +def step(name: str) -> dict: + (found,) = [s for s in job()["steps"] if s.get("name") == name] + return found + + +def test_it_runs_when_ci_completes_on_main(): + ci = yaml.safe_load(CI.read_text()) + assert workflow()[True] == { + "workflow_run": { + "workflows": [ci["name"]], + "types": ["completed"], + "branches": ["main"], + } + } + assert ci[True]["push"] == {"branches": ["main"]} + assert list(workflow()["jobs"]) == ["publish-db"] + assert workflow()["concurrency"] == { + "group": "publish-db", + "cancel-in-progress": False, + } + + +def test_it_is_the_only_workflow_run_workflow(): + """A workflow_run workflow runs with the repository's secrets for every + run it follows, pull requests' included. Another one needs the same + guard as publish-db and its own test.""" + found = { + path.name + for path in WORKFLOWS.glob("*.y*ml") + if "workflow_run" in yaml.safe_load(path.read_text())[True] + } + assert found == {"publish-db.yml"} + + +# The guard, one clause per field of the triggering run. The fields are the +# run object's: GET /repos/{owner}/{repo}/actions/runs returns `event`, +# `head_branch`, `head_repository.full_name` and `path` for this repository's +# CI runs, and the workflow_run event carries that object. +GUARD = { + "github.event.workflow_run.event == 'push'", + "github.event.workflow_run.head_branch == 'main'", + "github.event.workflow_run.head_repository.full_name == github.repository", + "github.event.workflow_run.path == '.github/workflows/ci.yml'", +} + + +def guard_admits(run: dict, repository: str = REPOSITORY) -> bool: + """The job's `if`, evaluated: a conjunction of equalities, each between + a field of the triggering run and a quoted literal or github.repository.""" + condition = job()["if"] + assert "||" not in condition and "!" not in condition + context = {"github": {"repository": repository, "event": {"workflow_run": run}}} + + def value(term): + term = term.strip() + if term.startswith("'"): + return term.strip("'") + node = context + for part in term.split("."): + node = node[part] + return node + + clauses = [c.split("==") for c in condition.split("&&")] + return all(value(left) == value(right) for left, right in clauses) + + +def ci_run(**changes) -> dict: + run = { + "event": "push", + "head_branch": "main", + "head_repository": {"full_name": REPOSITORY}, + "path": ".github/workflows/ci.yml", + } + return {**run, **changes} + + +def test_the_guard_admits_only_cis_pushes_to_main_here(): + """workflow_run runs main's copy of the file with the repository's + secrets after any CI run, a fork's pull request included, and the + `branches` filter names a branch only: a fork's pull request from a + branch called main gets past it.""" + assert {c.strip() for c in job()["if"].split("&&")} == GUARD + assert guard_admits(ci_run()) + fork = {"full_name": "someone/policyengine-scorecard"} + for refused in ( + ci_run(event="pull_request"), + ci_run(event="pull_request", head_repository=fork), + ci_run(event="pull_request_target"), + ci_run(event="workflow_dispatch"), + ci_run(head_branch="feature"), + ci_run(head_repository=fork), + ci_run(path=".github/workflows/other.yml"), + ): + assert not guard_admits(refused), refused + + +def test_it_downloads_what_ci_uploaded_for_that_commit(): + ci = yaml.safe_load(CI.read_text()) + (upload,) = [ + s + for s in ci["jobs"]["python"]["steps"] + if s.get("uses", "").startswith("actions/upload-artifact") + ] + assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" + assert upload["with"]["path"] == "data/scorecard.db" + download = step(DOWNLOAD) + assert download["uses"] == "actions/download-artifact@v4" + assert download["with"] == { + "name": "scorecard-db-${{ github.event.workflow_run.head_sha }}", + "run-id": "${{ github.event.workflow_run.id }}", + "github-token": "${{ github.token }}", + } + assert download["if"] == step(UPLOAD)["if"] == "steps.find.outputs.found == 'true'" + assert step(FIND)["id"] == "find" + assert 'artifact="scorecard-db-$CI_SHA"' in step(FIND)["run"] + assert job()["env"] == { + "CI_RUN_ID": "${{ github.event.workflow_run.id }}", + "CI_SHA": "${{ github.event.workflow_run.head_sha }}", + } + + +def test_no_script_interpolates_the_triggering_run(): + """A pull request's author chooses its branch name, and GitHub pastes a + ${{ }} into a script before the shell reads it. The scripts take the run + id and commit from the environment and interpolate nothing.""" + for s in job()["steps"]: + assert "${{" not in s.get("run", ""), s.get("name") + + +def test_the_job_runs_no_package_code(): + """No checkout and no package manager or interpreter: what runs beside + the key is actions/download-artifact and the runner image's gh, gzip + and curl.""" + assert [s["uses"] for s in job()["steps"] if "uses" in s] == [ + "actions/download-artifact@v4" + ] + commands = re.compile( + r"\b(uvx?|pip3?|python3?|bun|bunx|npm|npx|node|make|bash|sh|source)\b" + r"|\.github/scripts|\./" + ) + for s in job()["steps"]: + assert not commands.search(s.get("run", "")), s["name"] + + +def test_the_tokens_are_where_they_are_used(): + """GITHUB_TOKEN reaches the steps that read CI's run, and the Supabase + secrets the step that uploads, never the workflow or job environment.""" + with_github_token = [ + s["name"] for s in job()["steps"] if "github.token" in json.dumps(s) + ] + assert with_github_token == [FIND, DOWNLOAD] + assert step(FIND)["env"] == {"GH_TOKEN": "${{ github.token }}"} + with_secrets = [s["name"] for s in job()["steps"] if "secrets." in json.dumps(s)] + assert with_secrets == [UPLOAD] + assert step(UPLOAD)["env"] == { + "SUPABASE_URL": "${{ secrets.SUPABASE_SCORECARD_URL }}", + "SUPABASE_KEY": "${{ secrets.SUPABASE_SCORECARD_KEY }}", + } + assert "env" not in workflow() + + +# Stand-ins for the runner image's gh and curl. Each records its arguments +# and the variables the script reads; curl also keeps the bytes it would +# send. Like the real curl, a failing stub prints its error, host included, +# unless one of its short options includes -s. +GH_STUB = """#!{python} -I +import json, os, sys +with open({record!r}, "a") as handle: + handle.write(json.dumps({{"argv": sys.argv[1:], "token": os.environ.get("GH_TOKEN")}}) + "\\n") +if {status}: + sys.stderr.write("HTTP 502\\n") + sys.exit({status}) +print({count!r}) +""" +CURL_STUB = """#!{python} -I +import json, sys +args = sys.argv[1:] +payload = args[args.index("--data-binary") + 1].removeprefix("@") +with open(payload, "rb") as handle: + sent = handle.read().hex() +with open({record!r}, "a") as handle: + handle.write(json.dumps({{"argv": args, "sent": sent}}) + "\\n") +if {status}: + silent = any(a.startswith("-") and not a.startswith("--") and "s" in a for a in args) + if not silent: + url = [a for a in args if a.startswith("https://")][0] + sys.stderr.write("curl: (6) Could not resolve host: " + url.split("/")[2] + "\\n") + sys.exit({status}) +""" + + +def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): + """``name``'s script under `bash -e`, in a fresh workspace holding a + database, with gh and curl stubs first on PATH.""" + work = tmp_path / "work" + work.mkdir() + (work / "scorecard.db").write_bytes(b"SQLite format 3\x00" + bytes(range(256))) + stubs = tmp_path / "stubs" + stubs.mkdir() + records = {tool: tmp_path / f"{tool}.jsonl" for tool in ("gh", "curl")} + (stubs / "gh").write_text( + GH_STUB.format( + python=sys.executable, + record=str(records["gh"]), + count=gh_count, + status=status, + ) + ) + (stubs / "curl").write_text( + CURL_STUB.format( + python=sys.executable, record=str(records["curl"]), status=status + ) + ) + for stub in stubs.iterdir(): + stub.chmod(0o755) + path = tmp_path / "step.sh" + path.write_text(script if script is not None else step(name)["run"]) + output, summary = tmp_path / "github_output", tmp_path / "step_summary" + output.touch() + summary.touch() + environment = { + "PATH": f"{stubs}:/usr/bin:/bin", + "HOME": str(tmp_path), + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_SERVER_URL": "https://github.com", + "CI_RUN_ID": RUN_ID, + "CI_SHA": SHA, + "GH_TOKEN": "ghs_runtoken", + "SUPABASE_URL": f"https://{HOST}", + "SUPABASE_KEY": KEY, + **dict(env), + } + result = subprocess.run( + ["bash", "-e", str(path)], + cwd=work, + env=environment, + capture_output=True, + text=True, + ) + calls = { + tool: [json.loads(line) for line in record.read_text().splitlines()] + if record.exists() + else [] + for tool, record in records.items() + } + return result, calls, output.read_text(), summary.read_text(), work + + +def test_find_says_found_when_ci_uploaded_the_database(tmp_path): + result, calls, output, _, _ = run_step(tmp_path, FIND, gh_count="1") + assert result.returncode == 0, result.stderr + assert output == "found=true\n" + assert "::notice::" not in result.stdout + assert calls["gh"] == [ + { + "argv": [ + "api", + f"repos/{REPOSITORY}/actions/runs/{RUN_ID}/artifacts" + f"?name=scorecard-db-{SHA}", + "--jq", + "[.artifacts[] | select(.expired | not)] | length", + ], + "token": "ghs_runtoken", + } + ] + + +def test_find_skips_with_a_notice_when_the_pytest_job_uploaded_nothing(tmp_path): + result, calls, output, _, _ = run_step(tmp_path, FIND, gh_count="0") + assert result.returncode == 0, result.stderr + assert output == "found=false\n" + assert result.stdout.startswith( + f"::notice::CI run {RUN_ID} has no scorecard-db-{SHA}: its pytest job" + ) + assert not calls["curl"] + + +def test_find_fails_the_run_when_it_cannot_ask(tmp_path): + result, _, output, _, _ = run_step(tmp_path, FIND, status=1) + assert result.returncode != 0 + assert output == "" + + +@pytest.mark.parametrize("unset", ["SUPABASE_URL", "SUPABASE_KEY"]) +def test_publish_skips_with_a_notice_when_a_secret_is_unset(tmp_path, unset): + result, calls, _, summary, work = run_step(tmp_path, UPLOAD, env={unset: ""}) + assert result.returncode == 0, result.stderr + assert result.stdout == "::notice::Supabase secrets not set — skipping publish\n" + assert not calls["curl"] and summary == "" + assert not (work / "scorecard.db.gz").exists() + + +def test_publish_uploads_the_commits_database_then_latest(tmp_path): + result, calls, _, summary, work = run_step(tmp_path, UPLOAD) + assert result.returncode == 0, result.stderr + names = [f"{SHA}.db.gz", "latest.db.gz"] + assert result.stdout == "".join( + f"uploaded scorecard-artifacts/{n}\n" for n in names + ) + assert summary == "" + database = (work / "scorecard.db").read_bytes() + for call, name in zip(calls["curl"], names, strict=True): + assert call["argv"] == [ + "-sf", + "-X", + "POST", + f"https://{HOST}/storage/v1/object/scorecard-artifacts/{name}", + "-H", + f"Authorization: Bearer {KEY}", + "-H", + "Content-Type: application/gzip", + "-H", + "x-upsert: true", + "--data-binary", + "@scorecard.db.gz", + ] + assert gzip.decompress(bytes.fromhex(call["sent"])) == database + + +def test_a_failed_upload_is_a_warning_and_never_prints_the_host(tmp_path): + result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, status=6) + assert result.returncode == 0, result.stderr + assert len(calls["curl"]) == 1 + message = ( + f"curl exit 6 uploading scorecard-artifacts/{SHA}.db.gz: the published " + f"database was not updated. The database CI built is the " + f"scorecard-db-{SHA} artifact of " + f"https://github.com/{REPOSITORY}/actions/runs/{RUN_ID}." + ) + assert result.stdout == f"::warning title=Supabase publish failed::{message}\n" + assert summary == f"**Supabase publish failed.** {message}\n" + for shown in (result.stdout, result.stderr, summary): + assert HOST not in shown and KEY not in shown + + +def test_without_curls_silence_the_host_would_print(tmp_path): + """The stub does print the host when curl is not silenced, so the test + above is not passing for want of anything to print.""" + script = step(UPLOAD)["run"] + assert script.count("curl -sf ") == 1 + result, *_ = run_step( + tmp_path, UPLOAD, status=6, script=script.replace("curl -sf ", "curl -f ") + ) + assert HOST in result.stderr From 7b2b50adc736f2a5acad0cda0ec2ef00b41e8e4f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 23:08:58 -0400 Subject: [PATCH 02/10] Say in the database README where publishing runs and that it has failed since 6 October Co-Authored-By: Claude Opus 5.5 --- scorecard_db/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scorecard_db/README.md b/scorecard_db/README.md index 1152e8845..6964877c7 100644 --- a/scorecard_db/README.md +++ b/scorecard_db/README.md @@ -41,9 +41,13 @@ old file first; nothing is lost, it is derived). Plain `pytest` also works from a fresh clone: a pytest configuration hook builds the database before collection when it is absent. Individual ingest modules (`ingest_urban`, `ingest_platform`, …) remain runnable one at a time against an existing database, in the -dependency order documented in `build_db.py`. Built artifacts for every -main commit are published to the `scorecard-artifacts` Supabase storage -bucket as `.db.gz` and `latest.db.gz`. +dependency order documented in `build_db.py`. Each main commit's CI run +keeps the database it built and tested as its `scorecard-db-` artifact +for 30 days, and `.github/workflows/publish-db.yml` then publishes it to the +`scorecard-artifacts` Supabase storage bucket as `.db.gz` and +`latest.db.gz`. Every publish since 6 October 2026 has failed (the storage +host no longer resolves), which the workflow reports as a warning; the last +upload that worked was on 28 September. ```python from scorecard_db import ScorecardDB From 77f6945cfd4779d2aed09bcbd842baa836af09cb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 23:32:19 -0400 Subject: [PATCH 03/10] Keep latest.db.gz on main's tip, read expressions as GitHub does, pin the download From the independent review: - latest.db.gz moves only when the CI commit is still main's tip (GITHUB_SHA in a workflow_run run), so re-running an older main run no longer rolls it back; .db.gz is uploaded either way. - The secrets scanner reads each ${{ }} as the runner's TemplateReader does, so a }} inside a quoted string no longer hides a reference, and it scans every event a pull request starts. - Concurrency is on the job, so a run the guard skips never displaces a waiting publish. - download-artifact is pinned to v4.3.0's commit (what v4 names today), as uk-replay-results.yml pins the actions beside its key. - The size note: ~196 MB built, ~14 MB gzipped. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 34 +++++++++++++-------- tests/test_ci_gates.py | 52 ++++++++++++++++++++++++++++---- tests/test_publish_db.py | 38 +++++++++++++++++------ 3 files changed, 96 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index 4355886b1..1eb97b30a 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -2,7 +2,7 @@ name: publish db # Publishes the database CI built for each push to main to the # `scorecard-artifacts` bucket on the PE Supabase project, as .db.gz -# (gzipped ~6 MB from ~58 MB: the project is on the free plan, 50 MB +# (gzipped ~14 MB from ~196 MB: the project is on the free plan, 50 MB # per-object cap) and a moving latest.db.gz, so consumers fetch and gunzip # instead of rebuilding. # @@ -37,9 +37,11 @@ name: publish db # # The artifact exists only if CI's pytest job got to its last step, so a # run whose pytest job failed or was cancelled publishes nothing, with a -# notice. Any other CI conclusion still publishes, as it did: a newer push -# cancels the older run, often after its pytest job has passed. Re-running -# an older main CI run publishes its database again, as latest.db.gz too. +# notice. Any other CI conclusion still publishes .db.gz, as it did: a +# newer push cancels the older run, often after its pytest job has passed. +# latest.db.gz moves only for the commit that is still main's tip when CI +# completes (GITHUB_SHA, in a workflow_run run), so neither a cancelled run +# nor a re-run of an older one rolls it back. # # Skips with a notice when the secrets are unset. A failed upload is a # warning, not a failed run: nothing in this repository reads the published @@ -57,12 +59,6 @@ on: permissions: contents: read -# One publish at a time, so two uploads of latest.db.gz never interleave. A -# publish waiting behind a running one gives way to a newer one. -concurrency: - group: publish-db - cancel-in-progress: false - jobs: publish-db: name: publish db to supabase storage @@ -71,6 +67,12 @@ jobs: github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.path == '.github/workflows/ci.yml' + # One publish at a time, so two uploads of latest.db.gz never interleave. + # A publish waiting behind a running one gives way to a newer one. On the + # job, not the workflow, so a run the guard skips never takes a place. + concurrency: + group: publish-db + cancel-in-progress: false runs-on: ubuntu-latest # Listing and downloading another run's artifact takes actions: read, # and nothing else is needed: there is no checkout. @@ -95,9 +97,11 @@ jobs: else echo "found=true" >> "$GITHUB_OUTPUT" fi + # Pinned to a commit, as uk-replay-results.yml's propose job pins its + # actions, because it runs beside the key. - name: Download it if: steps.find.outputs.found == 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: scorecard-db-${{ github.event.workflow_run.head_sha }} run-id: ${{ github.event.workflow_run.id }} @@ -113,7 +117,13 @@ jobs: exit 0 fi gzip -9 -c scorecard.db > scorecard.db.gz - for name in "${CI_SHA}.db.gz" "latest.db.gz"; do + set -- "${CI_SHA}.db.gz" + if [ "$CI_SHA" = "$GITHUB_SHA" ]; then + set -- "$@" latest.db.gz + else + echo "::notice::main is at $GITHUB_SHA now, not $CI_SHA: latest.db.gz is left to the run for main's tip" + fi + for name in "$@"; do status=0 curl -sf -X POST \ "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ diff --git a/tests/test_ci_gates.py b/tests/test_ci_gates.py index 54bd16959..c60736e25 100644 --- a/tests/test_ci_gates.py +++ b/tests/test_ci_gates.py @@ -47,7 +47,15 @@ # takes actions: read. It has no checkout, so it needs no contents scope. PUBLISH_NEEDS = {"publish-db": {"actions": "read"}} -PULL_REQUEST_EVENTS = {"pull_request", "pull_request_target"} +# The events a pull request starts runs for: opening or updating it, a review +# or review comment on it, and adding it to a merge queue. +PULL_REQUEST_EVENTS = { + "pull_request", + "pull_request_target", + "pull_request_review", + "pull_request_review_comment", + "merge_group", +} # The secrets a workflow that runs on pull requests may name: none. A debug # re-run of #215's pull-request CI (job 119587048984) evaluated a secret its # job named only in a skipped step, so naming one anywhere in such a workflow @@ -73,7 +81,21 @@ def triggers(workflow: dict) -> set[str]: return {on} if isinstance(on, str) else set(on) -EXPRESSION = re.compile(r"\$\{\{(.*?)\}\}", re.S) +def templates(text: str) -> list[str]: + """The body of each ${{ }} in ``text``, read as GitHub's parser reads + it: a }} inside a single-quoted string literal does not end it. An + escaped quote ('') toggles twice, which leaves the state as it was.""" + found, start = [], text.find("${{") + while start != -1: + end, quoted = start + 3, False + while end < len(text) and (quoted or not text.startswith("}}", end)): + quoted ^= text[end] == "'" + end += 1 + found.append(text[start + 3 : end]) + start = text.find("${{", end + 2) + return found + + # The secrets context itself, not a property named secrets (inputs.secrets), # in any case, so Secrets.x or SECRETS.X cannot slip past. CONTEXT = re.compile(r"(? set[str]: def expressions(node, key=None): """Every expression in a parsed workflow: each ${{ }} in any key or - string, and each `if`, which GitHub evaluates with or without ${{ }}. - Comments are never evaluated, and YAML drops them.""" + string, and each `if` whole, since GitHub evaluates it with or without + ${{ }}. Comments are never evaluated, and YAML drops them.""" if isinstance(node, dict): for k, v in node.items(): yield from expressions(k) @@ -96,8 +118,7 @@ def expressions(node, key=None): for item in node: yield from expressions(item) elif isinstance(node, str): - found = EXPRESSION.findall(node) - yield from found if found or key != "if" else [node] + yield from [node] if key == "if" else templates(node) def named_secrets(workflow: dict) -> set[str]: @@ -357,6 +378,15 @@ def test_only_publish_db_names_the_supabase_secrets(): lambda w: w["jobs"].update( call={"uses": "./.github/workflows/x.yml", "secrets": "inherit"} ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ format('}}', secrets.SUPABASE_SCORECARD_KEY) }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ format('it''s }}', secrets.ANY_TOKEN) }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"if": "${{ '}}' != secrets.ANY_TOKEN }}", "run": "true"} + ), ], ids=[ "step-env", @@ -367,6 +397,9 @@ def test_only_publish_db_names_the_supabase_secrets(): "bare-if", "case-and-spaces", "reusable-inherit", + "braces-in-a-string", + "escaped-quote-then-braces", + "braces-in-an-if", ], ) def test_the_scanner_catches_every_way_to_name_a_secret(edit): @@ -376,6 +409,13 @@ def test_the_scanner_catches_every_way_to_name_a_secret(edit): assert named_secrets(workflow) - SECRETS_ON_PULL_REQUESTS +def test_the_template_reader_ends_where_github_does(): + assert templates("a ${{ x }} b ${{ y }}") == [" x ", " y "] + assert templates("${{ format('}}', s) }}") == [" format('}}', s) "] + assert templates("${{ 'it''s }}' }} tail") == [" 'it''s }}' "] + assert templates("no expression }}") == [] + + def test_the_scanner_ignores_what_names_no_secret(): """A property called secrets, the word in a script, and comments.""" workflow = yaml.safe_load(CI.read_text() + " # ${{ secrets.IN_A_COMMENT }}\n") diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index 8b5fc55c1..d9fd92760 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -28,7 +28,10 @@ FIND = "Find the database CI built" DOWNLOAD = "Download it" UPLOAD = "Publish it" +# v4.3.0, the commit actions/download-artifact's v4 tag also names. +DOWNLOAD_ACTION = "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093" SHA = "0123456789abcdef0123456789abcdef01234567" +NEWER = "fedcba9876543210fedcba9876543210fedcba98" RUN_ID = "39698300855" REPOSITORY = "PolicyEngine/policyengine-scorecard" HOST = "secret-project-ref.supabase.co" @@ -59,10 +62,9 @@ def test_it_runs_when_ci_completes_on_main(): } assert ci[True]["push"] == {"branches": ["main"]} assert list(workflow()["jobs"]) == ["publish-db"] - assert workflow()["concurrency"] == { - "group": "publish-db", - "cancel-in-progress": False, - } + # On the job, so a run its guard skips never displaces a waiting publish. + assert "concurrency" not in workflow() + assert job()["concurrency"] == {"group": "publish-db", "cancel-in-progress": False} def test_it_is_the_only_workflow_run_workflow(): @@ -91,7 +93,8 @@ def test_it_is_the_only_workflow_run_workflow(): def guard_admits(run: dict, repository: str = REPOSITORY) -> bool: """The job's `if`, evaluated: a conjunction of equalities, each between - a field of the triggering run and a quoted literal or github.repository.""" + a field of the triggering run and a quoted literal or github.repository. + GitHub compares strings ignoring case, and so does this.""" condition = job()["if"] assert "||" not in condition and "!" not in condition context = {"github": {"repository": repository, "event": {"workflow_run": run}}} @@ -106,7 +109,7 @@ def value(term): return node clauses = [c.split("==") for c in condition.split("&&")] - return all(value(left) == value(right) for left, right in clauses) + return all(value(left).lower() == value(right).lower() for left, right in clauses) def ci_run(**changes) -> dict: @@ -149,7 +152,7 @@ def test_it_downloads_what_ci_uploaded_for_that_commit(): assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" assert upload["with"]["path"] == "data/scorecard.db" download = step(DOWNLOAD) - assert download["uses"] == "actions/download-artifact@v4" + assert download["uses"] == DOWNLOAD_ACTION assert download["with"] == { "name": "scorecard-db-${{ github.event.workflow_run.head_sha }}", "run-id": "${{ github.event.workflow_run.id }}", @@ -176,9 +179,7 @@ def test_the_job_runs_no_package_code(): """No checkout and no package manager or interpreter: what runs beside the key is actions/download-artifact and the runner image's gh, gzip and curl.""" - assert [s["uses"] for s in job()["steps"] if "uses" in s] == [ - "actions/download-artifact@v4" - ] + assert [s["uses"] for s in job()["steps"] if "uses" in s] == [DOWNLOAD_ACTION] commands = re.compile( r"\b(uvx?|pip3?|python3?|bun|bunx|npm|npx|node|make|bash|sh|source)\b" r"|\.github/scripts|\./" @@ -272,6 +273,7 @@ def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): "GITHUB_SERVER_URL": "https://github.com", "CI_RUN_ID": RUN_ID, "CI_SHA": SHA, + "GITHUB_SHA": SHA, "GH_TOKEN": "ghs_runtoken", "SUPABASE_URL": f"https://{HOST}", "SUPABASE_KEY": KEY, @@ -364,6 +366,22 @@ def test_publish_uploads_the_commits_database_then_latest(tmp_path): assert gzip.decompress(bytes.fromhex(call["sent"])) == database +def test_latest_waits_for_mains_tip(tmp_path): + """GITHUB_SHA in a workflow_run run is main's tip. When main has moved on + (a newer push cancelled this run, or an older run was re-run), the + commit's own copy is uploaded and latest.db.gz is not rolled back.""" + result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, env={"GITHUB_SHA": NEWER}) + assert result.returncode == 0, result.stderr + assert result.stdout == ( + f"::notice::main is at {NEWER} now, not {SHA}: latest.db.gz is left " + f"to the run for main's tip\nuploaded scorecard-artifacts/{SHA}.db.gz\n" + ) + assert [call["argv"][3].rsplit("/", 1)[1] for call in calls["curl"]] == [ + f"{SHA}.db.gz" + ] + assert summary == "" + + def test_a_failed_upload_is_a_warning_and_never_prints_the_host(tmp_path): result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, status=6) assert result.returncode == 0, result.stderr From 6e82162c0c61d20f739ce5ff44762a3eeae0b47b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 00:19:30 -0400 Subject: [PATCH 04/10] Scan issue_comment workflows too, and say only what GitHub documents about the concurrency group From the second review: issue_comment runs are started by a comment on a pull request, so they join the scanned events. The comments on the job-level concurrency group no longer claim how GitHub treats a job its guard skips, which GitHub does not document; a workflow-level group would hold every run of the file, which it does. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 3 ++- tests/test_ci_gates.py | 4 +++- tests/test_publish_db.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index 1eb97b30a..46710052a 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -69,7 +69,8 @@ jobs: github.event.workflow_run.path == '.github/workflows/ci.yml' # One publish at a time, so two uploads of latest.db.gz never interleave. # A publish waiting behind a running one gives way to a newer one. On the - # job, not the workflow, so a run the guard skips never takes a place. + # job, not the workflow: a workflow-level group would hold every run of + # this file, the ones the guard skips included. concurrency: group: publish-db cancel-in-progress: false diff --git a/tests/test_ci_gates.py b/tests/test_ci_gates.py index c60736e25..625c10bc3 100644 --- a/tests/test_ci_gates.py +++ b/tests/test_ci_gates.py @@ -48,12 +48,14 @@ PUBLISH_NEEDS = {"publish-db": {"actions": "read"}} # The events a pull request starts runs for: opening or updating it, a review -# or review comment on it, and adding it to a merge queue. +# or review comment on it, a comment on its conversation (issue_comment), and +# adding it to a merge queue. PULL_REQUEST_EVENTS = { "pull_request", "pull_request_target", "pull_request_review", "pull_request_review_comment", + "issue_comment", "merge_group", } # The secrets a workflow that runs on pull requests may name: none. A debug diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index d9fd92760..682b74101 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -62,7 +62,8 @@ def test_it_runs_when_ci_completes_on_main(): } assert ci[True]["push"] == {"branches": ["main"]} assert list(workflow()["jobs"]) == ["publish-db"] - # On the job, so a run its guard skips never displaces a waiting publish. + # On the job: a workflow-level group would hold every run of the file, + # the ones the guard skips included. assert "concurrency" not in workflow() assert job()["concurrency"] == {"group": "publish-db", "cancel-in-progress": False} From f1e7389c660a22367e03c170e88c2b959af6ebe6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 12:56:00 -0400 Subject: [PATCH 05/10] Read main's tip and the pytest attempt's conclusion, and drop the concurrency group GPT-6.1 Sol's review of #220 found three things the first design got wrong: - A re-run of an old publish run keeps its GITHUB_SHA, so comparing that with the CI commit let it move latest.db.gz back. The find step now reads main's tip from the API. - An artifact does not say the attempt that fired this run passed. The find step now reads the pytest job's conclusion for that attempt, as needs: python did. - GitHub cancels a concurrency group's pending job when another enters, so an older completion could drop the publish for main's tip. The group is gone. curl gives up after five minutes, so an upload for an older tip cannot outlast the next commit's CI. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 53 +++++++------ tests/test_ci_gates.py | 12 +-- tests/test_publish_db.py | 131 +++++++++++++++++++++++-------- 3 files changed, 136 insertions(+), 60 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index 46710052a..eb07ee166 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -35,13 +35,22 @@ name: publish db # script interpolates the triggering run's fields: the run id and commit go # to the scripts as environment variables (tests/test_publish_db.py). # -# The artifact exists only if CI's pytest job got to its last step, so a +# It reads the pytest job's conclusion in the CI attempt that fired it, so a # run whose pytest job failed or was cancelled publishes nothing, with a -# notice. Any other CI conclusion still publishes .db.gz, as it did: a -# newer push cancels the older run, often after its pytest job has passed. -# latest.db.gz moves only for the commit that is still main's tip when CI -# completes (GITHUB_SHA, in a workflow_run run), so neither a cancelled run -# nor a re-run of an older one rolls it back. +# notice, even if an earlier attempt left an artifact. Any other CI conclusion +# still publishes .db.gz, as it did: a newer push cancels the older run, +# often after its pytest job has passed. +# +# latest.db.gz moves only for the commit at main's tip, read from the API as +# the job starts, so neither a cancelled run nor a re-run of an older CI or +# publish run rolls it back. A re-run keeps its original GITHUB_SHA, so +# GITHUB_SHA cannot answer this. +# +# No concurrency group: GitHub cancels a group's pending job when another +# enters, so an older run's completion could drop the publish for main's tip. +# Overlapping publishes write different objects, or the same commit's +# database. curl gives up after five minutes, so an upload for an older tip +# cannot outlast the next commit's CI and overwrite latest.db.gz behind it. # # Skips with a notice when the secrets are unset. A failed upload is a # warning, not a failed run: nothing in this repository reads the published @@ -67,20 +76,15 @@ jobs: github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.path == '.github/workflows/ci.yml' - # One publish at a time, so two uploads of latest.db.gz never interleave. - # A publish waiting behind a running one gives way to a newer one. On the - # job, not the workflow: a workflow-level group would hold every run of - # this file, the ones the guard skips included. - concurrency: - group: publish-db - cancel-in-progress: false runs-on: ubuntu-latest - # Listing and downloading another run's artifact takes actions: read, - # and nothing else is needed: there is no checkout. + # Reading CI's jobs and artifacts takes actions: read, and main's tip + # contents: read. There is no checkout. permissions: actions: read + contents: read env: CI_RUN_ID: ${{ github.event.workflow_run.id }} + CI_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} CI_SHA: ${{ github.event.workflow_run.head_sha }} steps: - name: Find the database CI built @@ -90,14 +94,18 @@ jobs: run: | set -euo pipefail artifact="scorecard-db-$CI_SHA" + pytest=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/attempts/$CI_ATTEMPT/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "pytest · ruff format") | .conclusion') count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/artifacts?name=$artifact" \ --jq '[.artifacts[] | select(.expired | not)] | length') - if [ "$count" -eq 0 ]; then - echo "::notice::CI run $CI_RUN_ID has no $artifact: its pytest job did not get as far as uploading it, so there is nothing to publish" - echo "found=false" >> "$GITHUB_OUTPUT" - else + main_tip=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" --jq .object.sha) + if [ "$pytest" = "success" ] && [ "$count" -gt 0 ]; then echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::CI run $CI_RUN_ID attempt $CI_ATTEMPT has nothing to publish: its pytest job ended ${pytest:-unknown}, with $count $artifact" + echo "found=false" >> "$GITHUB_OUTPUT" fi + echo "main_tip=$main_tip" >> "$GITHUB_OUTPUT" # Pinned to a commit, as uk-replay-results.yml's propose job pins its # actions, because it runs beside the key. - name: Download it @@ -110,6 +118,7 @@ jobs: - name: Publish it if: steps.find.outputs.found == 'true' env: + MAIN_TIP: ${{ steps.find.outputs.main_tip }} SUPABASE_URL: ${{ secrets.SUPABASE_SCORECARD_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_SCORECARD_KEY }} run: | @@ -119,14 +128,14 @@ jobs: fi gzip -9 -c scorecard.db > scorecard.db.gz set -- "${CI_SHA}.db.gz" - if [ "$CI_SHA" = "$GITHUB_SHA" ]; then + if [ "$CI_SHA" = "$MAIN_TIP" ]; then set -- "$@" latest.db.gz else - echo "::notice::main is at $GITHUB_SHA now, not $CI_SHA: latest.db.gz is left to the run for main's tip" + echo "::notice::main is at $MAIN_TIP now, not $CI_SHA: latest.db.gz is left to the run for main's tip" fi for name in "$@"; do status=0 - curl -sf -X POST \ + curl -sf --max-time 300 -X POST \ "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ -H "Authorization: Bearer $SUPABASE_KEY" \ -H "Content-Type: application/gzip" \ diff --git a/tests/test_ci_gates.py b/tests/test_ci_gates.py index 625c10bc3..f9a08dcde 100644 --- a/tests/test_ci_gates.py +++ b/tests/test_ci_gates.py @@ -43,9 +43,9 @@ # setup-bun read public releases, and the database artifact is uploaded on # the Actions runtime token, not GITHUB_TOKEN. NEEDS_MORE: dict[str, dict[str, str]] = {} -# publish-db.yml lists and downloads another run's artifact, CI's, which -# takes actions: read. It has no checkout, so it needs no contents scope. -PUBLISH_NEEDS = {"publish-db": {"actions": "read"}} +# publish-db.yml reads CI's jobs and downloads its artifact, which takes +# actions: read, and reads main's tip, which takes contents: read. +PUBLISH_NEEDS = {"publish-db": {"actions": "read", "contents": "read"}} # The events a pull request starts runs for: opening or updating it, a review # or review comment on it, a comment on its conversation (issue_comment), and @@ -308,9 +308,9 @@ def test_ci_uploads_the_database_without_a_token(): assert "download-artifact" not in text -def test_publish_db_reads_actions_and_nothing_more(): +def test_publish_db_reads_what_it_needs_and_nothing_more(): workflow = yaml.safe_load(PUBLISH.read_text()) - assert granted(workflow, "publish-db") == {"actions": "read"} + assert granted(workflow, "publish-db") == PUBLISH_NEEDS["publish-db"] assert overreach(workflow, PUBLISH_NEEDS) == [] @@ -321,7 +321,7 @@ def test_publish_db_reads_actions_and_nothing_more(): "read-all", {"actions": "write"}, {"actions": "read", "contents": "write"}, - {"actions": "read", "id-token": "write"}, + {"actions": "read", "contents": "read", "id-token": "write"}, ], ) def test_the_check_catches_a_wider_publish_token(scopes): diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index 682b74101..dd6c431e7 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -4,9 +4,10 @@ artifact, once CI completes on a push to main. It moved out of ci.yml so that no workflow that runs on pull requests names a secret (tests/test_ci_gates.py). These tests pin what the move had to keep: it -publishes only for main's own pushes, only what CI built, and runs no package -code beside the key; unset secrets skip with a notice, a failed upload is a -warning, and curl never prints the host. The step scripts run here under +publishes only for main's own pushes, only what passed CI's pytest job, and +runs no package code beside the key; latest.db.gz never moves off main's tip; +unset secrets skip with a notice, a failed upload is a warning, and curl never +prints the host. The step scripts run here under `bash -e`, the shell GitHub's log shows for them, with `gh` and `curl` stubs. """ @@ -33,6 +34,7 @@ SHA = "0123456789abcdef0123456789abcdef01234567" NEWER = "fedcba9876543210fedcba9876543210fedcba98" RUN_ID = "39698300855" +ATTEMPT = "2" REPOSITORY = "PolicyEngine/policyengine-scorecard" HOST = "secret-project-ref.supabase.co" KEY = "sb_secret_key_value" @@ -62,10 +64,9 @@ def test_it_runs_when_ci_completes_on_main(): } assert ci[True]["push"] == {"branches": ["main"]} assert list(workflow()["jobs"]) == ["publish-db"] - # On the job: a workflow-level group would hold every run of the file, - # the ones the guard skips included. - assert "concurrency" not in workflow() - assert job()["concurrency"] == {"group": "publish-db", "cancel-in-progress": False} + # GitHub cancels a concurrency group's pending job when another enters, + # so an older run's completion could drop the publish for main's tip. + assert "concurrency" not in workflow() and "concurrency" not in job() def test_it_is_the_only_workflow_run_workflow(): @@ -164,10 +165,18 @@ def test_it_downloads_what_ci_uploaded_for_that_commit(): assert 'artifact="scorecard-db-$CI_SHA"' in step(FIND)["run"] assert job()["env"] == { "CI_RUN_ID": "${{ github.event.workflow_run.id }}", + "CI_ATTEMPT": "${{ github.event.workflow_run.run_attempt }}", "CI_SHA": "${{ github.event.workflow_run.head_sha }}", } +def test_find_reads_the_job_ci_names_pytest(): + """A rename in ci.yml would otherwise leave nothing to match, and every + publish would skip with a notice.""" + name = yaml.safe_load(CI.read_text())["jobs"]["python"]["name"] + assert f'select(.name == "{name}")' in step(FIND)["run"] + + def test_no_script_interpolates_the_triggering_run(): """A pull request's author chooses its branch name, and GitHub pastes a ${{ }} into a script before the shell reads it. The scripts take the run @@ -200,6 +209,7 @@ def test_the_tokens_are_where_they_are_used(): with_secrets = [s["name"] for s in job()["steps"] if "secrets." in json.dumps(s)] assert with_secrets == [UPLOAD] assert step(UPLOAD)["env"] == { + "MAIN_TIP": "${{ steps.find.outputs.main_tip }}", "SUPABASE_URL": "${{ secrets.SUPABASE_SCORECARD_URL }}", "SUPABASE_KEY": "${{ secrets.SUPABASE_SCORECARD_KEY }}", } @@ -208,16 +218,18 @@ def test_the_tokens_are_where_they_are_used(): # Stand-ins for the runner image's gh and curl. Each records its arguments # and the variables the script reads; curl also keeps the bytes it would -# send. Like the real curl, a failing stub prints its error, host included, -# unless one of its short options includes -s. +# send. gh answers by endpoint, and fails for the one named in `fails_on`. +# Like the real curl, a failing stub prints its error, host included, unless +# one of its short options includes -s. GH_STUB = """#!{python} -I import json, os, sys with open({record!r}, "a") as handle: handle.write(json.dumps({{"argv": sys.argv[1:], "token": os.environ.get("GH_TOKEN")}}) + "\\n") -if {status}: +endpoint = sys.argv[2] +if {fails_on!r} and {fails_on!r} in endpoint: sys.stderr.write("HTTP 502\\n") - sys.exit({status}) -print({count!r}) + sys.exit(1) +print(next(answer for path, answer in {answers!r}.items() if path in endpoint)) """ CURL_STUB = """#!{python} -I import json, sys @@ -236,7 +248,17 @@ def test_the_tokens_are_where_they_are_used(): """ -def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): +def run_step( + tmp_path, + name, + *, + env=(), + script=None, + pytest_conclusion="success", + artifacts="1", + fails_on="", + status=0, +): """``name``'s script under `bash -e`, in a fresh workspace holding a database, with gh and curl stubs first on PATH.""" work = tmp_path / "work" @@ -249,8 +271,12 @@ def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): GH_STUB.format( python=sys.executable, record=str(records["gh"]), - count=gh_count, - status=status, + answers={ + "/jobs": pytest_conclusion, + "/artifacts": artifacts, + "/git/ref/": NEWER, + }, + fails_on=fails_on, ) ) (stubs / "curl").write_text( @@ -273,7 +299,11 @@ def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): "GITHUB_REPOSITORY": REPOSITORY, "GITHUB_SERVER_URL": "https://github.com", "CI_RUN_ID": RUN_ID, + "CI_ATTEMPT": ATTEMPT, "CI_SHA": SHA, + "MAIN_TIP": SHA, + # What the runner sets on a re-run of this commit's publish, whatever + # main has moved to since. "GITHUB_SHA": SHA, "GH_TOKEN": "ghs_runtoken", "SUPABASE_URL": f"https://{HOST}", @@ -296,37 +326,70 @@ def run_step(tmp_path, name, *, env=(), script=None, gh_count="1", status=0): return result, calls, output.read_text(), summary.read_text(), work -def test_find_says_found_when_ci_uploaded_the_database(tmp_path): - result, calls, output, _, _ = run_step(tmp_path, FIND, gh_count="1") +def test_find_says_found_when_ci_passed_and_uploaded_the_database(tmp_path): + result, calls, output, _, _ = run_step(tmp_path, FIND) assert result.returncode == 0, result.stderr - assert output == "found=true\n" + assert output == f"found=true\nmain_tip={NEWER}\n" assert "::notice::" not in result.stdout + run = f"repos/{REPOSITORY}/actions/runs/{RUN_ID}" assert calls["gh"] == [ { "argv": [ "api", - f"repos/{REPOSITORY}/actions/runs/{RUN_ID}/artifacts" - f"?name=scorecard-db-{SHA}", + f"{run}/attempts/{ATTEMPT}/jobs?per_page=100", + "--jq", + '.jobs[] | select(.name == "pytest · ruff format") | .conclusion', + ], + "token": "ghs_runtoken", + }, + { + "argv": [ + "api", + f"{run}/artifacts?name=scorecard-db-{SHA}", "--jq", "[.artifacts[] | select(.expired | not)] | length", ], "token": "ghs_runtoken", - } + }, + { + "argv": [ + "api", + f"repos/{REPOSITORY}/git/ref/heads/main", + "--jq", + ".object.sha", + ], + "token": "ghs_runtoken", + }, ] def test_find_skips_with_a_notice_when_the_pytest_job_uploaded_nothing(tmp_path): - result, calls, output, _, _ = run_step(tmp_path, FIND, gh_count="0") + result, calls, output, _, _ = run_step(tmp_path, FIND, artifacts="0") assert result.returncode == 0, result.stderr - assert output == "found=false\n" - assert result.stdout.startswith( - f"::notice::CI run {RUN_ID} has no scorecard-db-{SHA}: its pytest job" + assert output == f"found=false\nmain_tip={NEWER}\n" + assert result.stdout == ( + f"::notice::CI run {RUN_ID} attempt {ATTEMPT} has nothing to publish: its " + f"pytest job ended success, with 0 scorecard-db-{SHA}\n" ) assert not calls["curl"] -def test_find_fails_the_run_when_it_cannot_ask(tmp_path): - result, _, output, _, _ = run_step(tmp_path, FIND, status=1) +@pytest.mark.parametrize("conclusion", ["failure", "cancelled", "skipped", ""]) +def test_find_skips_an_artifact_this_attempts_pytest_job_did_not_pass( + tmp_path, conclusion +): + """An artifact can outlive the attempt that uploaded it: a re-run of a + job that passed shares its attempt's artifact (run 39698458972), so the + artifact alone does not say the attempt that fired this run passed.""" + result, _, output, _, _ = run_step(tmp_path, FIND, pytest_conclusion=conclusion) + assert result.returncode == 0, result.stderr + assert output == f"found=false\nmain_tip={NEWER}\n" + assert f"pytest job ended {conclusion or 'unknown'}," in result.stdout + + +@pytest.mark.parametrize("endpoint", ["/jobs", "/artifacts", "/git/ref/"]) +def test_find_fails_the_run_when_it_cannot_ask(tmp_path, endpoint): + result, _, output, _, _ = run_step(tmp_path, FIND, fails_on=endpoint) assert result.returncode != 0 assert output == "" @@ -352,6 +415,8 @@ def test_publish_uploads_the_commits_database_then_latest(tmp_path): for call, name in zip(calls["curl"], names, strict=True): assert call["argv"] == [ "-sf", + "--max-time", + "300", "-X", "POST", f"https://{HOST}/storage/v1/object/scorecard-artifacts/{name}", @@ -368,16 +433,18 @@ def test_publish_uploads_the_commits_database_then_latest(tmp_path): def test_latest_waits_for_mains_tip(tmp_path): - """GITHUB_SHA in a workflow_run run is main's tip. When main has moved on - (a newer push cancelled this run, or an older run was re-run), the - commit's own copy is uploaded and latest.db.gz is not rolled back.""" - result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, env={"GITHUB_SHA": NEWER}) + """When main has moved on (a newer push cancelled this run, or an older + CI or publish run was re-run), the commit's own copy is uploaded and + latest.db.gz is not rolled back. On a re-run of an old publish GITHUB_SHA + is still that commit, as run_step sets it, so only the tip the find step + read tells the two apart.""" + result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, env={"MAIN_TIP": NEWER}) assert result.returncode == 0, result.stderr assert result.stdout == ( f"::notice::main is at {NEWER} now, not {SHA}: latest.db.gz is left " f"to the run for main's tip\nuploaded scorecard-artifacts/{SHA}.db.gz\n" ) - assert [call["argv"][3].rsplit("/", 1)[1] for call in calls["curl"]] == [ + assert [call["argv"][5].rsplit("/", 1)[1] for call in calls["curl"]] == [ f"{SHA}.db.gz" ] assert summary == "" From 3415c264e180e6fbaf8c1164e115ce6466ed013f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 12:59:50 -0400 Subject: [PATCH 06/10] Pin that no publish script reads GITHUB_SHA Co-Authored-By: Claude Opus 5.5 --- tests/test_publish_db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index dd6c431e7..d84bb7423 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -185,6 +185,13 @@ def test_no_script_interpolates_the_triggering_run(): assert "${{" not in s.get("run", ""), s.get("name") +def test_no_script_reads_github_sha(): + """In a workflow_run run GITHUB_SHA is not the CI commit, and a re-run + keeps its original value, so it is not main's tip either.""" + for s in job()["steps"]: + assert "GITHUB_SHA" not in s.get("run", ""), s.get("name") + + def test_the_job_runs_no_package_code(): """No checkout and no package manager or interpreter: what runs beside the key is actions/download-artifact and the runner image's gh, gzip From c030a3447f87977405e1e0c1273d3f9b965faf11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:12:52 -0400 Subject: [PATCH 07/10] Bound the publish job, so the claim about overlapping publishes holds The second review of #220 found the comment overclaimed. curl's five-minute limit covered one upload, but the window after the tip read also holds the download, gzip and a second upload, longer than the fastest CI run. The job now has five minutes in all, and the fastest of main's last 40 successful CI runs took seven. Each upload has two minutes, so a stall still ends in the warning. A real publish takes about ten seconds. The comment also now says latest.db.gz lags main while the tip's pytest job has not passed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index eb07ee166..3d2abb7d9 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -41,16 +41,20 @@ name: publish db # still publishes .db.gz, as it did: a newer push cancels the older run, # often after its pytest job has passed. # -# latest.db.gz moves only for the commit at main's tip, read from the API as +# latest.db.gz moves only to the commit at main's tip, read from the API as # the job starts, so neither a cancelled run nor a re-run of an older CI or # publish run rolls it back. A re-run keeps its original GITHUB_SHA, so -# GITHUB_SHA cannot answer this. +# GITHUB_SHA cannot answer this. It lags main while the tip's pytest job has +# not passed. # # No concurrency group: GitHub cancels a group's pending job when another # enters, so an older run's completion could drop the publish for main's tip. # Overlapping publishes write different objects, or the same commit's -# database. curl gives up after five minutes, so an upload for an older tip -# cannot outlast the next commit's CI and overwrite latest.db.gz behind it. +# database. The job has five minutes, and the next commit's publish waits for +# that commit's CI, so while CI takes longer (seven minutes at its fastest in +# October 2026) a publish for an older tip ends before the next one starts. +# Each upload has two of those minutes, so a stalled upload still ends in the +# warning below, not a timed-out job. # # Skips with a notice when the secrets are unset. A failed upload is a # warning, not a failed run: nothing in this repository reads the published @@ -77,6 +81,7 @@ jobs: github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.path == '.github/workflows/ci.yml' runs-on: ubuntu-latest + timeout-minutes: 5 # Reading CI's jobs and artifacts takes actions: read, and main's tip # contents: read. There is no checkout. permissions: @@ -135,7 +140,7 @@ jobs: fi for name in "$@"; do status=0 - curl -sf --max-time 300 -X POST \ + curl -sf --max-time 120 -X POST \ "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ -H "Authorization: Bearer $SUPABASE_KEY" \ -H "Content-Type: application/gzip" \ From f6bea138128e3775a2f97bba816e67dc241d18cd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:17:15 -0400 Subject: [PATCH 08/10] Pin the job's and each upload's time limits Co-Authored-By: Claude Opus 5.5 --- tests/test_publish_db.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index d84bb7423..273413b09 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -67,6 +67,11 @@ def test_it_runs_when_ci_completes_on_main(): # GitHub cancels a concurrency group's pending job when another enters, # so an older run's completion could drop the publish for main's tip. assert "concurrency" not in workflow() and "concurrency" not in job() + # Without a group, this limit keeps a publish for an older tip from ending + # after the next commit's, which waits for that commit's CI. Each upload's + # time limit fits inside it, so a stalled upload warns before the job ends. + assert job()["timeout-minutes"] == 5 + assert "--max-time 120 " in step(UPLOAD)["run"] def test_it_is_the_only_workflow_run_workflow(): @@ -423,7 +428,7 @@ def test_publish_uploads_the_commits_database_then_latest(tmp_path): assert call["argv"] == [ "-sf", "--max-time", - "300", + "120", "-X", "POST", f"https://{HOST}/storage/v1/object/scorecard-artifacts/{name}", From 45347ce56ba69a82b5f7d63c71a599c2ce266857 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:32:07 -0400 Subject: [PATCH 09/10] Publish one at a time with queue: max, in place of a timing argument The third review of #220 found the timing argument behind the dropped concurrency group could not be made a guarantee, and that the job limit added to support it could end a job before its warning. A concurrency group with queue: max gives the ordering directly: one publish runs at a time, and a waiting one survives where the default queue would cancel it. Three pushes on a throwaway branch ran in order with none cancelled (runs 41138146103, 41138172124, 41138195694). The job limit goes, and each upload keeps its own, so a stall warns and frees the queue. A test now pins that the upload is the pytest job's last step, so an artifact of that name passed the job's checks whichever attempt uploaded it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 32 ++++++++++++++++++++------------ tests/test_publish_db.py | 23 +++++++++++++++-------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index 3d2abb7d9..479a6494c 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -37,9 +37,11 @@ name: publish db # # It reads the pytest job's conclusion in the CI attempt that fired it, so a # run whose pytest job failed or was cancelled publishes nothing, with a -# notice, even if an earlier attempt left an artifact. Any other CI conclusion -# still publishes .db.gz, as it did: a newer push cancels the older run, -# often after its pytest job has passed. +# notice, even if an earlier attempt left an artifact. The upload is that job's +# last step, so whichever artifact of that name the download takes passed the +# job's checks on this commit. Any other CI conclusion still publishes +# .db.gz, as it did: a newer push cancels the older run, often after its +# pytest job has passed. # # latest.db.gz moves only to the commit at main's tip, read from the API as # the job starts, so neither a cancelled run nor a re-run of an older CI or @@ -47,14 +49,11 @@ name: publish db # GITHUB_SHA cannot answer this. It lags main while the tip's pytest job has # not passed. # -# No concurrency group: GitHub cancels a group's pending job when another -# enters, so an older run's completion could drop the publish for main's tip. -# Overlapping publishes write different objects, or the same commit's -# database. The job has five minutes, and the next commit's publish waits for -# that commit's CI, so while CI takes longer (seven minutes at its fastest in -# October 2026) a publish for an older tip ends before the next one starts. -# Each upload has two of those minutes, so a stalled upload still ends in the -# warning below, not a timed-out job. +# One publish runs at a time and the rest wait, so a publish for an older tip +# cannot overwrite latest.db.gz behind a newer one, and each reads the tip once +# its turn comes. The job has no time limit of its own, which could end it +# before the warning below. Each upload has two minutes, so a stalled one warns +# and lets the next publish start. # # Skips with a notice when the secrets are unset. A failed upload is a # warning, not a failed run: nothing in this repository reads the published @@ -81,7 +80,16 @@ jobs: github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.path == '.github/workflows/ci.yml' runs-on: ubuntu-latest - timeout-minutes: 5 + # `queue: max` keeps every waiting publish: by default GitHub cancels a + # group's pending job when another enters, so an older run's completion + # could drop the publish for main's tip. Checked on a throwaway branch on + # 30 October 2026: runs 41138146103, 41138172124 and 41138195694, pushed 14 + # seconds apart, ran one after another and none was cancelled. On the job, + # not the workflow: a workflow-level group would hold every run of this + # file, the ones the guard skips included. + concurrency: + group: publish-db + queue: max # Reading CI's jobs and artifacts takes actions: read, and main's tip # contents: read. There is no checkout. permissions: diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index 273413b09..e34e035c4 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -5,7 +5,7 @@ no workflow that runs on pull requests names a secret (tests/test_ci_gates.py). These tests pin what the move had to keep: it publishes only for main's own pushes, only what passed CI's pytest job, and -runs no package code beside the key; latest.db.gz never moves off main's tip; +runs no package code beside the key; latest.db.gz moves only to main's tip; unset secrets skip with a notice, a failed upload is a warning, and curl never prints the host. The step scripts run here under `bash -e`, the shell GitHub's log shows for them, with `gh` and `curl` stubs. @@ -64,13 +64,15 @@ def test_it_runs_when_ci_completes_on_main(): } assert ci[True]["push"] == {"branches": ["main"]} assert list(workflow()["jobs"]) == ["publish-db"] - # GitHub cancels a concurrency group's pending job when another enters, - # so an older run's completion could drop the publish for main's tip. - assert "concurrency" not in workflow() and "concurrency" not in job() - # Without a group, this limit keeps a publish for an older tip from ending - # after the next commit's, which waits for that commit's CI. Each upload's - # time limit fits inside it, so a stalled upload warns before the job ends. - assert job()["timeout-minutes"] == 5 + # One publish at a time, so an older tip's cannot land behind a newer one's, + # and none dropped: by default GitHub cancels a group's pending job when + # another enters. On the job: a workflow-level group would hold every run + # of the file, the ones the guard skips included. + assert "concurrency" not in workflow() + assert job()["concurrency"] == {"group": "publish-db", "queue": "max"} + # A job limit could end the job before the warning. Each upload's own limit + # turns a stall into that warning and lets the next publish start. + assert "timeout-minutes" not in job() assert "--max-time 120 " in step(UPLOAD)["run"] @@ -158,6 +160,11 @@ def test_it_downloads_what_ci_uploaded_for_that_commit(): ] assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" assert upload["with"]["path"] == "data/scorecard.db" + # So any artifact of this name passed the job's checks, whichever attempt + # uploaded it and whichever one the download takes. + steps = ci["jobs"]["python"]["steps"] + assert steps[-1] is upload and "if" not in upload + assert not any("continue-on-error" in s for s in steps) download = step(DOWNLOAD) assert download["uses"] == DOWNLOAD_ACTION assert download["with"] == { From 4b297feb8e773f4c8b231f68ba9298db1759dbf7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 30 Oct 2026 18:43:54 -0400 Subject: [PATCH 10/10] Bound the steps before the uploads, and say what queue: max holds The fourth review of #220 found two overstatements. queue: max keeps up to 100 pending publishes, not every one. And with no job limit, a stalled gh call could hold the queue for GitHub's 360-minute default. The find and download steps now have five minutes each, and a stall there fails the run. The publish step keeps only curl's limits, so no limit can end it before its warning. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-db.yml | 13 ++++++++----- tests/test_publish_db.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml index 479a6494c..3b9358bb7 100644 --- a/.github/workflows/publish-db.yml +++ b/.github/workflows/publish-db.yml @@ -51,9 +51,10 @@ name: publish db # # One publish runs at a time and the rest wait, so a publish for an older tip # cannot overwrite latest.db.gz behind a newer one, and each reads the tip once -# its turn comes. The job has no time limit of its own, which could end it -# before the warning below. Each upload has two minutes, so a stalled one warns -# and lets the next publish start. +# its turn comes. Time limits keep a stall from holding the queue: each upload +# has two minutes, which ends in the warning below, and the two steps before +# them five each, which fails the run. The job has no limit of its own, which +# could end it inside an upload, before that warning. # # Skips with a notice when the secrets are unset. A failed upload is a # warning, not a failed run: nothing in this repository reads the published @@ -80,8 +81,8 @@ jobs: github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.path == '.github/workflows/ci.yml' runs-on: ubuntu-latest - # `queue: max` keeps every waiting publish: by default GitHub cancels a - # group's pending job when another enters, so an older run's completion + # `queue: max` keeps up to 100 waiting publishes: by default GitHub cancels + # a group's pending job when another enters, so an older run's completion # could drop the publish for main's tip. Checked on a throwaway branch on # 30 October 2026: runs 41138146103, 41138172124 and 41138195694, pushed 14 # seconds apart, ran one after another and none was cancelled. On the job, @@ -102,6 +103,7 @@ jobs: steps: - name: Find the database CI built id: find + timeout-minutes: 5 env: GH_TOKEN: ${{ github.token }} run: | @@ -123,6 +125,7 @@ jobs: # actions, because it runs beside the key. - name: Download it if: steps.find.outputs.found == 'true' + timeout-minutes: 5 uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: scorecard-db-${{ github.event.workflow_run.head_sha }} diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py index e34e035c4..750189ecf 100644 --- a/tests/test_publish_db.py +++ b/tests/test_publish_db.py @@ -65,15 +65,17 @@ def test_it_runs_when_ci_completes_on_main(): assert ci[True]["push"] == {"branches": ["main"]} assert list(workflow()["jobs"]) == ["publish-db"] # One publish at a time, so an older tip's cannot land behind a newer one's, - # and none dropped: by default GitHub cancels a group's pending job when - # another enters. On the job: a workflow-level group would hold every run - # of the file, the ones the guard skips included. + # and up to 100 waiting: by default GitHub cancels a group's pending job + # when another enters. On the job: a workflow-level group would hold every + # run of the file, the ones the guard skips included. assert "concurrency" not in workflow() assert job()["concurrency"] == {"group": "publish-db", "queue": "max"} - # A job limit could end the job before the warning. Each upload's own limit - # turns a stall into that warning and lets the next publish start. - assert "timeout-minutes" not in job() + # A stall must not hold that queue. Each upload's limit turns a stall into + # the warning, and the steps before them fail the run. A limit on the job + # or the publish step could end it inside an upload, before the warning. assert "--max-time 120 " in step(UPLOAD)["run"] + assert step(FIND)["timeout-minutes"] == step(DOWNLOAD)["timeout-minutes"] == 5 + assert "timeout-minutes" not in job() and "timeout-minutes" not in step(UPLOAD) def test_it_is_the_only_workflow_run_workflow():