diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 725676042..10ba2982c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,11 @@ concurrency: # GITHUB_TOKEN reads the repository and nothing else in every job without # its own block. The repository's default is write, which the pytest job # would otherwise hold (persisted by checkout) while it runs PyPI packages -# on pushes to main. Uploading the built database and publish-db's download -# of it from this run use the Actions runtime token, not GITHUB_TOKEN, so -# neither needs the actions scope (tests/test_ci_gates.py pins all of this). +# on pushes to main. Uploading the built database uses the Actions runtime +# token, not GITHUB_TOKEN, so it needs no actions scope. publish-db.yml +# publishes it after this workflow completes on a push to main: it holds the +# Supabase secrets, and no workflow that runs on pull requests names any +# secret (tests/test_ci_gates.py pins all of this). permissions: contents: read @@ -65,6 +67,7 @@ jobs: PYTHONPATH: . run: bash .github/scripts/uk_replay_regenerate.sh - run: test -z "$(git status --porcelain)" + # publish-db.yml downloads it from this run by this name. - name: Upload built database uses: actions/upload-artifact@v4 with: @@ -398,52 +401,3 @@ jobs: - run: bun run lint - run: bun test src - run: bun run build - - publish-db: - name: publish db to supabase storage - needs: python - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v4 - with: - name: scorecard-db-${{ github.sha }} - # Storage bucket `scorecard-artifacts` on the PE Supabase project: - # every main commit's built DB (gzipped ~6 MB from ~58 MB — the - # project is on the free plan, 50 MB per-object cap) at .db.gz - # plus a moving latest.db.gz, so consumers fetch + gunzip instead - # of rebuilding. Skips with a notice until the - # SUPABASE_SCORECARD_URL/KEY secrets are set. - # - # A failed upload is a warning, not a failed run: nothing in this - # repository reads the published copy, and the built database is - # already attached to the run as an artifact. Every main run from - # 6 October 2026 failed here with curl exit 6 (the storage host no - # longer resolves; the last upload that worked was 28 September), - # which hid whether main was otherwise green. curl stays silent so - # the host name in the secret URL is never printed. - - run: | - if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then - echo "::notice::Supabase secrets not set — skipping publish" - exit 0 - fi - gzip -9 -c scorecard.db > scorecard.db.gz - for name in "${GITHUB_SHA}.db.gz" "latest.db.gz"; do - status=0 - curl -sf -X POST \ - "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ - -H "Authorization: Bearer $SUPABASE_KEY" \ - -H "Content-Type: application/gzip" \ - -H "x-upsert: true" \ - --data-binary @scorecard.db.gz > /dev/null || status=$? - if [ "$status" -ne 0 ]; then - message="curl exit $status uploading scorecard-artifacts/$name: the published database was not updated. This run's built database is its scorecard-db artifact." - echo "::warning title=Supabase publish failed::$message" - echo "**Supabase publish failed.** $message" >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi - echo "uploaded scorecard-artifacts/$name" - done - env: - SUPABASE_URL: ${{ secrets.SUPABASE_SCORECARD_URL }} - SUPABASE_KEY: ${{ secrets.SUPABASE_SCORECARD_KEY }} diff --git a/.github/workflows/publish-db.yml b/.github/workflows/publish-db.yml new file mode 100644 index 000000000..3b9358bb7 --- /dev/null +++ b/.github/workflows/publish-db.yml @@ -0,0 +1,167 @@ +name: publish db + +# Publishes the database CI built for each push to main to the +# `scorecard-artifacts` bucket on the PE Supabase project, as .db.gz +# (gzipped ~14 MB from ~196 MB: the project is on the free plan, 50 MB +# per-object cap) and a moving latest.db.gz, so consumers fetch and gunzip +# instead of rebuilding. +# +# WHY IT IS NOT IN ci.yml +# +# This is the only workflow that names SUPABASE_SCORECARD_URL and +# SUPABASE_SCORECARD_KEY. ci.yml runs on pull requests and installs and runs +# PyPI packages and each certified variant's pins, as a runner user with +# passwordless sudo. A debug re-run of #215's pull-request CI showed a job's +# runner receiving a secret the job named only in a skipped step, and GitHub +# scrubs only the secrets "not referenced in the workflow". So while +# publish-db was a push-only job of ci.yml, the pull-request runners there may +# have held the key. No workflow that runs on pull requests names a secret +# now (tests/test_ci_gates.py). +# +# WHY workflow_run, NOT A REBUILD +# +# It publishes the bytes CI built and tested, once CI's pytest job has +# passed, as `needs: python` did inside ci.yml. A push workflow that rebuilt +# the database would run the project's PyPI packages in the job that holds +# the key, or in a sibling job of the workflow that names it, and would +# publish a database whose suite had not run. This job runs no package code: +# no checkout, no uv, no Python, only actions/download-artifact and the +# runner image's gh, gzip and curl. +# +# workflow_run runs main's copy of this file with the repository's secrets, +# whatever started the CI run, a fork's pull request included, and the +# `branches` filter names a branch, not an event or a repository. So the +# job runs only for ci.yml on a push to main of this repository, and no +# script interpolates the triggering run's fields: the run id and commit go +# to the scripts as environment variables (tests/test_publish_db.py). +# +# It reads the pytest job's conclusion in the CI attempt that fired it, so a +# run whose pytest job failed or was cancelled publishes nothing, with a +# notice, even if an earlier attempt left an artifact. The upload is that job's +# last step, so whichever artifact of that name the download takes passed the +# job's checks on this commit. Any other CI conclusion still publishes +# .db.gz, as it did: a newer push cancels the older run, often after its +# pytest job has passed. +# +# latest.db.gz moves only to the commit at main's tip, read from the API as +# the job starts, so neither a cancelled run nor a re-run of an older CI or +# publish run rolls it back. A re-run keeps its original GITHUB_SHA, so +# GITHUB_SHA cannot answer this. It lags main while the tip's pytest job has +# not passed. +# +# One publish runs at a time and the rest wait, so a publish for an older tip +# cannot overwrite latest.db.gz behind a newer one, and each reads the tip once +# its turn comes. Time limits keep a stall from holding the queue: each upload +# has two minutes, which ends in the warning below, and the two steps before +# them five each, which fails the run. The job has no limit of its own, which +# could end it inside an upload, before that warning. +# +# Skips with a notice when the secrets are unset. A failed upload is a +# warning, not a failed run: nothing in this repository reads the published +# copy, and CI's run keeps the built database as an artifact for 30 days. +# Every publish from 6 October 2026 failed with curl exit 6 (the storage host +# no longer resolves; the last upload that worked was 28 September). curl +# stays silent so the host name in the secret URL is never printed. + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + +permissions: + contents: read + +jobs: + publish-db: + name: publish db to supabase storage + if: >- + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.path == '.github/workflows/ci.yml' + runs-on: ubuntu-latest + # `queue: max` keeps up to 100 waiting publishes: by default GitHub cancels + # a group's pending job when another enters, so an older run's completion + # could drop the publish for main's tip. Checked on a throwaway branch on + # 30 October 2026: runs 41138146103, 41138172124 and 41138195694, pushed 14 + # seconds apart, ran one after another and none was cancelled. On the job, + # not the workflow: a workflow-level group would hold every run of this + # file, the ones the guard skips included. + concurrency: + group: publish-db + queue: max + # Reading CI's jobs and artifacts takes actions: read, and main's tip + # contents: read. There is no checkout. + permissions: + actions: read + contents: read + env: + CI_RUN_ID: ${{ github.event.workflow_run.id }} + CI_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + CI_SHA: ${{ github.event.workflow_run.head_sha }} + steps: + - name: Find the database CI built + id: find + timeout-minutes: 5 + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + artifact="scorecard-db-$CI_SHA" + pytest=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/attempts/$CI_ATTEMPT/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "pytest · ruff format") | .conclusion') + count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/artifacts?name=$artifact" \ + --jq '[.artifacts[] | select(.expired | not)] | length') + main_tip=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" --jq .object.sha) + if [ "$pytest" = "success" ] && [ "$count" -gt 0 ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::CI run $CI_RUN_ID attempt $CI_ATTEMPT has nothing to publish: its pytest job ended ${pytest:-unknown}, with $count $artifact" + echo "found=false" >> "$GITHUB_OUTPUT" + fi + echo "main_tip=$main_tip" >> "$GITHUB_OUTPUT" + # Pinned to a commit, as uk-replay-results.yml's propose job pins its + # actions, because it runs beside the key. + - name: Download it + if: steps.find.outputs.found == 'true' + timeout-minutes: 5 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: scorecard-db-${{ github.event.workflow_run.head_sha }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Publish it + if: steps.find.outputs.found == 'true' + env: + MAIN_TIP: ${{ steps.find.outputs.main_tip }} + SUPABASE_URL: ${{ secrets.SUPABASE_SCORECARD_URL }} + SUPABASE_KEY: ${{ secrets.SUPABASE_SCORECARD_KEY }} + run: | + if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then + echo "::notice::Supabase secrets not set — skipping publish" + exit 0 + fi + gzip -9 -c scorecard.db > scorecard.db.gz + set -- "${CI_SHA}.db.gz" + if [ "$CI_SHA" = "$MAIN_TIP" ]; then + set -- "$@" latest.db.gz + else + echo "::notice::main is at $MAIN_TIP now, not $CI_SHA: latest.db.gz is left to the run for main's tip" + fi + for name in "$@"; do + status=0 + curl -sf --max-time 120 -X POST \ + "$SUPABASE_URL/storage/v1/object/scorecard-artifacts/$name" \ + -H "Authorization: Bearer $SUPABASE_KEY" \ + -H "Content-Type: application/gzip" \ + -H "x-upsert: true" \ + --data-binary @scorecard.db.gz > /dev/null || status=$? + if [ "$status" -ne 0 ]; then + message="curl exit $status uploading scorecard-artifacts/$name: the published database was not updated. The database CI built is the scorecard-db-$CI_SHA artifact of $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID." + echo "::warning title=Supabase publish failed::$message" + echo "**Supabase publish failed.** $message" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "uploaded scorecard-artifacts/$name" + done diff --git a/scorecard_db/README.md b/scorecard_db/README.md index b2a8d0bcd..d12273dbe 100644 --- a/scorecard_db/README.md +++ b/scorecard_db/README.md @@ -41,9 +41,13 @@ old file first; nothing is lost, it is derived). Plain `pytest` also works from a fresh clone: a pytest configuration hook builds the database before collection when it is absent. Individual ingest modules (`ingest_urban`, `ingest_platform`, …) remain runnable one at a time against an existing database, in the -dependency order documented in `build_db.py`. Built artifacts for every -main commit are published to the `scorecard-artifacts` Supabase storage -bucket as `.db.gz` and `latest.db.gz`. +dependency order documented in `build_db.py`. Each main commit's CI run +keeps the database it built and tested as its `scorecard-db-` artifact +for 30 days, and `.github/workflows/publish-db.yml` then publishes it to the +`scorecard-artifacts` Supabase storage bucket as `.db.gz` and +`latest.db.gz`. Every publish since 6 October 2026 has failed (the storage +host no longer resolves), which the workflow reports as a warning; the last +upload that worked was on 28 September. ```python from scorecard_db import ScorecardDB diff --git a/tests/test_ci_gates.py b/tests/test_ci_gates.py index 99fafde87..2f1296738 100644 --- a/tests/test_ci_gates.py +++ b/tests/test_ci_gates.py @@ -16,6 +16,11 @@ They also pin the token ci.yml's jobs hold: the repository's default GITHUB_TOKEN is write, so a job the file does not narrow would hold it. + +And they keep every secret out of the workflows that run on pull requests, +which install and run third-party packages: a runner receives a secret its +job names even in a step that never runs, and GitHub scrubs only the secrets +"not referenced in the workflow". """ import re @@ -25,23 +30,114 @@ import yaml ROOT = Path(__file__).resolve().parent.parent -CI = ROOT / ".github" / "workflows" / "ci.yml" -DETERMINISM = ROOT / ".github" / "workflows" / "uk-replay-determinism.yml" -FRESHNESS = ROOT / ".github" / "workflows" / "gate-freshness.yml" +WORKFLOWS = ROOT / ".github" / "workflows" +CI = WORKFLOWS / "ci.yml" +DETERMINISM = WORKFLOWS / "uk-replay-determinism.yml" +FRESHNESS = WORKFLOWS / "gate-freshness.yml" +PUBLISH = WORKFLOWS / "publish-db.yml" READ = {"contents": "read"} LEVEL = {"none": 0, "read": 1, "write": 2} # A ci.yml job that needs more than READ goes here with exactly the scopes it # needs, and why. None does: checkout fetches this repository, setup-uv and -# setup-bun read public releases, and the database artifact moves within one -# run on the Actions runtime token, not GITHUB_TOKEN. +# setup-bun read public releases, and the database artifact is uploaded on +# the Actions runtime token, not GITHUB_TOKEN. NEEDS_MORE: dict[str, dict[str, str]] = {} +# publish-db.yml reads CI's jobs and downloads its artifact, which takes +# actions: read, and reads main's tip, which takes contents: read. +PUBLISH_NEEDS = {"publish-db": {"actions": "read", "contents": "read"}} + +# The events a pull request starts runs for: opening or updating it, a review +# or review comment on it, a comment on its conversation (issue_comment), and +# adding it to a merge queue. +PULL_REQUEST_EVENTS = { + "pull_request", + "pull_request_target", + "pull_request_review", + "pull_request_review_comment", + "issue_comment", + "merge_group", +} +# The secrets a workflow that runs on pull requests may name: none. A debug +# re-run of #215's pull-request CI (job 119587048984) evaluated a secret its +# job named only in a skipped step, so naming one anywhere in such a workflow +# may hand it to runners that run a pull request's packages. +SECRETS_ON_PULL_REQUESTS: frozenset[str] = frozenset() +SUPABASE = {"SUPABASE_SCORECARD_URL", "SUPABASE_SCORECARD_KEY"} def ci() -> dict: return yaml.safe_load(CI.read_text()) +def workflows() -> dict[str, dict]: + return { + path.name: yaml.safe_load(path.read_text()) + for path in sorted(WORKFLOWS.glob("*.y*ml")) + } + + +def triggers(workflow: dict) -> set[str]: + """A workflow's events. PyYAML reads the bare key `on` as True.""" + on = workflow[True] + return {on} if isinstance(on, str) else set(on) + + +def templates(text: str) -> list[str]: + """The body of each ${{ }} in ``text``, read as GitHub's parser reads + it: a }} inside a single-quoted string literal does not end it. An + escaped quote ('') toggles twice, which leaves the state as it was.""" + found, start = [], text.find("${{") + while start != -1: + end, quoted = start + 3, False + while end < len(text) and (quoted or not text.startswith("}}", end)): + quoted ^= text[end] == "'" + end += 1 + found.append(text[start + 3 : end]) + start = text.find("${{", end + 2) + return found + + +# The secrets context itself, not a property named secrets (inputs.secrets), +# in any case, so Secrets.x or SECRETS.X cannot slip past. +CONTEXT = re.compile(r"(? set[str]: + """The secrets ``workflow`` names, upper-cased. A reference that names + no one secret, such as toJSON(secrets), secrets[format(...)] or a + reusable-workflow call's `secrets:`, counts as "*", every secret.""" + names = set() + for expression in expressions(workflow): + matches = NAMED.findall(expression) + names |= {next(n for n in groups if n).upper() for groups in matches} + if len(CONTEXT.findall(expression)) > len(matches): + names.add("*") + if any("secrets" in job for job in workflow["jobs"].values()): + names.add("*") + return names + + def granted(workflow: dict, job: str): """The GITHUB_TOKEN scopes ``job`` gets from the file. GitHub applies the workflow's block, then the job's, and a block that names any scope sets @@ -50,13 +146,13 @@ def granted(workflow: dict, job: str): return workflow["jobs"][job].get("permissions", workflow.get("permissions")) -def overreach(workflow: dict) -> list[str]: - """Every grant beyond READ, or beyond a job's NEEDS_MORE entry.""" +def overreach(workflow: dict, needs_more=NEEDS_MORE) -> list[str]: + """Every grant beyond READ, or beyond a job's ``needs_more`` entry.""" found = [] if workflow.get("permissions") != READ: found.append(f"the workflow level is {workflow.get('permissions')!r}") for job in workflow["jobs"]: - scopes, allowed = granted(workflow, job), NEEDS_MORE.get(job, READ) + scopes, allowed = granted(workflow, job), needs_more.get(job, READ) if not isinstance(scopes, dict): found.append(f"{job} gets {scopes!r}") continue @@ -135,7 +231,8 @@ def test_the_failure_message_says_what_to_do(): def test_every_ci_job_reads_the_repository_and_nothing_more(): """The pytest job runs PyPI packages on pushes to main with checkout's persisted token. Until the workflow-level block, that job, the app job - and publish-db held the repository's write default.""" + and publish-db (in publish-db.yml since) held the repository's write + default.""" workflow = ci() assert workflow["permissions"] == READ assert set(NEEDS_MORE) <= set(workflow["jobs"]), "NEEDS_MORE names a gone job" @@ -160,7 +257,7 @@ def test_every_real_replay_job_reads_the_repository_and_nothing_more(): lambda w: w.update(permissions={**READ, "actions": "read"}), lambda w: w["jobs"]["python"].update(permissions={"contents": "write"}), lambda w: w["jobs"]["app"].update(permissions="write-all"), - lambda w: w["jobs"]["publish-db"].update( + lambda w: w["jobs"]["replay-variants"].update( permissions={**READ, "actions": "read"} ), lambda w: w["jobs"]["replay-engine"].update(permissions={"id-token": "write"}), @@ -176,7 +273,7 @@ def test_every_real_replay_job_reads_the_repository_and_nothing_more(): "workflow-adds-actions", "python-contents-write", "app-write-all", - "publish-db-adds-actions", + "replay-variants-adds-actions", "replay-engine-id-token", "real-replay-sandbox-adds-actions", ], @@ -190,18 +287,17 @@ def test_the_check_catches_every_wider_token(edit): def test_the_check_lets_a_job_narrow_its_token(): workflow = ci() workflow["jobs"]["app"]["permissions"] = {} - workflow["jobs"]["publish-db"]["permissions"] = {"contents": "none"} + workflow["jobs"]["receiver-sandbox"]["permissions"] = {"contents": "none"} assert overreach(workflow) == [] -def test_publish_db_downloads_this_runs_database_without_a_token(): - """actions/download-artifact v4 without `github-token` finds the artifact - in the current run through the Actions runtime token - (ACTIONS_RUNTIME_TOKEN), which is why publish-db needs no actions scope; - a `github-token` would need actions: read. Upload has no token input. - Re-check this when either action changes major version. No step is - handed GITHUB_TOKEN explicitly; setup-uv and setup-bun take it by - default, to read public releases.""" +def test_ci_uploads_the_database_without_a_token(): + """actions/upload-artifact v4 has no token input: it uploads through the + Actions runtime token (ACTIONS_RUNTIME_TOKEN), which is why ci.yml needs + no actions scope. Re-check this when the action changes major version. + No step is handed GITHUB_TOKEN explicitly; setup-uv and setup-bun take + it by default, to read public releases. publish-db.yml downloads the + artifact once the run completes (tests/test_publish_db.py).""" text = CI.read_text() assert "secrets.GITHUB_TOKEN" not in text and "github.token" not in text jobs = ci()["jobs"] @@ -210,15 +306,132 @@ def test_publish_db_downloads_this_runs_database_without_a_token(): for s in jobs["python"]["steps"] if s.get("uses", "").startswith("actions/upload-artifact") ] - (download,) = [ - s - for s in jobs["publish-db"]["steps"] - if s.get("uses", "").startswith("actions/download-artifact") - ] assert upload["uses"] == "actions/upload-artifact@v4" - assert download["uses"] == "actions/download-artifact@v4" - assert download["with"] == {"name": upload["with"]["name"]} - assert jobs["publish-db"]["needs"] == "python" + assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" + assert "publish-db" not in jobs + assert "download-artifact" not in text + + +def test_publish_db_reads_what_it_needs_and_nothing_more(): + workflow = yaml.safe_load(PUBLISH.read_text()) + assert granted(workflow, "publish-db") == PUBLISH_NEEDS["publish-db"] + assert overreach(workflow, PUBLISH_NEEDS) == [] + + +@pytest.mark.parametrize( + "scopes", + [ + "write-all", + "read-all", + {"actions": "write"}, + {"actions": "read", "contents": "write"}, + {"actions": "read", "contents": "read", "id-token": "write"}, + ], +) +def test_the_check_catches_a_wider_publish_token(scopes): + workflow = yaml.safe_load(PUBLISH.read_text()) + workflow["jobs"]["publish-db"]["permissions"] = scopes + assert overreach(workflow, PUBLISH_NEEDS) + + +def test_no_workflow_that_runs_on_pull_requests_names_a_secret(): + """ci.yml runs PyPI packages and each certified variant's pins on pull + requests, as a user with passwordless sudo. No workflow that runs on + pull requests may name a secret, even in a job or step that never runs + for one: publish-db named the Supabase secrets inside ci.yml, push-only, + until it moved to publish-db.yml.""" + found = { + name: named_secrets(workflow) - SECRETS_ON_PULL_REQUESTS + for name, workflow in workflows().items() + if triggers(workflow) & PULL_REQUEST_EVENTS + } + assert {"ci.yml", "gate-freshness.yml"} <= set(found) + assert {name: names for name, names in found.items() if names} == {} + + +def test_only_publish_db_names_the_supabase_secrets(): + """And the scanner sees them where they are.""" + named = {name: named_secrets(w) for name, w in workflows().items()} + assert {name for name, names in named.items() if names & SUPABASE} == { + "publish-db.yml" + } + assert named["publish-db.yml"] == SUPABASE + assert not triggers(workflows()["publish-db.yml"]) & PULL_REQUEST_EVENTS + + +@pytest.mark.parametrize( + "edit", + [ + lambda w: w["jobs"]["app"]["steps"][0].update( + env={"K": "${{ secrets.SUPABASE_SCORECARD_KEY }}"} + ), + lambda w: w["jobs"]["app"]["steps"][0].update( + {"with": {"k": "${{ secrets['ANY_TOKEN'] }}"}} + ), + lambda w: w["jobs"]["app"]["steps"][0].update( + {"with": {"k": '${{ secrets["ANY_TOKEN"] }}'}} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo '${{ toJSON(secrets) }}'"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ secrets[format('{0}_KEY', 'X')] }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"if": "secrets.ANY_TOKEN != ''", "run": "true"} + ), + lambda w: w.update(env={"K": "${{ Secrets . any_token }}"}), + lambda w: w["jobs"].update( + call={"uses": "./.github/workflows/x.yml", "secrets": "inherit"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ format('}}', secrets.SUPABASE_SCORECARD_KEY) }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"run": "echo ${{ format('it''s }}', secrets.ANY_TOKEN) }}"} + ), + lambda w: w["jobs"]["app"]["steps"].append( + {"if": "${{ '}}' != secrets.ANY_TOKEN }}", "run": "true"} + ), + ], + ids=[ + "step-env", + "bracket-single", + "bracket-double", + "to-json", + "computed-name", + "bare-if", + "case-and-spaces", + "reusable-inherit", + "braces-in-a-string", + "escaped-quote-then-braces", + "braces-in-an-if", + ], +) +def test_the_scanner_catches_every_way_to_name_a_secret(edit): + workflow = ci() + assert named_secrets(workflow) == set() + edit(workflow) + assert named_secrets(workflow) - SECRETS_ON_PULL_REQUESTS + + +def test_the_template_reader_ends_where_github_does(): + assert templates("a ${{ x }} b ${{ y }}") == [" x ", " y "] + assert templates("${{ format('}}', s) }}") == [" format('}}', s) "] + assert templates("${{ 'it''s }}' }} tail") == [" 'it''s }}' "] + assert templates("no expression }}") == [] + + +def test_the_scanner_ignores_what_names_no_secret(): + """A property called secrets, the word in a script, and comments.""" + workflow = yaml.safe_load(CI.read_text() + " # ${{ secrets.IN_A_COMMENT }}\n") + workflow["jobs"]["app"]["steps"].append( + { + "if": "github.event.inputs.secrets == 'x'", + "run": "echo 'no secrets here' ${{ github.event.inputs.secrets }}", + } + ) + assert named_secrets(workflow) == set() @pytest.mark.skipif(not (ROOT / ".git").exists(), reason="needs a git checkout") diff --git a/tests/test_publish_db.py b/tests/test_publish_db.py new file mode 100644 index 000000000..750189ecf --- /dev/null +++ b/tests/test_publish_db.py @@ -0,0 +1,498 @@ +"""publish-db.yml: the one workflow that names the Supabase secrets. + +It publishes the database ci.yml's pytest job built, from that run's +artifact, once CI completes on a push to main. It moved out of ci.yml so that +no workflow that runs on pull requests names a secret +(tests/test_ci_gates.py). These tests pin what the move had to keep: it +publishes only for main's own pushes, only what passed CI's pytest job, and +runs no package code beside the key; latest.db.gz moves only to main's tip; +unset secrets skip with a notice, a failed upload is a warning, and curl never +prints the host. The step scripts run here under +`bash -e`, the shell GitHub's log shows for them, with `gh` and `curl` stubs. +""" + +import gzip +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS = ROOT / ".github" / "workflows" +CI = WORKFLOWS / "ci.yml" +PUBLISH = WORKFLOWS / "publish-db.yml" + +FIND = "Find the database CI built" +DOWNLOAD = "Download it" +UPLOAD = "Publish it" +# v4.3.0, the commit actions/download-artifact's v4 tag also names. +DOWNLOAD_ACTION = "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093" +SHA = "0123456789abcdef0123456789abcdef01234567" +NEWER = "fedcba9876543210fedcba9876543210fedcba98" +RUN_ID = "39698300855" +ATTEMPT = "2" +REPOSITORY = "PolicyEngine/policyengine-scorecard" +HOST = "secret-project-ref.supabase.co" +KEY = "sb_secret_key_value" + + +def workflow() -> dict: + return yaml.safe_load(PUBLISH.read_text()) + + +def job() -> dict: + return workflow()["jobs"]["publish-db"] + + +def step(name: str) -> dict: + (found,) = [s for s in job()["steps"] if s.get("name") == name] + return found + + +def test_it_runs_when_ci_completes_on_main(): + ci = yaml.safe_load(CI.read_text()) + assert workflow()[True] == { + "workflow_run": { + "workflows": [ci["name"]], + "types": ["completed"], + "branches": ["main"], + } + } + assert ci[True]["push"] == {"branches": ["main"]} + assert list(workflow()["jobs"]) == ["publish-db"] + # One publish at a time, so an older tip's cannot land behind a newer one's, + # and up to 100 waiting: by default GitHub cancels a group's pending job + # when another enters. On the job: a workflow-level group would hold every + # run of the file, the ones the guard skips included. + assert "concurrency" not in workflow() + assert job()["concurrency"] == {"group": "publish-db", "queue": "max"} + # A stall must not hold that queue. Each upload's limit turns a stall into + # the warning, and the steps before them fail the run. A limit on the job + # or the publish step could end it inside an upload, before the warning. + assert "--max-time 120 " in step(UPLOAD)["run"] + assert step(FIND)["timeout-minutes"] == step(DOWNLOAD)["timeout-minutes"] == 5 + assert "timeout-minutes" not in job() and "timeout-minutes" not in step(UPLOAD) + + +def test_it_is_the_only_workflow_run_workflow(): + """A workflow_run workflow runs with the repository's secrets for every + run it follows, pull requests' included. Another one needs the same + guard as publish-db and its own test.""" + found = { + path.name + for path in WORKFLOWS.glob("*.y*ml") + if "workflow_run" in yaml.safe_load(path.read_text())[True] + } + assert found == {"publish-db.yml"} + + +# The guard, one clause per field of the triggering run. The fields are the +# run object's: GET /repos/{owner}/{repo}/actions/runs returns `event`, +# `head_branch`, `head_repository.full_name` and `path` for this repository's +# CI runs, and the workflow_run event carries that object. +GUARD = { + "github.event.workflow_run.event == 'push'", + "github.event.workflow_run.head_branch == 'main'", + "github.event.workflow_run.head_repository.full_name == github.repository", + "github.event.workflow_run.path == '.github/workflows/ci.yml'", +} + + +def guard_admits(run: dict, repository: str = REPOSITORY) -> bool: + """The job's `if`, evaluated: a conjunction of equalities, each between + a field of the triggering run and a quoted literal or github.repository. + GitHub compares strings ignoring case, and so does this.""" + condition = job()["if"] + assert "||" not in condition and "!" not in condition + context = {"github": {"repository": repository, "event": {"workflow_run": run}}} + + def value(term): + term = term.strip() + if term.startswith("'"): + return term.strip("'") + node = context + for part in term.split("."): + node = node[part] + return node + + clauses = [c.split("==") for c in condition.split("&&")] + return all(value(left).lower() == value(right).lower() for left, right in clauses) + + +def ci_run(**changes) -> dict: + run = { + "event": "push", + "head_branch": "main", + "head_repository": {"full_name": REPOSITORY}, + "path": ".github/workflows/ci.yml", + } + return {**run, **changes} + + +def test_the_guard_admits_only_cis_pushes_to_main_here(): + """workflow_run runs main's copy of the file with the repository's + secrets after any CI run, a fork's pull request included, and the + `branches` filter names a branch only: a fork's pull request from a + branch called main gets past it.""" + assert {c.strip() for c in job()["if"].split("&&")} == GUARD + assert guard_admits(ci_run()) + fork = {"full_name": "someone/policyengine-scorecard"} + for refused in ( + ci_run(event="pull_request"), + ci_run(event="pull_request", head_repository=fork), + ci_run(event="pull_request_target"), + ci_run(event="workflow_dispatch"), + ci_run(head_branch="feature"), + ci_run(head_repository=fork), + ci_run(path=".github/workflows/other.yml"), + ): + assert not guard_admits(refused), refused + + +def test_it_downloads_what_ci_uploaded_for_that_commit(): + ci = yaml.safe_load(CI.read_text()) + (upload,) = [ + s + for s in ci["jobs"]["python"]["steps"] + if s.get("uses", "").startswith("actions/upload-artifact") + ] + assert upload["with"]["name"] == "scorecard-db-${{ github.sha }}" + assert upload["with"]["path"] == "data/scorecard.db" + # So any artifact of this name passed the job's checks, whichever attempt + # uploaded it and whichever one the download takes. + steps = ci["jobs"]["python"]["steps"] + assert steps[-1] is upload and "if" not in upload + assert not any("continue-on-error" in s for s in steps) + download = step(DOWNLOAD) + assert download["uses"] == DOWNLOAD_ACTION + assert download["with"] == { + "name": "scorecard-db-${{ github.event.workflow_run.head_sha }}", + "run-id": "${{ github.event.workflow_run.id }}", + "github-token": "${{ github.token }}", + } + assert download["if"] == step(UPLOAD)["if"] == "steps.find.outputs.found == 'true'" + assert step(FIND)["id"] == "find" + assert 'artifact="scorecard-db-$CI_SHA"' in step(FIND)["run"] + assert job()["env"] == { + "CI_RUN_ID": "${{ github.event.workflow_run.id }}", + "CI_ATTEMPT": "${{ github.event.workflow_run.run_attempt }}", + "CI_SHA": "${{ github.event.workflow_run.head_sha }}", + } + + +def test_find_reads_the_job_ci_names_pytest(): + """A rename in ci.yml would otherwise leave nothing to match, and every + publish would skip with a notice.""" + name = yaml.safe_load(CI.read_text())["jobs"]["python"]["name"] + assert f'select(.name == "{name}")' in step(FIND)["run"] + + +def test_no_script_interpolates_the_triggering_run(): + """A pull request's author chooses its branch name, and GitHub pastes a + ${{ }} into a script before the shell reads it. The scripts take the run + id and commit from the environment and interpolate nothing.""" + for s in job()["steps"]: + assert "${{" not in s.get("run", ""), s.get("name") + + +def test_no_script_reads_github_sha(): + """In a workflow_run run GITHUB_SHA is not the CI commit, and a re-run + keeps its original value, so it is not main's tip either.""" + for s in job()["steps"]: + assert "GITHUB_SHA" not in s.get("run", ""), s.get("name") + + +def test_the_job_runs_no_package_code(): + """No checkout and no package manager or interpreter: what runs beside + the key is actions/download-artifact and the runner image's gh, gzip + and curl.""" + assert [s["uses"] for s in job()["steps"] if "uses" in s] == [DOWNLOAD_ACTION] + commands = re.compile( + r"\b(uvx?|pip3?|python3?|bun|bunx|npm|npx|node|make|bash|sh|source)\b" + r"|\.github/scripts|\./" + ) + for s in job()["steps"]: + assert not commands.search(s.get("run", "")), s["name"] + + +def test_the_tokens_are_where_they_are_used(): + """GITHUB_TOKEN reaches the steps that read CI's run, and the Supabase + secrets the step that uploads, never the workflow or job environment.""" + with_github_token = [ + s["name"] for s in job()["steps"] if "github.token" in json.dumps(s) + ] + assert with_github_token == [FIND, DOWNLOAD] + assert step(FIND)["env"] == {"GH_TOKEN": "${{ github.token }}"} + with_secrets = [s["name"] for s in job()["steps"] if "secrets." in json.dumps(s)] + assert with_secrets == [UPLOAD] + assert step(UPLOAD)["env"] == { + "MAIN_TIP": "${{ steps.find.outputs.main_tip }}", + "SUPABASE_URL": "${{ secrets.SUPABASE_SCORECARD_URL }}", + "SUPABASE_KEY": "${{ secrets.SUPABASE_SCORECARD_KEY }}", + } + assert "env" not in workflow() + + +# Stand-ins for the runner image's gh and curl. Each records its arguments +# and the variables the script reads; curl also keeps the bytes it would +# send. gh answers by endpoint, and fails for the one named in `fails_on`. +# Like the real curl, a failing stub prints its error, host included, unless +# one of its short options includes -s. +GH_STUB = """#!{python} -I +import json, os, sys +with open({record!r}, "a") as handle: + handle.write(json.dumps({{"argv": sys.argv[1:], "token": os.environ.get("GH_TOKEN")}}) + "\\n") +endpoint = sys.argv[2] +if {fails_on!r} and {fails_on!r} in endpoint: + sys.stderr.write("HTTP 502\\n") + sys.exit(1) +print(next(answer for path, answer in {answers!r}.items() if path in endpoint)) +""" +CURL_STUB = """#!{python} -I +import json, sys +args = sys.argv[1:] +payload = args[args.index("--data-binary") + 1].removeprefix("@") +with open(payload, "rb") as handle: + sent = handle.read().hex() +with open({record!r}, "a") as handle: + handle.write(json.dumps({{"argv": args, "sent": sent}}) + "\\n") +if {status}: + silent = any(a.startswith("-") and not a.startswith("--") and "s" in a for a in args) + if not silent: + url = [a for a in args if a.startswith("https://")][0] + sys.stderr.write("curl: (6) Could not resolve host: " + url.split("/")[2] + "\\n") + sys.exit({status}) +""" + + +def run_step( + tmp_path, + name, + *, + env=(), + script=None, + pytest_conclusion="success", + artifacts="1", + fails_on="", + status=0, +): + """``name``'s script under `bash -e`, in a fresh workspace holding a + database, with gh and curl stubs first on PATH.""" + work = tmp_path / "work" + work.mkdir() + (work / "scorecard.db").write_bytes(b"SQLite format 3\x00" + bytes(range(256))) + stubs = tmp_path / "stubs" + stubs.mkdir() + records = {tool: tmp_path / f"{tool}.jsonl" for tool in ("gh", "curl")} + (stubs / "gh").write_text( + GH_STUB.format( + python=sys.executable, + record=str(records["gh"]), + answers={ + "/jobs": pytest_conclusion, + "/artifacts": artifacts, + "/git/ref/": NEWER, + }, + fails_on=fails_on, + ) + ) + (stubs / "curl").write_text( + CURL_STUB.format( + python=sys.executable, record=str(records["curl"]), status=status + ) + ) + for stub in stubs.iterdir(): + stub.chmod(0o755) + path = tmp_path / "step.sh" + path.write_text(script if script is not None else step(name)["run"]) + output, summary = tmp_path / "github_output", tmp_path / "step_summary" + output.touch() + summary.touch() + environment = { + "PATH": f"{stubs}:/usr/bin:/bin", + "HOME": str(tmp_path), + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_SERVER_URL": "https://github.com", + "CI_RUN_ID": RUN_ID, + "CI_ATTEMPT": ATTEMPT, + "CI_SHA": SHA, + "MAIN_TIP": SHA, + # What the runner sets on a re-run of this commit's publish, whatever + # main has moved to since. + "GITHUB_SHA": SHA, + "GH_TOKEN": "ghs_runtoken", + "SUPABASE_URL": f"https://{HOST}", + "SUPABASE_KEY": KEY, + **dict(env), + } + result = subprocess.run( + ["bash", "-e", str(path)], + cwd=work, + env=environment, + capture_output=True, + text=True, + ) + calls = { + tool: [json.loads(line) for line in record.read_text().splitlines()] + if record.exists() + else [] + for tool, record in records.items() + } + return result, calls, output.read_text(), summary.read_text(), work + + +def test_find_says_found_when_ci_passed_and_uploaded_the_database(tmp_path): + result, calls, output, _, _ = run_step(tmp_path, FIND) + assert result.returncode == 0, result.stderr + assert output == f"found=true\nmain_tip={NEWER}\n" + assert "::notice::" not in result.stdout + run = f"repos/{REPOSITORY}/actions/runs/{RUN_ID}" + assert calls["gh"] == [ + { + "argv": [ + "api", + f"{run}/attempts/{ATTEMPT}/jobs?per_page=100", + "--jq", + '.jobs[] | select(.name == "pytest · ruff format") | .conclusion', + ], + "token": "ghs_runtoken", + }, + { + "argv": [ + "api", + f"{run}/artifacts?name=scorecard-db-{SHA}", + "--jq", + "[.artifacts[] | select(.expired | not)] | length", + ], + "token": "ghs_runtoken", + }, + { + "argv": [ + "api", + f"repos/{REPOSITORY}/git/ref/heads/main", + "--jq", + ".object.sha", + ], + "token": "ghs_runtoken", + }, + ] + + +def test_find_skips_with_a_notice_when_the_pytest_job_uploaded_nothing(tmp_path): + result, calls, output, _, _ = run_step(tmp_path, FIND, artifacts="0") + assert result.returncode == 0, result.stderr + assert output == f"found=false\nmain_tip={NEWER}\n" + assert result.stdout == ( + f"::notice::CI run {RUN_ID} attempt {ATTEMPT} has nothing to publish: its " + f"pytest job ended success, with 0 scorecard-db-{SHA}\n" + ) + assert not calls["curl"] + + +@pytest.mark.parametrize("conclusion", ["failure", "cancelled", "skipped", ""]) +def test_find_skips_an_artifact_this_attempts_pytest_job_did_not_pass( + tmp_path, conclusion +): + """An artifact can outlive the attempt that uploaded it: a re-run of a + job that passed shares its attempt's artifact (run 39698458972), so the + artifact alone does not say the attempt that fired this run passed.""" + result, _, output, _, _ = run_step(tmp_path, FIND, pytest_conclusion=conclusion) + assert result.returncode == 0, result.stderr + assert output == f"found=false\nmain_tip={NEWER}\n" + assert f"pytest job ended {conclusion or 'unknown'}," in result.stdout + + +@pytest.mark.parametrize("endpoint", ["/jobs", "/artifacts", "/git/ref/"]) +def test_find_fails_the_run_when_it_cannot_ask(tmp_path, endpoint): + result, _, output, _, _ = run_step(tmp_path, FIND, fails_on=endpoint) + assert result.returncode != 0 + assert output == "" + + +@pytest.mark.parametrize("unset", ["SUPABASE_URL", "SUPABASE_KEY"]) +def test_publish_skips_with_a_notice_when_a_secret_is_unset(tmp_path, unset): + result, calls, _, summary, work = run_step(tmp_path, UPLOAD, env={unset: ""}) + assert result.returncode == 0, result.stderr + assert result.stdout == "::notice::Supabase secrets not set — skipping publish\n" + assert not calls["curl"] and summary == "" + assert not (work / "scorecard.db.gz").exists() + + +def test_publish_uploads_the_commits_database_then_latest(tmp_path): + result, calls, _, summary, work = run_step(tmp_path, UPLOAD) + assert result.returncode == 0, result.stderr + names = [f"{SHA}.db.gz", "latest.db.gz"] + assert result.stdout == "".join( + f"uploaded scorecard-artifacts/{n}\n" for n in names + ) + assert summary == "" + database = (work / "scorecard.db").read_bytes() + for call, name in zip(calls["curl"], names, strict=True): + assert call["argv"] == [ + "-sf", + "--max-time", + "120", + "-X", + "POST", + f"https://{HOST}/storage/v1/object/scorecard-artifacts/{name}", + "-H", + f"Authorization: Bearer {KEY}", + "-H", + "Content-Type: application/gzip", + "-H", + "x-upsert: true", + "--data-binary", + "@scorecard.db.gz", + ] + assert gzip.decompress(bytes.fromhex(call["sent"])) == database + + +def test_latest_waits_for_mains_tip(tmp_path): + """When main has moved on (a newer push cancelled this run, or an older + CI or publish run was re-run), the commit's own copy is uploaded and + latest.db.gz is not rolled back. On a re-run of an old publish GITHUB_SHA + is still that commit, as run_step sets it, so only the tip the find step + read tells the two apart.""" + result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, env={"MAIN_TIP": NEWER}) + assert result.returncode == 0, result.stderr + assert result.stdout == ( + f"::notice::main is at {NEWER} now, not {SHA}: latest.db.gz is left " + f"to the run for main's tip\nuploaded scorecard-artifacts/{SHA}.db.gz\n" + ) + assert [call["argv"][5].rsplit("/", 1)[1] for call in calls["curl"]] == [ + f"{SHA}.db.gz" + ] + assert summary == "" + + +def test_a_failed_upload_is_a_warning_and_never_prints_the_host(tmp_path): + result, calls, _, summary, _ = run_step(tmp_path, UPLOAD, status=6) + assert result.returncode == 0, result.stderr + assert len(calls["curl"]) == 1 + message = ( + f"curl exit 6 uploading scorecard-artifacts/{SHA}.db.gz: the published " + f"database was not updated. The database CI built is the " + f"scorecard-db-{SHA} artifact of " + f"https://github.com/{REPOSITORY}/actions/runs/{RUN_ID}." + ) + assert result.stdout == f"::warning title=Supabase publish failed::{message}\n" + assert summary == f"**Supabase publish failed.** {message}\n" + for shown in (result.stdout, result.stderr, summary): + assert HOST not in shown and KEY not in shown + + +def test_without_curls_silence_the_host_would_print(tmp_path): + """The stub does print the host when curl is not silenced, so the test + above is not passing for want of anything to print.""" + script = step(UPLOAD)["run"] + assert script.count("curl -sf ") == 1 + result, *_ = run_step( + tmp_path, UPLOAD, status=6, script=script.replace("curl -sf ", "curl -f ") + ) + assert HOST in result.stderr