From 1d99bf410e394891b0dc5f2f355aaa215cdba3b0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 06:44:23 -0400 Subject: [PATCH 1/3] Open a weekly PR that refreshes uv.lock CI's test suite installs uv.lock exactly as committed, so a new upstream release reaches it only when the lock changes. This workflow runs `uv lock --upgrade` on main every Monday and opens a PR with the result. It runs as two jobs because resolving can run a dependency's build backend. `lock` resolves with no secrets, a read-only GITHUB_TOKEN and no uv cache, and passes the new uv.lock on as an artifact. `publish` starts on a fresh runner, mints the org GitHub App token (scoped to contents and pull requests) so the PR gets CI, runs no dependency code, and makes every decision itself. It keeps at most one update PR open. It refreshes that PR while every commit on it, other than merges of main, is the workflow's own and it changes only the lock and the fragment; it leaves the PR alone once anyone else commits, and closes it when main's lock is already current. Fixes #2568 Co-Authored-By: Claude Opus 5.5 --- .github/CONTRIBUTING.md | 6 + .github/workflows/weekly-uv-lock.yaml | 289 ++++++++++++++++++++++++++ changelog.d/weekly-uv-lock.changed.md | 1 + 3 files changed, 296 insertions(+) create mode 100644 .github/workflows/weekly-uv-lock.yaml create mode 100644 changelog.d/weekly-uv-lock.changed.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 77a63f217e..1c2ad7fc9e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -14,6 +14,12 @@ uv run pytest policyengine_uk/tests/path/to/test_file.py::test_name -v The package supports Python 3.11–3.14. Default branch: `main`. +## Dependencies + +CI's test suite installs exactly what `uv.lock` pins (`uv sync --frozen`). If you change dependencies in `pyproject.toml`, run `uv lock` and commit `uv.lock` with the change; otherwise CI tests the dependencies in the old lock, not the ones you asked for. + +Every Monday, `.github/workflows/weekly-uv-lock.yaml` runs `uv lock --upgrade` on `main` and opens a pull request with the result, or refreshes the one already open. That is how CI moves to new dependency releases. Merge it when its CI passes. If a new release breaks a test, push the fix to that pull request's branch; the workflow then stops refreshing it. To take one release sooner, run `uv lock --upgrade-package ` in your own pull request. + ## Writing variables and reforms Four types of files usually change together: diff --git a/.github/workflows/weekly-uv-lock.yaml b/.github/workflows/weekly-uv-lock.yaml new file mode 100644 index 0000000000..5416df386e --- /dev/null +++ b/.github/workflows/weekly-uv-lock.yaml @@ -0,0 +1,289 @@ +# Runs `uv lock --upgrade` on main each week and opens a pull request with the +# result. +# +# CI's test suite installs uv.lock exactly as committed (`uv sync --frozen` in +# .github/actions/test-environment), so a new release of policyengine-core or +# any other dependency reaches it only when the lock changes. This PR is that +# change, and its CI runs the full suite against the new releases before they +# land. +# +# Two jobs, because resolving can run a dependency's build backend: +# +# - `lock` runs `uv lock --upgrade` with no secrets and a read-only +# GITHUB_TOKEN, and hands the new uv.lock to `publish` as an artifact. +# - `publish` starts on a fresh runner, mints a GitHub App token from the +# org-level APP_ID / APP_PRIVATE_KEY secrets, and runs no dependency code. +# It treats the artifact as data and decides everything else itself. +# +# The App token is needed because GitHub does not start workflows for pushes +# or PRs made with GITHUB_TOKEN, so a PR opened with it would get no CI. +# +# There is at most one open update PR. Each run refreshes it with the newest +# releases, unless someone else has committed to it (say, a fix for a test the +# new releases break); then the run leaves it alone. Merging main into it +# ("Update branch") does not count. A run that finds main's lock already +# current closes it. + +name: Weekly uv.lock update + +on: + schedule: + # Mondays at 06:00 UTC, before the UK working day. + - cron: "0 6 * * 1" + workflow_dispatch: + +# Each job grants its own GITHUB_TOKEN permissions, all read-only. +permissions: {} + +concurrency: + group: weekly-uv-lock + cancel-in-progress: false + +env: + BRANCH: bot/weekly-uv-lock-update + FRAGMENT: changelog.d/bot-weekly-uv-lock-update.changed.md + BOT_NAME: github-actions[bot] + BOT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com + GH_REPO: ${{ github.repository }} + +jobs: + lock: + name: Upgrade uv.lock + runs-on: ubuntu-latest + timeout-minutes: 15 + # Both jobs check out github.sha, the main commit this run started from. + if: github.repository == 'PolicyEngine/policyengine-uk' && github.ref == 'refs/heads/main' + permissions: + contents: read + defaults: + run: + # Explicit bash adds -o pipefail, so `uv lock | tee` fails when uv does. + shell: bash + steps: + - name: Checkout main + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + # The same unpinned setup as the CI workflows: the latest uv release. + # No cache: this job runs dependency build code, and a cache it saved + # would be restored by other workflows on main. + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Upgrade uv.lock + run: | + mkdir -p "$RUNNER_TEMP/lock" + uv lock --upgrade 2>&1 | tee "$RUNNER_TEMP/uv-lock.log" + grep -E '^(Updated|Added|Removed) ' "$RUNNER_TEMP/uv-lock.log" \ + > "$RUNNER_TEMP/lock/changes.txt" || true + cp uv.lock "$RUNNER_TEMP/lock/uv.lock" + + - name: Upload the new lock + uses: actions/upload-artifact@v7 + with: + name: uv-lock + path: ${{ runner.temp }}/lock + retention-days: 1 + if-no-files-found: error + overwrite: true + + publish: + name: Open or update the PR + needs: lock + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + defaults: + run: + shell: bash + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Checkout main + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download the new lock + uses: actions/download-artifact@v8 + with: + name: uv-lock + path: ${{ runner.temp }}/lock + + - name: Apply the new lock + run: | + cp "$RUNNER_TEMP/lock/uv.lock" uv.lock + # Keep only well-formed uv summary lines for the PR body. + grep -E '^(Updated|Added|Removed) [A-Za-z0-9._-]+ v[^`<]*$' \ + "$RUNNER_TEMP/lock/changes.txt" > "$RUNNER_TEMP/changes.txt" || true + + - name: Find an open update PR + id: existing + env: + GH_TOKEN: ${{ github.token }} + run: | + # --head matches the branch name only, so drop fork PRs that reuse it. + number=$(gh pr list --head "$BRANCH" --base main --state open \ + --json number,isCrossRepository \ + --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty') + echo "number=$number" >> "$GITHUB_OUTPUT" + if [ -z "$number" ]; then + # The branch can outlive a merged or closed PR. Record its head (or + # nothing) so the push below fails if anyone moves it meanwhile. + head=$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1) + echo "head=$head" >> "$GITHUB_OUTPUT" + echo "action=create" >> "$GITHUB_OUTPUT" + exit 0 + fi + head=$(gh api "repos/$GH_REPO/pulls/$number" --jq .head.sha) + echo "head=$head" >> "$GITHUB_OUTPUT" + # The PR is this workflow's own if every commit on it, other than + # merges of main, is the workflow's, and it changes only the lock and + # the fragment. A merge that resolved a conflict in any other file + # shows up in the file list. Fetch first, so a failed API call stops + # the run instead of reading as "nobody else committed". + gh api --paginate "repos/$GH_REPO/pulls/$number/commits" \ + --jq '.[] | select((.parents | length) == 1) | .commit.author.email' \ + > "$RUNNER_TEMP/authors.txt" + gh api --paginate "repos/$GH_REPO/pulls/$number/files" --jq '.[].filename' \ + > "$RUNNER_TEMP/files.txt" + others=$(sort -u "$RUNNER_TEMP/authors.txt" | grep -vxF "$BOT_EMAIL" | paste -sd' ' - || true) + files=$(grep -vxF -e uv.lock -e "$FRAGMENT" "$RUNNER_TEMP/files.txt" | paste -sd' ' - || true) + if [ -z "$others" ] && [ -z "$files" ]; then + echo "action=refresh" >> "$GITHUB_OUTPUT" + git fetch --quiet --depth=1 origin "$head" + else + echo "action=skip" >> "$GITHUB_OUTPUT" + echo "::notice::#$number has commits from someone other than this workflow (${others:-none}) or changes other files (${files:-none}), so it is left as it is." + fi + + - name: Decide what to do + id: changes + if: steps.existing.outputs.action != 'skip' + env: + ACTION: ${{ steps.existing.outputs.action }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + if git diff --quiet uv.lock; then + echo "push=false" >> "$GITHUB_OUTPUT" + echo "close=$([ "$ACTION" = refresh ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "::notice::uv.lock on main is already up to date." + elif [ "$ACTION" = refresh ] && git diff --quiet "$HEAD" -- uv.lock; then + # Nothing new since the open PR was last refreshed; pushing the + # same lock again would only rerun its CI. + echo "push=false" >> "$GITHUB_OUTPUT" + echo "close=false" >> "$GITHUB_OUTPUT" + echo "::notice::The open PR already has this lock." + else + echo "push=true" >> "$GITHUB_OUTPUT" + echo "close=false" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push + if: steps.changes.outputs.push == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + # shellcheck disable=SC2016 # the backticks are Markdown, not a command + echo '- Updated `uv.lock` to the newest dependency releases.' > "$FRAGMENT" + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + git checkout -B "$BRANCH" + git add uv.lock "$FRAGMENT" + git commit --quiet -m "Update uv.lock dependencies" + # Replaces the branch. With an open PR, the check above found only + # this workflow's commits on it. Without one, the branch is left over + # from a merged or closed PR and is overwritten. The lease refuses the + # push if the branch moved since it was checked above. + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push --force-with-lease="refs/heads/$BRANCH:$HEAD" origin "HEAD:refs/heads/$BRANCH" + + # REST calls rather than `gh pr edit`, which also reads project fields + # that a token scoped to contents and pull requests cannot. + - name: Open or update the PR + if: steps.changes.outputs.push == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ACTION: ${{ steps.existing.outputs.action }} + NUMBER: ${{ steps.existing.outputs.number }} + run: | + body="$RUNNER_TEMP/body.md" + count=$(wc -l < "$RUNNER_TEMP/changes.txt" | tr -d ' ') + { + echo "Weekly refresh of \`uv.lock\` by \`.github/workflows/weekly-uv-lock.yaml\`: \`uv lock --upgrade\` on \`main\`." + echo + echo "CI's test suite installs \`uv.lock\` exactly as committed, so this PR is how new dependency releases reach it. Its CI runs the full suite against the versions below." + echo + echo "- CI passes: merge it through the merge queue. There is no need to update the branch first; each run rebuilds it from \`main\`." + echo "- CI fails: push a fix to this branch. Once anyone else commits here, the workflow stops refreshing this PR." + echo "- Not wanted this week: close it. The next run opens a new one." + echo + echo "axiom: n/a: dependency lock update" + echo + if [ "$count" -eq 0 ]; then + echo "No package versions changed, but other contents of the lock did; see the diff." + else + echo "
Changes from uv lock --upgrade ($count)" + echo + echo '```' + cat "$RUNNER_TEMP/changes.txt" + echo '```' + echo + echo "
" + fi + } > "$body" + if [ "$ACTION" = create ]; then + number=$(gh api "repos/$GH_REPO/pulls" -f base=main -f head="$BRANCH" \ + -f title="Update uv.lock to the newest dependency releases" \ + -F body=@"$body" --jq .number) + echo "Opened #$number" + gh api "repos/$GH_REPO/issues/$number/labels" -f 'labels[]=dependencies' > /dev/null \ + || echo "::warning::Could not add the dependencies label to #$number." + else + gh api -X PATCH "repos/$GH_REPO/pulls/$NUMBER" -F body=@"$body" > /dev/null + echo "Updated #$NUMBER" + fi + { + echo "### uv lock --upgrade" + echo + echo '```' + cat "$RUNNER_TEMP/changes.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Close the PR if main is already current + if: steps.changes.outputs.close == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + NUMBER: ${{ steps.existing.outputs.number }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + gh api "repos/$GH_REPO/issues/$NUMBER/comments" \ + -f body="\`uv lock --upgrade\` on \`main\` changes nothing now, so this update is no longer needed." \ + > /dev/null || echo "::warning::Could not comment on #$NUMBER." + gh api -X PATCH "repos/$GH_REPO/pulls/$NUMBER" -f state=closed > /dev/null + # Delete the branch only if it is still the head that was checked. + if [ "$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)" = "$HEAD" ]; then + gh api -X DELETE "repos/$GH_REPO/git/refs/heads/$BRANCH" + fi diff --git a/changelog.d/weekly-uv-lock.changed.md b/changelog.d/weekly-uv-lock.changed.md new file mode 100644 index 0000000000..6539b479fd --- /dev/null +++ b/changelog.d/weekly-uv-lock.changed.md @@ -0,0 +1 @@ +- A weekly workflow now runs `uv lock --upgrade` and opens a pull request with the result, so CI, which installs from `uv.lock`, moves to new dependency releases. From 416f458f1cc71240e7763c6563df77451ea32bae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 09:57:26 -0400 Subject: [PATCH 2/3] Delete the closed update PR's branch with a lease, not a separate check The close step compared the branch head and then deleted it over REST, so a push landing between the two was deleted. A git push delete with --force-with-lease on the inspected head does both in one step. Also say that GITHUB_TOKEN PR runs wait for approval rather than not running. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/weekly-uv-lock.yaml | 16 ++++++++++------ changelog.d/weekly-uv-lock.changed.md | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-uv-lock.yaml b/.github/workflows/weekly-uv-lock.yaml index 5416df386e..b75f0c1f5f 100644 --- a/.github/workflows/weekly-uv-lock.yaml +++ b/.github/workflows/weekly-uv-lock.yaml @@ -15,8 +15,9 @@ # org-level APP_ID / APP_PRIVATE_KEY secrets, and runs no dependency code. # It treats the artifact as data and decides everything else itself. # -# The App token is needed because GitHub does not start workflows for pushes -# or PRs made with GITHUB_TOKEN, so a PR opened with it would get no CI. +# The App token is needed because GitHub starts no workflows for a push made +# with GITHUB_TOKEN, and holds the runs for a PR opened with it until someone +# approves them; with the App token the PR's CI starts on its own. # # There is at most one open update PR. Each run refreshes it with the newest # releases, unless someone else has committed to it (say, a fix for a test the @@ -283,7 +284,10 @@ jobs: -f body="\`uv lock --upgrade\` on \`main\` changes nothing now, so this update is no longer needed." \ > /dev/null || echo "::warning::Could not comment on #$NUMBER." gh api -X PATCH "repos/$GH_REPO/pulls/$NUMBER" -f state=closed > /dev/null - # Delete the branch only if it is still the head that was checked. - if [ "$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)" = "$HEAD" ]; then - gh api -X DELETE "repos/$GH_REPO/git/refs/heads/$BRANCH" - fi + # Delete the branch only if it is still the head that was checked: + # the lease makes the check and the delete one step, so a push that + # lands in between keeps the branch. + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push --force-with-lease="refs/heads/$BRANCH:$HEAD" origin ":refs/heads/$BRANCH" \ + || echo "::notice::$BRANCH moved after it was checked, so it was not deleted." diff --git a/changelog.d/weekly-uv-lock.changed.md b/changelog.d/weekly-uv-lock.changed.md index 6539b479fd..f34d6df956 100644 --- a/changelog.d/weekly-uv-lock.changed.md +++ b/changelog.d/weekly-uv-lock.changed.md @@ -1 +1 @@ -- A weekly workflow now runs `uv lock --upgrade` and opens a pull request with the result, so CI, which installs from `uv.lock`, moves to new dependency releases. +- A weekly workflow now runs `uv lock --upgrade` and opens a pull request with the result, so CI's test suite, which installs from `uv.lock`, moves to new dependency releases. From 050d96d81e0f4a813c6bc437c90f21144fbf6fbc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 10:02:59 -0400 Subject: [PATCH 3/3] Word the branch-delete notice for any push failure Co-Authored-By: Claude Opus 5.5 --- .github/workflows/weekly-uv-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-uv-lock.yaml b/.github/workflows/weekly-uv-lock.yaml index b75f0c1f5f..19840b2bd9 100644 --- a/.github/workflows/weekly-uv-lock.yaml +++ b/.github/workflows/weekly-uv-lock.yaml @@ -290,4 +290,4 @@ jobs: auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ push --force-with-lease="refs/heads/$BRANCH:$HEAD" origin ":refs/heads/$BRANCH" \ - || echo "::notice::$BRANCH moved after it was checked, so it was not deleted." + || echo "::notice::Did not delete $BRANCH: it moved after it was checked, or the push failed."