diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 77a63f217e..1c2ad7fc9e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -14,6 +14,12 @@ uv run pytest policyengine_uk/tests/path/to/test_file.py::test_name -v The package supports Python 3.11–3.14. Default branch: `main`. +## Dependencies + +CI's test suite installs exactly what `uv.lock` pins (`uv sync --frozen`). If you change dependencies in `pyproject.toml`, run `uv lock` and commit `uv.lock` with the change; otherwise CI tests the dependencies in the old lock, not the ones you asked for. + +Every Monday, `.github/workflows/weekly-uv-lock.yaml` runs `uv lock --upgrade` on `main` and opens a pull request with the result, or refreshes the one already open. That is how CI moves to new dependency releases. Merge it when its CI passes. If a new release breaks a test, push the fix to that pull request's branch; the workflow then stops refreshing it. To take one release sooner, run `uv lock --upgrade-package ` in your own pull request. + ## Writing variables and reforms Four types of files usually change together: diff --git a/.github/workflows/weekly-uv-lock.yaml b/.github/workflows/weekly-uv-lock.yaml new file mode 100644 index 0000000000..19840b2bd9 --- /dev/null +++ b/.github/workflows/weekly-uv-lock.yaml @@ -0,0 +1,293 @@ +# Runs `uv lock --upgrade` on main each week and opens a pull request with the +# result. +# +# CI's test suite installs uv.lock exactly as committed (`uv sync --frozen` in +# .github/actions/test-environment), so a new release of policyengine-core or +# any other dependency reaches it only when the lock changes. This PR is that +# change, and its CI runs the full suite against the new releases before they +# land. +# +# Two jobs, because resolving can run a dependency's build backend: +# +# - `lock` runs `uv lock --upgrade` with no secrets and a read-only +# GITHUB_TOKEN, and hands the new uv.lock to `publish` as an artifact. +# - `publish` starts on a fresh runner, mints a GitHub App token from the +# org-level APP_ID / APP_PRIVATE_KEY secrets, and runs no dependency code. +# It treats the artifact as data and decides everything else itself. +# +# The App token is needed because GitHub starts no workflows for a push made +# with GITHUB_TOKEN, and holds the runs for a PR opened with it until someone +# approves them; with the App token the PR's CI starts on its own. +# +# There is at most one open update PR. Each run refreshes it with the newest +# releases, unless someone else has committed to it (say, a fix for a test the +# new releases break); then the run leaves it alone. Merging main into it +# ("Update branch") does not count. A run that finds main's lock already +# current closes it. + +name: Weekly uv.lock update + +on: + schedule: + # Mondays at 06:00 UTC, before the UK working day. + - cron: "0 6 * * 1" + workflow_dispatch: + +# Each job grants its own GITHUB_TOKEN permissions, all read-only. +permissions: {} + +concurrency: + group: weekly-uv-lock + cancel-in-progress: false + +env: + BRANCH: bot/weekly-uv-lock-update + FRAGMENT: changelog.d/bot-weekly-uv-lock-update.changed.md + BOT_NAME: github-actions[bot] + BOT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com + GH_REPO: ${{ github.repository }} + +jobs: + lock: + name: Upgrade uv.lock + runs-on: ubuntu-latest + timeout-minutes: 15 + # Both jobs check out github.sha, the main commit this run started from. + if: github.repository == 'PolicyEngine/policyengine-uk' && github.ref == 'refs/heads/main' + permissions: + contents: read + defaults: + run: + # Explicit bash adds -o pipefail, so `uv lock | tee` fails when uv does. + shell: bash + steps: + - name: Checkout main + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + # The same unpinned setup as the CI workflows: the latest uv release. + # No cache: this job runs dependency build code, and a cache it saved + # would be restored by other workflows on main. + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Upgrade uv.lock + run: | + mkdir -p "$RUNNER_TEMP/lock" + uv lock --upgrade 2>&1 | tee "$RUNNER_TEMP/uv-lock.log" + grep -E '^(Updated|Added|Removed) ' "$RUNNER_TEMP/uv-lock.log" \ + > "$RUNNER_TEMP/lock/changes.txt" || true + cp uv.lock "$RUNNER_TEMP/lock/uv.lock" + + - name: Upload the new lock + uses: actions/upload-artifact@v7 + with: + name: uv-lock + path: ${{ runner.temp }}/lock + retention-days: 1 + if-no-files-found: error + overwrite: true + + publish: + name: Open or update the PR + needs: lock + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read + defaults: + run: + shell: bash + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Checkout main + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download the new lock + uses: actions/download-artifact@v8 + with: + name: uv-lock + path: ${{ runner.temp }}/lock + + - name: Apply the new lock + run: | + cp "$RUNNER_TEMP/lock/uv.lock" uv.lock + # Keep only well-formed uv summary lines for the PR body. + grep -E '^(Updated|Added|Removed) [A-Za-z0-9._-]+ v[^`<]*$' \ + "$RUNNER_TEMP/lock/changes.txt" > "$RUNNER_TEMP/changes.txt" || true + + - name: Find an open update PR + id: existing + env: + GH_TOKEN: ${{ github.token }} + run: | + # --head matches the branch name only, so drop fork PRs that reuse it. + number=$(gh pr list --head "$BRANCH" --base main --state open \ + --json number,isCrossRepository \ + --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty') + echo "number=$number" >> "$GITHUB_OUTPUT" + if [ -z "$number" ]; then + # The branch can outlive a merged or closed PR. Record its head (or + # nothing) so the push below fails if anyone moves it meanwhile. + head=$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1) + echo "head=$head" >> "$GITHUB_OUTPUT" + echo "action=create" >> "$GITHUB_OUTPUT" + exit 0 + fi + head=$(gh api "repos/$GH_REPO/pulls/$number" --jq .head.sha) + echo "head=$head" >> "$GITHUB_OUTPUT" + # The PR is this workflow's own if every commit on it, other than + # merges of main, is the workflow's, and it changes only the lock and + # the fragment. A merge that resolved a conflict in any other file + # shows up in the file list. Fetch first, so a failed API call stops + # the run instead of reading as "nobody else committed". + gh api --paginate "repos/$GH_REPO/pulls/$number/commits" \ + --jq '.[] | select((.parents | length) == 1) | .commit.author.email' \ + > "$RUNNER_TEMP/authors.txt" + gh api --paginate "repos/$GH_REPO/pulls/$number/files" --jq '.[].filename' \ + > "$RUNNER_TEMP/files.txt" + others=$(sort -u "$RUNNER_TEMP/authors.txt" | grep -vxF "$BOT_EMAIL" | paste -sd' ' - || true) + files=$(grep -vxF -e uv.lock -e "$FRAGMENT" "$RUNNER_TEMP/files.txt" | paste -sd' ' - || true) + if [ -z "$others" ] && [ -z "$files" ]; then + echo "action=refresh" >> "$GITHUB_OUTPUT" + git fetch --quiet --depth=1 origin "$head" + else + echo "action=skip" >> "$GITHUB_OUTPUT" + echo "::notice::#$number has commits from someone other than this workflow (${others:-none}) or changes other files (${files:-none}), so it is left as it is." + fi + + - name: Decide what to do + id: changes + if: steps.existing.outputs.action != 'skip' + env: + ACTION: ${{ steps.existing.outputs.action }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + if git diff --quiet uv.lock; then + echo "push=false" >> "$GITHUB_OUTPUT" + echo "close=$([ "$ACTION" = refresh ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "::notice::uv.lock on main is already up to date." + elif [ "$ACTION" = refresh ] && git diff --quiet "$HEAD" -- uv.lock; then + # Nothing new since the open PR was last refreshed; pushing the + # same lock again would only rerun its CI. + echo "push=false" >> "$GITHUB_OUTPUT" + echo "close=false" >> "$GITHUB_OUTPUT" + echo "::notice::The open PR already has this lock." + else + echo "push=true" >> "$GITHUB_OUTPUT" + echo "close=false" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push + if: steps.changes.outputs.push == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + # shellcheck disable=SC2016 # the backticks are Markdown, not a command + echo '- Updated `uv.lock` to the newest dependency releases.' > "$FRAGMENT" + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + git checkout -B "$BRANCH" + git add uv.lock "$FRAGMENT" + git commit --quiet -m "Update uv.lock dependencies" + # Replaces the branch. With an open PR, the check above found only + # this workflow's commits on it. Without one, the branch is left over + # from a merged or closed PR and is overwritten. The lease refuses the + # push if the branch moved since it was checked above. + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push --force-with-lease="refs/heads/$BRANCH:$HEAD" origin "HEAD:refs/heads/$BRANCH" + + # REST calls rather than `gh pr edit`, which also reads project fields + # that a token scoped to contents and pull requests cannot. + - name: Open or update the PR + if: steps.changes.outputs.push == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ACTION: ${{ steps.existing.outputs.action }} + NUMBER: ${{ steps.existing.outputs.number }} + run: | + body="$RUNNER_TEMP/body.md" + count=$(wc -l < "$RUNNER_TEMP/changes.txt" | tr -d ' ') + { + echo "Weekly refresh of \`uv.lock\` by \`.github/workflows/weekly-uv-lock.yaml\`: \`uv lock --upgrade\` on \`main\`." + echo + echo "CI's test suite installs \`uv.lock\` exactly as committed, so this PR is how new dependency releases reach it. Its CI runs the full suite against the versions below." + echo + echo "- CI passes: merge it through the merge queue. There is no need to update the branch first; each run rebuilds it from \`main\`." + echo "- CI fails: push a fix to this branch. Once anyone else commits here, the workflow stops refreshing this PR." + echo "- Not wanted this week: close it. The next run opens a new one." + echo + echo "axiom: n/a: dependency lock update" + echo + if [ "$count" -eq 0 ]; then + echo "No package versions changed, but other contents of the lock did; see the diff." + else + echo "
Changes from uv lock --upgrade ($count)" + echo + echo '```' + cat "$RUNNER_TEMP/changes.txt" + echo '```' + echo + echo "
" + fi + } > "$body" + if [ "$ACTION" = create ]; then + number=$(gh api "repos/$GH_REPO/pulls" -f base=main -f head="$BRANCH" \ + -f title="Update uv.lock to the newest dependency releases" \ + -F body=@"$body" --jq .number) + echo "Opened #$number" + gh api "repos/$GH_REPO/issues/$number/labels" -f 'labels[]=dependencies' > /dev/null \ + || echo "::warning::Could not add the dependencies label to #$number." + else + gh api -X PATCH "repos/$GH_REPO/pulls/$NUMBER" -F body=@"$body" > /dev/null + echo "Updated #$NUMBER" + fi + { + echo "### uv lock --upgrade" + echo + echo '```' + cat "$RUNNER_TEMP/changes.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Close the PR if main is already current + if: steps.changes.outputs.close == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + NUMBER: ${{ steps.existing.outputs.number }} + HEAD: ${{ steps.existing.outputs.head }} + run: | + gh api "repos/$GH_REPO/issues/$NUMBER/comments" \ + -f body="\`uv lock --upgrade\` on \`main\` changes nothing now, so this update is no longer needed." \ + > /dev/null || echo "::warning::Could not comment on #$NUMBER." + gh api -X PATCH "repos/$GH_REPO/pulls/$NUMBER" -f state=closed > /dev/null + # Delete the branch only if it is still the head that was checked: + # the lease makes the check and the delete one step, so a push that + # lands in between keeps the branch. + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" \ + push --force-with-lease="refs/heads/$BRANCH:$HEAD" origin ":refs/heads/$BRANCH" \ + || echo "::notice::Did not delete $BRANCH: it moved after it was checked, or the push failed." diff --git a/changelog.d/weekly-uv-lock.changed.md b/changelog.d/weekly-uv-lock.changed.md new file mode 100644 index 0000000000..f34d6df956 --- /dev/null +++ b/changelog.d/weekly-uv-lock.changed.md @@ -0,0 +1 @@ +- A weekly workflow now runs `uv lock --upgrade` and opens a pull request with the result, so CI's test suite, which installs from `uv.lock`, moves to new dependency releases.