Rebuild the daily dependency-update branch from main instead of merging into it

.github/scripts/dependency-update-commit-and-push.sh

@@ -1,39 +1,57 @@
 #!/usr/bin/env bash
 
-# Commit generated dependency changes and update the reusable PR branch.
+# Commit the regenerated clients and upgraded locks on the update branch, which
+# the prepare step rebuilt from main, and replace the remote update branch.
+#
+# - When the update changes no file on main, nothing is committed or pushed.
+# - When the remote branch already holds the same files, the push is skipped.
+#   The commit message carries the date, so commits are compared by their files.
+# - Otherwise the push replaces the remote branch only while it still points at
+#   the commit the prepare step recorded (--force-with-lease), so a push made
+#   during the run is not lost silently. Without a remote branch, a plain push
+#   creates it.
+#
+# Environment:
+#   GITHUB_OUTPUT  receives changes_detected (true when the update changed files
+#                  on main, so an update PR should exist) and branch_pushed
+#   REMOTE_HEAD    the prepare step's remote_head output: the remote update
+#                  branch's commit, or empty when the branch does not exist
 
 set -euo pipefail
 
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}"
-: "${HAS_OPEN_PR:?HAS_OPEN_PR is required}"
 
-if [[ "${HAS_OPEN_PR}" != "true" && "${HAS_OPEN_PR}" != "false" ]]; then
-  echo "HAS_OPEN_PR must be either true or false" >&2
+branch="update-dependencies"
+remote_head="${REMOTE_HEAD:-}"
+if [[ -n "${remote_head}" && ! "${remote_head}" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "REMOTE_HEAD must be empty or a full commit hash: ${remote_head}" >&2
   exit 1
 fi
 
 changes_detected=false
+branch_pushed=false
 if [[ -z "$(git status --porcelain)" ]]; then
   echo "No changes detected"
 else
   echo "Changes detected"
+  changes_detected=true
   git add --all
   git commit -m "Update dependencies ($(date -u +%Y-%m-%d))"
-  changes_detected=true
-fi
 
-branch_pushed=false
-if [[ "${HAS_OPEN_PR}" == "true" ]]; then
-  commits_to_push="$(git rev-list --count origin/update-dependencies..HEAD)"
-  if [[ "${commits_to_push}" -gt 0 ]]; then
-    git push origin HEAD:update-dependencies
+  if [[ -n "${remote_head}" ]] && git diff --quiet "${remote_head}" HEAD --; then
+    echo "${branch} already holds this update; not pushing."
+  elif [[ -n "${remote_head}" ]]; then
+    git push \
+      --force-with-lease="refs/heads/${branch}:${remote_head}" \
+      origin "HEAD:refs/heads/${branch}"
+    branch_pushed=true
+  else
+    git push origin "HEAD:refs/heads/${branch}"
     branch_pushed=true
   fi
-elif [[ "${changes_detected}" == "true" ]]; then
-  # No open PR owns this branch, so replacing a stale remote branch cannot
-  # rewrite active review history.
-  git push --force-with-lease origin HEAD:update-dependencies
-  branch_pushed=true
 fi
 
-echo "branch_pushed=${branch_pushed}" >> "${GITHUB_OUTPUT}"
+{
+  echo "changes_detected=${changes_detected}"
+  echo "branch_pushed=${branch_pushed}"
+} >> "${GITHUB_OUTPUT}"

.github/scripts/dependency-update-create-pr.sh

@@ -1,18 +1,20 @@
 #!/usr/bin/env bash
 
-# After an update branch has been pushed, make sure an open dependency-update
-# PR exists and that the first line of its description names its tracking
-# issue, as rule sim.process.pr-tracking-issue requires of every PR.
+# After the dependency update changed files on main, make sure an open
+# dependency-update PR exists and that the first line of its description names
+# its tracking issue, as rule sim.process.pr-tracking-issue requires of every PR.
 #
 # The tracking issue is the oldest open issue titled "Update dependencies", or
 # a new one when none is open. "Fixes #N" closes it when the PR merges, so the
 # next update PR gets a new issue.
 #
-# - Without an open update PR, open one whose description starts with
-#   "Fixes #N".
-# - With one, make "Fixes #N" its first line: replace an earlier "Fixes #"
-#   line, or add the line above a description that has none (such as a PR
-#   opened before the rule).
+# - Without an open update PR, open one.
+# - With one, keep it, and replace its description when it differs from the
+#   current one. Each run rebuilds the branch from main, so the description is
+#   the workflow's, not a reviewer's; this also rewrites the description of a
+#   PR opened before the rule, or before the branch was rebuilt on every run.
+#
+# Either way the description is "Fixes #N", a blank line and the summary.
 #
 # Environment:
 #   GITHUB_REPOSITORY  owner/name of this repository (set by the runner)
@@ -25,7 +27,6 @@ set -euo pipefail
 
 ISSUE_TITLE="Update dependencies"
 ISSUE_BODY="Tracks the automated daily dependency updates from update-dependencies.yml. The update pull request names this issue on the first line of its description and closes it when it merges."
-PR_SUMMARY="Automated daily dependency updates. Subsequent runs append commits to this branch while the pull request remains open."
 
 # Set ISSUE_NUMBER to the oldest open "Update dependencies" issue, opening one
 # when none is open. The issues API lists pull requests too; they have a
@@ -61,35 +62,30 @@ find_or_open_tracking_issue() {
   ISSUE_NUMBER="${issue_number}"
 }
 
-# Print the description with "Fixes #<issue>" as its first line.
-description_naming_issue() {
-  local description="$1"
-  local issue_number="$2"
-  local first_line="${description%%$'\n'*}"
-  local fixes_line="Fixes #${issue_number}"
+# Print the update PR's description, naming the tracking issue first.
+update_pr_description() {
+  printf 'Fixes #%s\n\n' "${ISSUE_NUMBER}"
+  cat <<'EOF'
+Automated daily dependency update by `.github/workflows/update-dependencies.yml`. It runs `./scripts/generate-clients.sh` (the generated clients are not committed), then `make update`, which runs `uv lock --upgrade` in every project, so this pull request changes only `uv.lock` files. CI on this pull request tests the upgraded versions.
 
-  if [[ "${first_line}" =~ ^Fixes\ \#[0-9]+[[:space:]]*$ ]]; then
-    if [[ "${description}" == *$'\n'* ]]; then
-      printf '%s\n%s' "${fixes_line}" "${description#*$'\n'}"
-    else
-      printf '%s' "${fixes_line}"
-    fi
-  else
-    printf '%s\n\n%s' "${fixes_line}" "${description}"
-  fi
+Each run rebuilds `update-dependencies` from `main` and replaces it, so commits pushed to this branch by hand are lost on the next run.
+EOF
 }
 
+# --head matches a branch of the same name in a fork too; only this
+# repository's branch is the update branch.
 open_pr="$(
   gh pr list \
     --repo "${GITHUB_REPOSITORY}" \
     --base main \
     --head update-dependencies \
     --state open \
-    --json number \
-    --jq '.[0].number // empty'
+    --json number,isCrossRepository \
+    --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty'
 )"
 ISSUE_NUMBER=""
 find_or_open_tracking_issue
+description="$(update_pr_description)"
 
 if [[ -z "${open_pr}" ]]; then
   gh pr create \
@@ -98,24 +94,23 @@ if [[ -z "${open_pr}" ]]; then
     --body-file - \
     --base main \
     --head update-dependencies \
-    <<<"Fixes #${ISSUE_NUMBER}"$'\n\n'"${PR_SUMMARY}"
+    <<<"${description}"
   exit 0
 fi
 
-description="$(
+current_description="$(
   gh pr view "${open_pr}" \
     --repo "${GITHUB_REPOSITORY}" \
     --json body \
     --jq '.body'
 )"
-linked_description="$(description_naming_issue "${description}" "${ISSUE_NUMBER}")"
-if [[ "${linked_description}" == "${description}" ]]; then
-  echo "Pull request #${open_pr} already names tracking issue #${ISSUE_NUMBER}."
+if [[ "${current_description}" == "${description}" ]]; then
+  echo "Pull request #${open_pr} already has the current description."
   exit 0
 fi
 
 gh pr edit "${open_pr}" \
   --repo "${GITHUB_REPOSITORY}" \
   --body-file - \
-  <<<"${linked_description}"
-echo "Pull request #${open_pr} now names tracking issue #${ISSUE_NUMBER}."
+  <<<"${description}"
+echo "Pull request #${open_pr} now has the current description, naming tracking issue #${ISSUE_NUMBER}."

.github/scripts/dependency-update-prepare-branch.sh

@@ -1,36 +1,31 @@
 #!/usr/bin/env bash
 
-# Reuse the branch for an open dependency-update PR, or start a new branch
-# from current main when no update PR exists.
+# Rebuild the update branch from current main, and record the commit the
+# remote update branch points at, so the push can replace exactly that commit.
+#
+# Every run starts from main, even when an update PR is open. Merging main into
+# an older update branch merges generated uv.lock files line by line, which
+# mixes two resolutions into a lock that uv refuses to parse.
+#
+# Environment:
+#   GITHUB_OUTPUT  receives remote_head: the remote update branch's commit, or
+#                  empty when the branch does not exist. git reads the remote
+#                  with the credentials the checkout stored.
 
 set -euo pipefail
 
-: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}"
 
+branch="update-dependencies"
+
 git config user.name "policyengine-auto"
 git config user.email "policyengine-auto@users.noreply.github.com"
 
-open_pr="$(
-  gh pr list \
-    --repo "${GITHUB_REPOSITORY}" \
-    --base main \
-    --head update-dependencies \
-    --state open \
-    --json number \
-    --jq '.[0].number // empty'
-)"
-
-if [[ -n "${open_pr}" ]]; then
-  echo "has_open_pr=true" >> "${GITHUB_OUTPUT}"
-  git fetch origin update-dependencies
-  git checkout -B update-dependencies origin/update-dependencies
-
-  # Preserve the open PR's commits while incorporating current main. Prefer
-  # main's version of conflicts because later steps regenerate clients and
-  # recreate dependency lock changes.
-  git merge --no-edit -X theirs origin/main
-else
-  echo "has_open_pr=false" >> "${GITHUB_OUTPUT}"
-  git checkout -B update-dependencies origin/main
+remote_head="$(git ls-remote --heads origin "refs/heads/${branch}" | cut -f1)"
+if [[ -n "${remote_head}" ]]; then
+  # The push step compares the rebuilt branch with this commit.
+  git fetch --quiet origin "${remote_head}"
 fi
+
+git checkout -B "${branch}" origin/main
+echo "remote_head=${remote_head}" >> "${GITHUB_OUTPUT}"

.github/workflows/update-dependencies.yml

@@ -43,10 +43,10 @@ jobs:
         with:
           python-version: "3.13"
 
+      # Rebuilds update-dependencies from main on every run, and records the
+      # commit the remote branch points at for the push step's lease.
       - name: Prepare update branch
         id: branch
-        env:
-          GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: .github/scripts/dependency-update-prepare-branch.sh
 
       - name: Generate API clients
@@ -58,14 +58,15 @@ jobs:
       - name: Commit and push update
         id: push
         env:
-          HAS_OPEN_PR: ${{ steps.branch.outputs.has_open_pr }}
+          REMOTE_HEAD: ${{ steps.branch.outputs.remote_head }}
         run: .github/scripts/dependency-update-commit-and-push.sh
 
-      # Opens the update PR, or, when one is open, makes sure the first line
-      # of its description names the tracking issue (rule
-      # sim.process.pr-tracking-issue).
+      # Opens the update PR, or, when one is open, replaces its description,
+      # whose first line names the tracking issue (rule
+      # sim.process.pr-tracking-issue). It runs whenever the update changed
+      # files on main, including when the branch already held them.
       - name: Open or update the pull request
-        if: steps.push.outputs.branch_pushed == 'true'
+        if: steps.push.outputs.changes_detected == 'true'
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: .github/scripts/dependency-update-create-pr.sh

changelog_entry.yaml

@@ -27,6 +27,7 @@
       - Ran the full PR workflow and the Modal image smoke only on the lowest unmerged pull request in a stack, behind a new CI complete gate job that fails on the others
     fixed:
       - Stopped trusting the compromised eMudhra (emSign) intermediate CA certificates; every lockfile now pins certifi 2026.10.19, which drops emSign ECC Root CA - G3, every Modal image upgrades Debian's ca-certificates package right after its base layer, and the Modal images and the Cloud Run entrypoint image set GRPC_DEFAULT_SSL_ROOTS_FILE_PATH so grpcio verifies servers against the operating-system CA bundle instead of the CA list compiled into it
+      - Rebuilt the daily dependency update's branch from main on every run and replaced the remote branch with a lease, instead of merging main into the open update PR's branch, which mixed two uv.lock resolutions into locks uv could not parse; the push is skipped when the branch already holds the update, and the open update PR is reused with a rewritten description
       - Preserved bounded reform stop dates in UK and US simulations while retaining current-law baseline artifact reuse
       - Resolved PolicyEngine-managed regional datasets using bundled data versions instead of release-manifest commit hashes when loading from GCS
       - Added a Modal app-release bundle backfill for missing data artifact revisions

docs/engineering/skills/github-prs.md

@@ -98,8 +98,9 @@ To fix a failure:
 Automated pull requests follow the rule too. The policyengine package update
 (`update-policyengine-package.sh`) opens an issue for each version. The daily
 dependency update (`dependency-update-create-pr.sh`) reuses the open "Update
-dependencies" issue or opens one, and also puts its `Fixes` line first on an
-update pull request that is already open.
+dependencies" issue or opens one. It also reuses an update pull request that is
+already open, and replaces its description, `Fixes` line first, because each run
+rebuilds the update branch from `main`.
 
 ## PR Title