push.yaml

# Unified workflow for pushes to main.
#
# Phase 1 (normal push): Lint + Test → Docs + Versioning
#   Versioning commits with message "Update package version".
#   The commit pushes to main, re-triggering this workflow for Phase 2.
#
# Phase 2 (sentinel commit): Publish to PyPI + GitHub Release
#   Skips Lint + Test since the code is identical to Phase 1.
#   After Publish, NotifyConsumers waits for the release to be visible on
#   PyPI and sends a policyengine-release repository dispatch (version plus
#   the certified UK bundle identity and qualification digests) to consumer
#   repos so they open version-bump PRs, then a released-bundle replay
#   request to policyengine-scorecard. New consumers are dispatched in
#   separate, non-blocking steps.

name: Push to main

on:
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: write
  pages: write
  id-token: write

jobs:
  # ── Phase 1 gates (skip on sentinel commit) ─────────────────
  Lint:
    if: github.event.head_commit.message != 'Update package version'
    runs-on: ${{ vars.CI_RUNNER_X64 != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_X64 || 'ubuntu-latest' }}
    permissions:
      contents: read
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6
      - name: Install ruff
        run: pip install ruff
      - name: Run ruff format check
        run: ruff format --check .
      - name: Run ruff check
        run: ruff check .

  Test:
    if: github.event.head_commit.message != 'Update package version'
    runs-on: ${{ vars.CI_RUNNER_X64 != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_X64 || 'ubuntu-latest' }}
    permissions:
      contents: read
    # Successful runs on main 2026-10-14..16: 3.11 33-54 min, 3.12 32-56,
    # 3.13 36-60, 3.14 23-42; at 60 minutes every push run on 16 Oct was
    # cancelled before Versioning, so nothing was released (#640). Matches
    # the pull-request Test job (#631).
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.11', '3.12', '3.13', '3.14']
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v8.1.0
        with:
          version: "0.13.4"
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          allow-prereleases: true
      - name: Install package
        run: uv pip install -e .[dev] --system
      - name: Install policyengine
        run: uv pip install policyengine --system
      - name: Run tests with coverage
        run: make test
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}

  # ── Phase 1: Docs + Versioning (skip on sentinel commit) ──
  Docs:
    name: Deploy documentation
    runs-on: ubuntu-latest
    needs: [Lint, Test]
    if: github.event.head_commit.message != 'Update package version'
    concurrency:
      group: 'pages'
      cancel-in-progress: false
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    env:
      BASE_URL: /${{ github.event.repository.name }}
    steps:
      - uses: actions/checkout@v6
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - uses: actions/setup-node@v4
        with:
          node-version: 18.x
      - name: Set up Quarto
        uses: quarto-dev/quarto-actions/setup@v2
      - name: Build HTML Assets
        run: make docs
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: './docs/_site'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4

  Versioning:
    runs-on: ubuntu-latest
    needs: [Lint, Test]
    if: github.event.head_commit.message != 'Update package version'
    steps:
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          token: ${{ steps.app-token.outputs.token }}
          fetch-depth: 0
      - name: Fetch tags
        run: git fetch --tags --force
      - name: Install uv
        uses: astral-sh/setup-uv@v8.1.0
        with:
          version: "0.13.4"
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.13'
      - name: Install package for TRO verification and regeneration
        run: uv pip install -e . h5py --system
      - name: Check reviewed release inputs before versioning
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
        run: |
          if [[ -z "${HUGGING_FACE_TOKEN:-}" ]]; then
            echo "::error::Full private release verification requires HUGGING_FACE_TOKEN"
            exit 1
          fi
          python scripts/check_release_credentials.py
          python scripts/bundle.py check --published-spm --include-tros --strict-tros
          python scripts/release_lock.py
      - name: Build changelog
        run: pip install yaml-changelog towncrier && make changelog
      - name: Generate derived bundle metadata
        run: python scripts/bundle.py generate
      - name: Refresh only the release version in the registry lock
        run: python scripts/release_lock.py --refresh
      - name: Preview changelog update
        run: ".github/get-changelog-diff.sh"
      - name: Regenerate bundled TRACE TROs
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
        run: |
          python scripts/bundle.py generate --include-tros --strict-tros
          python scripts/bundle.py check --published-spm --include-tros --strict-tros
      - name: Update changelog, bundle metadata, and TROs
        uses: EndBug/add-and-commit@v9
        with:
          add: "."
          message: Update package version

  # ── Phase 2: Publish (only on sentinel commit) ────────────
  Publish:
    runs-on: ubuntu-latest
    if: github.event.head_commit.message == 'Update package version'
    # PyPI trusted publishing: PyPI accepts uploads only from this workflow in
    # the pypi environment, which is limited to main. id-token: write comes
    # from the workflow-level permissions. No stored PyPI token.
    environment: pypi
    env:
      GH_TOKEN: ${{ github.token }}
    outputs:
      version: ${{ steps.version.outputs.version }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v8.1.0
        with:
          version: "0.13.4"
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.13'
      - name: Install package
        run: uv pip install -e .[dev] --system
      - name: Install policyengine
        run: uv pip install policyengine --system
      - name: Capture published version
        id: version
        run: bash .github/capture-version.sh
      - name: Validate complete bundle and registry lock before publication
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
        run: |
          if [[ -z "${HUGGING_FACE_TOKEN:-}" ]]; then
            echo "::error::Full private release verification requires HUGGING_FACE_TOKEN"
            exit 1
          fi
          python scripts/check_release_credentials.py
          python scripts/bundle.py check --published-spm --include-tros --strict-tros
          python scripts/release_lock.py
      - name: Publish a git tag
        run: ".github/publish-git-tag.sh"
      - name: Build package
        run: python -m build
      - name: Export bundle release assets
        run: python scripts/export_bundle_release_assets.py --dist-dir release-assets
      - name: Verify bundle package metadata
        env:
          POLICYENGINE_SKIP_COUNTRY_IMPORTS: "1"
        run: |
          VERSION=$(python .github/fetch_version.py)
          policyengine bundle verify --country us --country uk --packages-only --json \
            > "release-assets/policyengine-bundle-$VERSION.verification.json"
      - name: Publish a Python distribution to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          skip-existing: true
          packages-dir: dist
      - name: Create GitHub Release
        run: |
          VERSION=$(python .github/fetch_version.py)
          gh release create "$VERSION" \
            --title "v$VERSION" \
            --notes "See [CHANGELOG.md](https://github.com/PolicyEngine/policyengine.py/blob/main/CHANGELOG.md) for details." \
            --latest \
            "release-assets/policyengine-bundle-$VERSION.json" \
            "release-assets/policyengine-bundle-$VERSION.constraints.txt" \
            "release-assets/policyengine-bundle-$VERSION.citation.txt" \
            "release-assets/policyengine-bundle-$VERSION.verification.json"

  # ── Phase 2: Open consumer update PRs (after Publish) ─────
  NotifyConsumers:
    name: Open consumer update PRs
    runs-on: ubuntu-latest
    needs: [Publish]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.13'
      - name: Wait for release to be visible on PyPI
        env:
          VERSION: ${{ needs.Publish.outputs.version }}
        run: bash .github/wait-for-pypi.sh
      - name: Validate published SPM bundle before consumer updates
        run: python -S scripts/bundle.py check --published-spm
      - name: Install uv
        uses: astral-sh/setup-uv@v8.1.0
        with:
          version: "0.13.4"
      - name: Install policyengine without country models
        run: uv pip install -e . --system
      # The payload carries the certified UK bundle identity and, for an
      # automatically qualified pair, the qualification receipt digest.
      # client_payload.version stays first, as existing updaters read only it.
      - name: Build the release dispatch payload
        env:
          VERSION: ${{ needs.Publish.outputs.version }}
        run: python scripts/release_consumers.py payload --version "$VERSION" > "$RUNNER_TEMP/release-payload.json"
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          owner: PolicyEngine
          repositories: policyengine-api,policyengine-sim-api
      - name: Dispatch policyengine-release to consumer repos
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          VERSION: ${{ needs.Publish.outputs.version }}
          PAYLOAD_FILE: ${{ runner.temp }}/release-payload.json
        run: bash .github/dispatch-policyengine-release.sh
      # New consumers are dispatched separately, so a missing App
      # installation on one of them can never block the consumers above.
      - name: Generate GitHub App token for the Budget dashboard
        id: dashboard-token
        continue-on-error: true
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          owner: PolicyEngine
          repositories: uk-autumn-budget-dashboard-2026
      - name: Dispatch policyengine-release to the Budget dashboard
        if: steps.dashboard-token.outcome == 'success'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ steps.dashboard-token.outputs.token }}
          VERSION: ${{ needs.Publish.outputs.version }}
          PAYLOAD_FILE: ${{ runner.temp }}/release-payload.json
          CONSUMER_REPOS: uk-autumn-budget-dashboard-2026
        run: bash .github/dispatch-policyengine-release.sh
      - name: Install the released bundle for the replay request
        continue-on-error: true
        id: released
        env:
          VERSION: ${{ needs.Publish.outputs.version }}
        run: uv pip install --system "policyengine[uk]==$VERSION"
      - name: Build the scorecard's released-bundle replay request
        if: steps.released.outcome == 'success'
        id: scorecard-request
        continue-on-error: true
        env:
          VERSION: ${{ needs.Publish.outputs.version }}
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
        run: python scripts/release_consumers.py scorecard-request --version "$VERSION" > "$RUNNER_TEMP/scorecard-request.json"
      - name: Warn that no released-bundle replay was requested
        if: steps.released.outcome == 'failure' || steps.scorecard-request.outcome == 'failure'
        env:
          VERSION: ${{ needs.Publish.outputs.version }}
        run: echo "::warning title=No released-bundle replay::The scorecard's replay request for policyengine $VERSION could not be built; see the steps above."
      - name: Generate GitHub App token for the scorecard
        if: steps.scorecard-request.outcome == 'success'
        id: scorecard-token
        continue-on-error: true
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          owner: PolicyEngine
          repositories: policyengine-scorecard
      - name: Send the scorecard a released-bundle replay request
        if: steps.scorecard-token.outcome == 'success'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ steps.scorecard-token.outputs.token }}
          VERSION: ${{ needs.Publish.outputs.version }}
          PAYLOAD_FILE: ${{ runner.temp }}/scorecard-request.json
          CONSUMER_REPOS: policyengine-scorecard
        run: bash .github/dispatch-policyengine-release.sh