api-tests.yml

name: GitHub API Tests

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:
  # Nightly. Rolling-flip detection needs at least five scored runs before it
  # will classify anything, and a push-only trigger can go a fortnight without
  # producing one.
  schedule:
    - cron: "0 2 * * *"

# Grant GITHUB_TOKEN write permissions so it can deploy to gh-pages
permissions:
  contents: write
  pages: write
  id-token: write

jobs:
  api-tests:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install -r requirements.txt

      # The default GITHUB_TOKEN is an "integration" token and cannot access /user or create repos.
      # We must use a Personal Access Token (PAT) stored as a GitHub Secret.
      - name: Run API tests
        env:
          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: >
          pytest tests/ -v --alluredir=allure-results
          --junitxml=junit-results.xml

      - name: Upload machine-readable results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: results-junit
          path: junit-results.xml
          retention-days: 30

      # Feed this run into TestPulse, which is what puts this suite on the
      # public dashboard at testpulse-eight.vercel.app.
      #
      # always(), because a failing run is the data point that matters most -
      # skipping ingest on red would mean the flake history only ever sees green.
      #
      # Not on pull requests: branches that may never merge would pollute the
      # history, and the flake numbers would end up describing code nobody runs.
      - name: Record the run in TestPulse
        if: always() && github.event_name != 'pull_request'
        continue-on-error: true
        uses: Mohanad49/testpulse@main
        with:
          path: junit-results.xml
          format: junit
          suite: github-api
          database-url: ${{ secrets.TESTPULSE_DATABASE_URL }}
          dashboard-url: https://testpulse-eight.vercel.app
          comment: "false"

      - name: Load Allure history
        uses: actions/checkout@v4
        if: always()
        continue-on-error: true
        with:
          ref: gh-pages
          path: gh-pages

      - name: Generate & deploy Allure report
        uses: simple-elf/allure-report-action@master
        if: always()
        with:
          allure_results: allure-results
          allure_history: allure-history

      - name: Fix Allure report permissions
        if: always()
        run: sudo chown -R $USER:$USER allure-history

      - uses: peaceiris/actions-gh-pages@v3
        if: always()
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: gh-pages
          publish_dir: allure-history

  # Repeat runs of one unchanged commit, on a schedule only.
  #
  # pytest cannot report retries the way Playwright can, so the same-commit
  # strategy has to get its evidence the other way: several runs of one SHA. If
  # two of these three disagree, the code was identical and the outcome was not.
  #
  # Separate from `api-tests` rather than a matrix on it, because that job
  # publishes Allure to gh-pages and parallel copies would race on the push.
  #
  # `needs` and `max-parallel: 1` are the schedule-side half of the secondary
  # rate-limit fix, and they are the reason this job stopped being free.
  #
  # Added on 2026-07-29, this job ran its three attempts in parallel, alongside
  # `api-tests`, all four sharing one token. Every scheduled run from the next
  # night onward failed: four concurrent copies of a suite that creates
  # repositories, issues and comments is a burst of content creation, and
  # GitHub's secondary limit exists precisely to stop that. The suite was not
  # measuring flakiness in the API - it was measuring the interference between
  # its own jobs, which is the one thing a flake detector must not do.
  #
  # Serialised, the four runs take longer and produce the same evidence. Three
  # runs of one SHA disagreeing is only meaningful if nothing about the runs
  # differed, and "how many siblings were hammering the same token" is a
  # difference.
  repeat:
    name: Repeat run ${{ matrix.attempt }}
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    needs: api-tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        attempt: [1, 2, 3]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - run: pip install -r requirements.txt

      # A failing run is the data being collected, not a broken pipeline.
      - name: Run tests
        continue-on-error: true
        env:
          # PAT_TOKEN, matching the main job. The automatic GITHUB_TOKEN cannot
          # create repositories or PATCH /user, which is what this suite does, so
          # using it here would have produced a 401 every night and looked
          # exactly like a flaky suite.
          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: pytest tests/ -v --alluredir=allure-results --junitxml=junit-results.xml

      - name: Upload results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: results-attempt-${{ matrix.attempt }}
          path: |
            junit-results.xml
            allure-results/
          retention-days: 30

      # Each attempt is stored as its own run. TestPulse's natural key includes
      # the start time, so three runs of one unchanged commit do not collapse
      # into one - which is the entire reason this job exists.
      - name: Record the run in TestPulse
        if: always()
        continue-on-error: true
        uses: Mohanad49/testpulse@main
        with:
          path: junit-results.xml
          format: junit
          suite: github-api
          database-url: ${{ secrets.TESTPULSE_DATABASE_URL }}
          dashboard-url: https://testpulse-eight.vercel.app
          comment: "false"