From d8f295844c99bfb0c3f12cfcd039230fd04142b8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 09:04:29 -0400 Subject: [PATCH 1/2] Run the Python tests on seven shards Nine complete runs on 31 October took 57 to 73 minutes a shard against the 75-minute limit, and the limit cancelled shard 1/6 of run 41191216150 with its tests at 99%. A seventh shard brings the duration-balanced slow tests to about 115 minutes of worker time per shard instead of 134. Job names, outcome artifacts, Result and check_test_shards.py follow the matrix size, so only the matrix, comments and docs change. Fixes #2770 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr_code_changes.yaml | 2 +- .github/workflows/test-suite.yaml | 8 +++++++- Makefile | 2 +- changelog.d/2770.changed.md | 1 + docs/book/validation/ubuntu-vs-macos.md | 2 +- docs/engineering/skills/testing.md | 4 ++-- 6 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 changelog.d/2770.changed.md diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 10b94a8a68..7ddb1b80c7 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -219,7 +219,7 @@ jobs: uses: ./.github/actions/test-environment - name: Test documentation builds run: make documentation - # The YAML tests, the Python tests on six runners, and the check that + # The YAML tests, the Python tests on seven runners, and the check that # every test ran once and passed, in the locked environment. As one job # they took 59 minutes on 10 October 2026 (run 38075384588), and runs were # cancelled at a 60-minute limit (issues #2099 and #2257). The release diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 0ea9cd70c6..f641d00d5e 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -174,6 +174,12 @@ jobs: # 15.5 GB (run 40395541241). With six, the listed slow tests come to # about 134 minutes of worker time per shard instead of 161, and each # worker runs about a sixth fewer tests. + # + # Seven, not six, for time. On 31 October nine complete runs took 57 to + # 73 minutes a shard, and the 75-minute limit cancelled shard 1/6 with + # its tests at 99% (run 41191216150, first attempt). With seven, the + # listed slow tests come to about 115 minutes of worker time per shard + # instead of 134. name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ${{ vars.CI_RUNNER_ARM != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_ARM || 'ubuntu-24.04-arm' }} @@ -189,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7] steps: - name: Checkout repo uses: actions/checkout@v6 diff --git a/Makefile b/Makefile index c6ff4df44f..982d241540 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test-yaml: # Every Python test, serially. To spread them over four cores add # PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist); to run what one CI -# shard runs, add "--shard K/6" (policyengine_uk/tests/conftest.py). CI runs +# shard runs, add "--shard K/7" (policyengine_uk/tests/conftest.py). CI runs # these tests through .github/workflows/test-suite.yaml, in the locked # environment (uv sync --frozen --extra dev), and also checks each test's # outcome against .github/test_outcome_allowlist.toml. diff --git a/changelog.d/2770.changed.md b/changelog.d/2770.changed.md new file mode 100644 index 0000000000..1b1546e3da --- /dev/null +++ b/changelog.d/2770.changed.md @@ -0,0 +1 @@ +Run the Python tests on seven shards, so a slow runner does not reach the 75-minute limit. diff --git a/docs/book/validation/ubuntu-vs-macos.md b/docs/book/validation/ubuntu-vs-macos.md index 6988b5f119..d0ef7d5928 100644 --- a/docs/book/validation/ubuntu-vs-macos.md +++ b/docs/book/validation/ubuntu-vs-macos.md @@ -103,7 +103,7 @@ That change verified, on main at `3c48247e`, that: ## Checking for a regression CI now runs the test suite (`.github/workflows/test-suite.yaml`: the -YAML tests in one job and the Python tests split over six shards) on +YAML tests in one job and the Python tests split over seven shards) on Linux arm64 (`ubuntu-24.04-arm`), and the install and smoke-import checks on x86 `ubuntu-latest`. There is no macOS job. If an OS difference is suspected again: diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index e9ac793c92..2659f448a9 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -86,10 +86,10 @@ the same suite, tree; a release tests the wheel it is about to publish, installed on its own. The suite runs the YAML policy tests in one job and splits the Python tests -over six runners. Each runner runs one shard: +over seven runners. Each runner runs one shard: ```bash -pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/6 -o xfail_strict=true +pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/7 -o xfail_strict=true ``` `--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the From bbfb7c1f36552e55d78ea273c3e177ace244aeff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 09:23:37 -0400 Subject: [PATCH 2/2] Say seven in the workflow header and narrow the timing claim Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-suite.yaml | 14 +++++++------- changelog.d/2770.changed.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index f641d00d5e..585ff4096c 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -4,10 +4,10 @@ # publish. All three run the same jobs in the same locked environment. # # Jobs: the YAML policy tests (which also list every Python test), the Python -# tests split over six runners, and Result, which fails unless every listed +# tests split over seven runners, and Result, which fails unless every listed # Python test ran in exactly one shard and passed, apart from the outcomes # accepted in .github/test_outcome_allowlist.toml. A caller that names its job -# Test gets the checks "Test / YAML", "Test / Python K/6" and "Test / Result"; +# Test gets the checks "Test / YAML", "Test / Python K/7" and "Test / Result"; # "Test / Result" is the one to require. # # Runners. These jobs, and the check jobs that pull requests, merge queue @@ -175,11 +175,11 @@ jobs: # about 134 minutes of worker time per shard instead of 161, and each # worker runs about a sixth fewer tests. # - # Seven, not six, for time. On 31 October nine complete runs took 57 to - # 73 minutes a shard, and the 75-minute limit cancelled shard 1/6 with - # its tests at 99% (run 41191216150, first attempt). With seven, the - # listed slow tests come to about 115 minutes of worker time per shard - # instead of 134. + # Seven, not six, for time. Nine successful test runs started on 31 + # October (EDT) took 57 to 73 minutes a shard, and the 75-minute limit + # cancelled shard 1/6 with its tests at 99% (run 41191216150, first + # attempt). With seven, the listed slow tests come to about 115 minutes + # of worker time per shard instead of 134. name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ${{ vars.CI_RUNNER_ARM != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_ARM || 'ubuntu-24.04-arm' }} diff --git a/changelog.d/2770.changed.md b/changelog.d/2770.changed.md index 1b1546e3da..60edba456f 100644 --- a/changelog.d/2770.changed.md +++ b/changelog.d/2770.changed.md @@ -1 +1 @@ -Run the Python tests on seven shards, so a slow runner does not reach the 75-minute limit. +Spread the Python tests across seven shards to reduce the work per job.