diff --git a/.github/workflows/pr_code_changes.yaml b/.github/workflows/pr_code_changes.yaml index 829222d447..10b94a8a68 100644 --- a/.github/workflows/pr_code_changes.yaml +++ b/.github/workflows/pr_code_changes.yaml @@ -219,7 +219,7 @@ jobs: uses: ./.github/actions/test-environment - name: Test documentation builds run: make documentation - # The YAML tests, the Python tests on five runners, and the check that + # The YAML tests, the Python tests on six runners, and the check that # every test ran once and passed, in the locked environment. As one job # they took 59 minutes on 10 October 2026 (run 38075384588), and runs were # cancelled at a 60-minute limit (issues #2099 and #2257). The release diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index b90e2e2504..4ca22379be 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -4,10 +4,10 @@ # publish. All three run the same jobs in the same locked environment. # # Jobs: the YAML policy tests (which also list every Python test), the Python -# tests split over five runners, and Result, which fails unless every listed +# tests split over six runners, and Result, which fails unless every listed # Python test ran in exactly one shard and passed, apart from the outcomes # accepted in .github/test_outcome_allowlist.toml. A caller that names its job -# Test gets the checks "Test / YAML", "Test / Python K/5" and "Test / Result"; +# Test gets the checks "Test / YAML", "Test / Python K/6" and "Test / Result"; # "Test / Result" is the one to require. # # Runners. These jobs, and the check jobs that pull requests, merge queue @@ -153,6 +153,16 @@ jobs: # request adding about a fifth more parameters (#1855) hit the timeout # on shard 1/4 twice, and its other shards took 48 to 59 minutes (run # 39568525017). + # + # Six, not five, for both. On 26 October a pull request's shards took + # 63 to 75 minutes, and the 75-minute limit cancelled shard 4/5 (run + # 40308966462, first attempt); a merge queue run that evening took 66 + # to 69 (run 40406434741). The same evening one pull request lost + # shards 2/5, 3/5 and 5/5 to memory at 15.6 to 15.9 GB (run + # 40393666784), and the merge queue run for #2105 lost shard 2/5 at + # 15.5 GB (run 40395541241). With six, the listed slow tests come to + # about 134 minutes of worker time per shard instead of 161, and each + # worker runs about a sixth fewer tests. name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ${{ vars.CI_RUNNER_ARM != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_ARM || 'ubuntu-24.04-arm' }} @@ -168,7 +178,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5] + shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout repo uses: actions/checkout@v6 diff --git a/Makefile b/Makefile index 3e42b15a7d..c6ff4df44f 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test-yaml: # Every Python test, serially. To spread them over four cores add # PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist); to run what one CI -# shard runs, add "--shard K/5" (policyengine_uk/tests/conftest.py). CI runs +# shard runs, add "--shard K/6" (policyengine_uk/tests/conftest.py). CI runs # these tests through .github/workflows/test-suite.yaml, in the locked # environment (uv sync --frozen --extra dev), and also checks each test's # outcome against .github/test_outcome_allowlist.toml. diff --git a/changelog.d/ci-six-test-shards.changed.md b/changelog.d/ci-six-test-shards.changed.md new file mode 100644 index 0000000000..1a883258f9 --- /dev/null +++ b/changelog.d/ci-six-test-shards.changed.md @@ -0,0 +1 @@ +Split the Python tests in CI over six runners instead of five. diff --git a/docs/book/validation/ubuntu-vs-macos.md b/docs/book/validation/ubuntu-vs-macos.md index d3a8b4e2cd..6988b5f119 100644 --- a/docs/book/validation/ubuntu-vs-macos.md +++ b/docs/book/validation/ubuntu-vs-macos.md @@ -103,7 +103,7 @@ That change verified, on main at `3c48247e`, that: ## Checking for a regression CI now runs the test suite (`.github/workflows/test-suite.yaml`: the -YAML tests in one job and the Python tests split over five shards) on +YAML tests in one job and the Python tests split over six shards) on Linux arm64 (`ubuntu-24.04-arm`), and the install and smoke-import checks on x86 `ubuntu-latest`. There is no macOS job. If an OS difference is suspected again: diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 9e4df81565..e9ac793c92 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -86,10 +86,10 @@ the same suite, tree; a release tests the wheel it is about to publish, installed on its own. The suite runs the YAML policy tests in one job and splits the Python tests -over five runners. Each runner runs one shard: +over six runners. Each runner runs one shard: ```bash -pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/5 -o xfail_strict=true +pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/6 -o xfail_strict=true ``` `--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the