From 94a43ce01ed17357b179fc657e1966fe0e853a98 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 08:17:31 -0500 Subject: [PATCH 1/3] Run the certified aggregate tests in one CI shard The module fixture in test_aggregate_impact_certified.py builds its own simulations, and each shard that holds one of the file's tests pays for them. The duration table deals the file's long entries apart, so main spreads its nine tests over five shards. Deal and hash the file as one unit, so one shard pays. Co-Authored-By: Claude Opus 5.5 --- .../certified-aggregate-one-shard.changed.md | 1 + .../tests/code_health/test_ci_test_shards.py | 66 +++++++++++++++++++ policyengine_uk/tests/conftest.py | 53 +++++++++++---- 3 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 changelog.d/certified-aggregate-one-shard.changed.md diff --git a/changelog.d/certified-aggregate-one-shard.changed.md b/changelog.d/certified-aggregate-one-shard.changed.md new file mode 100644 index 0000000000..6e523dee82 --- /dev/null +++ b/changelog.d/certified-aggregate-one-shard.changed.md @@ -0,0 +1 @@ +Run every test in test_aggregate_impact_certified.py in one CI shard, so the other six shards no longer build its simulations. diff --git a/policyengine_uk/tests/code_health/test_ci_test_shards.py b/policyengine_uk/tests/code_health/test_ci_test_shards.py index 0193a5b67f..cae882cdef 100644 --- a/policyengine_uk/tests/code_health/test_ci_test_shards.py +++ b/policyengine_uk/tests/code_health/test_ci_test_shards.py @@ -40,6 +40,9 @@ the least loaded shard, whatever order the table lists them in, so no shard's listed load exceeds another's by more than the longest test; the table parses and each entry names a test function that exists. +13. The tests of a file in ``SHARED_FIXTURE_FILES`` all run in one shard, listed + or not, at any number of shards. The deal gives the file the weight of its + longest test, and the bound in 12 holds with that weight. """ import ast @@ -245,6 +248,69 @@ def test_a_missing_table_leaves_every_test_to_the_hash(conftest, tmp_path): assert conftest.read_shard_durations(tmp_path / "absent.toml") == {} +# Invariant 13: a file with a costly module fixture runs in one shard. + +CERTIFIED = "policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py" + +_certified_ids = st.builds( + lambda name, params: f"{CERTIFIED}::{name}{params}", _identifier, _params +) + + +def test_the_certified_aggregate_tests_share_one_shard(conftest): + source = (REPO_ROOT / CERTIFIED).read_text(encoding="utf-8") + names = [ + node.name + for node in ast.parse(source).body + if isinstance(node, ast.FunctionDef) and node.name.startswith("test_") + ] + assert len(names) > 1 + for shards in (1, 3, 5, 7): + placed = {conftest.shard_of(f"{CERTIFIED}::{name}", shards) for name in names} + assert len(placed) == 1, (shards, placed) + + +@PROPERTY_SETTINGS +@given(nodeid=_certified_ids, shards=st.integers(1, 8)) +def test_any_test_in_a_shared_fixture_file_joins_the_others(conftest, nodeid, shards): + assert conftest.shard_of(nodeid, shards) == conftest.shard_of( + f"{CERTIFIED}::test_anything", shards + ) + + +def test_a_shared_fixture_file_hashes_as_one_without_a_table(conftest, monkeypatch): + monkeypatch.setattr(conftest, "_BALANCED", {7: {}}) + assert len({conftest.shard_of(f"{CERTIFIED}::test_{k}", 7) for k in range(20)}) == 1 + + +@PROPERTY_SETTINGS +@given( + certified=st.dictionaries( + _certified_ids, st.floats(0, 3_000, allow_nan=False), max_size=8 + ), + others=_durations, + shards=st.integers(1, 8), + data=st.data(), +) +def test_the_deal_moves_a_shared_fixture_file_as_one( + conftest, certified, others, shards, data +): + durations = {**others, **certified} + placed = conftest.balanced_shards(durations, shards) + assert placed.keys() == durations.keys() + assert all(1 <= shard <= shards for shard in placed.values()) + assert len({placed[nodeid] for nodeid in certified}) <= 1 + load = [0.0] * shards + for nodeid, seconds in others.items(): + load[placed[nodeid] - 1] += seconds + if certified: + load[placed[next(iter(certified))] - 1] += max(certified.values()) + assert max(load) - min(load) <= max(durations.values(), default=0) + 1e-6 + # The same placement whatever order the table lists the tests in. + reordered = dict(data.draw(st.permutations(list(durations.items())))) + assert conftest.balanced_shards(reordered, shards) == placed + + # Invariant 4: the collection listing. diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d945afd3e5..9476822850 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -206,6 +206,17 @@ def cloned_uk_tax_benefit_system(): # slowest on CI, are instead dealt out by duration, longest first, each to the # shard with the least so far. A test missing from the table, or the table # missing, falls back to the hash. +# +# The tests of a file in SHARED_FIXTURE_FILES all run in one shard, because each +# shard that holds one of them builds that file's module fixture. In merge queue +# run 42212587066 the fixture in test_aggregate_impact_certified.py held one +# worker for 8 to 9 minutes at the start of five shards, which peaked at 14.2 to +# 15.3 GB on 16 GB runners, against 10.9 and 12.6 GB for the two without it. Runs +# 42218280596 and 42218280615 had those tests on all seven shards and lost eight +# of their 14 shard jobs at 15.4 to 15.7 GB. +SHARED_FIXTURE_FILES = frozenset( + {"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py"} +) SHARD_DURATIONS = ( Path(importlib.util.find_spec("policyengine_uk").origin).parent / "tests" @@ -225,19 +236,36 @@ def read_shard_durations(path: Path = SHARD_DURATIONS) -> dict[str, float]: } -def balanced_shards(durations: Mapping[str, float], shards: int) -> dict[str, int]: - """Deal tests to shards, longest first, each to the least loaded shard. +def placement_unit(nodeid: str) -> str: + """What the deal and the hash place: a shared-fixture file, or else the test.""" + path = nodeid.split("::", 1)[0] + return path if path in SHARED_FIXTURE_FILES else nodeid + - Ties go to the lower node id and the lower shard, so every process gives - the same answer. +def deal_units(durations: Mapping[str, float], shards: int) -> dict[str, int]: + """Deal placement units to shards, longest first, each to the least loaded. + + A shared-fixture file weighs as much as its longest test, since the file's + long entries each measured the same fixture. Ties go to the lower name and + the lower shard, so every process gives the same answer. """ + weights: dict[str, float] = {} + for nodeid, seconds in durations.items(): + unit = placement_unit(nodeid) + weights[unit] = max(seconds, weights.get(unit, seconds)) load = [0.0] * shards - placed = {} - for nodeid, seconds in sorted(durations.items(), key=lambda kv: (-kv[1], kv[0])): + dealt = {} + for unit, seconds in sorted(weights.items(), key=lambda kv: (-kv[1], kv[0])): shard = min(range(shards), key=lambda k: (load[k], k)) load[shard] += seconds - placed[nodeid] = shard + 1 - return placed + dealt[unit] = shard + 1 + return dealt + + +def balanced_shards(durations: Mapping[str, float], shards: int) -> dict[str, int]: + """The shard the deal gives each test in ``durations``.""" + dealt = deal_units(durations, shards) + return {nodeid: dealt[placement_unit(nodeid)] for nodeid in durations} _BALANCED: dict[int, dict[str, int]] = {} @@ -250,10 +278,11 @@ def shard_of(nodeid: str, shards: int) -> int: runner and every pytest-xdist worker must give the same answer. """ if shards not in _BALANCED: - _BALANCED[shards] = balanced_shards(read_shard_durations(), shards) - if nodeid in _BALANCED[shards]: - return _BALANCED[shards][nodeid] - digest = hashlib.sha256(nodeid.encode("utf-8")).digest() + _BALANCED[shards] = deal_units(read_shard_durations(), shards) + unit = placement_unit(nodeid) + if unit in _BALANCED[shards]: + return _BALANCED[shards][unit] + digest = hashlib.sha256(unit.encode("utf-8")).digest() return int.from_bytes(digest[:8], "big") % shards + 1 From 595fd3fdf55ba252ed9c6c995e27174e56df52c3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 09:37:02 -0500 Subject: [PATCH 2/3] Run the tests that need the dataset in their own CI shard The certified aggregate tests on one shard of seven still lost that shard's runner at 15.6 GB on a large merge, and the latest-data smoke tests lost another. The tests that carry the microsimulation marker now run in an eighth shard with one worker, and the other seven run none of them. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-suite.yaml | 31 +++-- Makefile | 2 +- .../certified-aggregate-one-shard.changed.md | 1 - changelog.d/data-tests-own-shard.changed.md | 1 + docs/engineering/skills/testing.md | 12 +- .../tests/code_health/test_ci_test_shards.py | 126 ++++++++++-------- policyengine_uk/tests/conftest.py | 83 +++++------- policyengine_uk/tests/shard_durations.toml | 25 +--- 8 files changed, 139 insertions(+), 142 deletions(-) delete mode 100644 changelog.d/certified-aggregate-one-shard.changed.md create mode 100644 changelog.d/data-tests-own-shard.changed.md diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 2d780c7c27..840c36e072 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -4,10 +4,10 @@ # publish. All three run the same jobs in the same locked environment. # # Jobs: the YAML policy tests (which also list every Python test), the Python -# tests split over seven runners, and Result, which fails unless every listed +# tests split over eight runners, and Result, which fails unless every listed # Python test ran in exactly one shard and passed, apart from the outcomes # accepted in .github/test_outcome_allowlist.toml. A caller that names its job -# Test gets the checks "Test / YAML", "Test / Python K/7" and "Test / Result"; +# Test gets the checks "Test / YAML", "Test / Python K/8" and "Test / Result"; # "Test / Result" is the one to require. # # Runners. These jobs, and the check jobs that pull requests, merge queue @@ -143,7 +143,8 @@ jobs: retention-days: 7 Python: - # Shard K of N runs the tests whose node id hashes to K (--shard in + # The last shard runs the tests that need the dataset. Each other shard K + # of N runs the remaining tests whose node id hashes to K (--shard in # policyengine_uk/tests/conftest.py), so each test is in exactly one # shard. N is the number of jobs this matrix makes: to add a shard, add # a number to the list. @@ -182,6 +183,18 @@ jobs: # cancelled shard 1/6 with its tests at 99% (run 41191216150, first # attempt). With seven, the listed slow tests come to about 115 minutes # of worker time per shard instead of 134. + # + # Eight, not seven, for memory: the eighth runs the tests that need the + # dataset, with one worker, so only one simulation of the dataset runs at + # a time, and the other seven run none of them. In merge queue run + # 42212587066 the five shards that built the fixture in + # test_aggregate_impact_certified.py peaked at 14.2 to 15.3 GB, and the two + # without it at 10.9 and 12.6 GB. On a merge of 28 pull requests with those + # tests on all seven shards, runs 42218280596 and 42218280615 lost eight of + # their 14 shard jobs within minutes at 15.4 to 15.7 GB, and shard 6/7 of + # the first lost its runner at 15.6 GB 51 minutes in, with + # test_latest_data_smoke.py running. With the certified tests on one shard + # of seven, run 42231098419 lost that shard at 15.6 GB. name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ${{ vars.CI_RUNNER_ARM != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_ARM || 'ubuntu-24.04-arm' }} @@ -203,7 +216,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout repo uses: actions/checkout@v6 @@ -230,6 +243,7 @@ jobs: POLICYENGINE_UK_REQUIRE_TEST_DATA: ${{ inputs.require-data && '1' || '' }} SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} OUTCOMES: outcomes-${{ matrix.shard }}.jsonl + WORKERS: ${{ matrix.shard == strategy.job-total && 1 || 4 }} # Measure coverage with sys.monitoring instead of the C trace # function: the same lines are reported, at less cost per line. COVERAGE_CORE: sysmon @@ -250,14 +264,15 @@ jobs: # .github/pytest_outcome_log.py records each test's outcome for # Result. xfail_strict makes an xfail marker without strict= fail # when its test passes, so a fixed bug cannot hide behind it. - # Four pytest-xdist workers, one per core. Their memory grows with - # each test they run (#2330): with three shards they peaked at - # 15.6 GB of the runner's 16 GB (run 38156967136). + # Four pytest-xdist workers, one per core, and one on the last shard. + # Their memory grows with each test they run (#2330): with three + # shards they peaked at 15.6 GB of the runner's 16 GB (run + # 38156967136). PYTHONPATH="$GITHUB_WORKSPACE/.github" pytest "$TESTS" \ --rootdir "$ROOTDIR" -c pyproject.toml -p no:cacheprovider \ -p pytest_outcome_log --outcome-log="$OUTCOMES" \ -o xfail_strict=true --maxfail=0 -v \ - -n 4 --dist worksteal --shard "$SHARD" "${coverage[@]}" + -n "$WORKERS" --dist worksteal --shard "$SHARD" "${coverage[@]}" - name: Upload the test outcomes if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 diff --git a/Makefile b/Makefile index 982d241540..6e8b2e0d24 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test-yaml: # Every Python test, serially. To spread them over four cores add # PYTEST_ARGS="-n 4 --dist worksteal" (pytest-xdist); to run what one CI -# shard runs, add "--shard K/7" (policyengine_uk/tests/conftest.py). CI runs +# shard runs, add "--shard K/8" (policyengine_uk/tests/conftest.py). CI runs # these tests through .github/workflows/test-suite.yaml, in the locked # environment (uv sync --frozen --extra dev), and also checks each test's # outcome against .github/test_outcome_allowlist.toml. diff --git a/changelog.d/certified-aggregate-one-shard.changed.md b/changelog.d/certified-aggregate-one-shard.changed.md deleted file mode 100644 index 6e523dee82..0000000000 --- a/changelog.d/certified-aggregate-one-shard.changed.md +++ /dev/null @@ -1 +0,0 @@ -Run every test in test_aggregate_impact_certified.py in one CI shard, so the other six shards no longer build its simulations. diff --git a/changelog.d/data-tests-own-shard.changed.md b/changelog.d/data-tests-own-shard.changed.md new file mode 100644 index 0000000000..dce89ae3db --- /dev/null +++ b/changelog.d/data-tests-own-shard.changed.md @@ -0,0 +1 @@ +Run the tests that need the dataset in their own CI shard, with one worker, so the other seven shards no longer build simulations of the whole dataset. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 2659f448a9..dd22873a20 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -86,15 +86,17 @@ the same suite, tree; a release tests the wheel it is about to publish, installed on its own. The suite runs the YAML policy tests in one job and splits the Python tests -over seven runners. Each runner runs one shard: +over eight runners. Each runner runs one shard: ```bash -pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/7 -o xfail_strict=true +pytest policyengine_uk/tests -n 4 --dist worksteal --shard K/8 -o xfail_strict=true ``` -`--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. It keeps the -tests whose node id hashes to shard K of N, so every test is in exactly one -shard, including a test added later. Each shard records every test's outcome +`--shard K/N` is defined in `policyengine_uk/tests/conftest.py`. Shard N of N +keeps the tests with the `microsimulation` marker, which need the dataset, and +CI runs it with one worker. Every other shard keeps the remaining tests whose +node id hashes to it, so every test is in exactly one shard, including a test +added later. Each shard records every test's outcome with `.github/pytest_outcome_log.py`. The final job, `Test / Result`, runs `.github/check_test_shards.py`. It fails unless every collected test ran in exactly one shard and passed. The exceptions are outcomes accepted in diff --git a/policyengine_uk/tests/code_health/test_ci_test_shards.py b/policyengine_uk/tests/code_health/test_ci_test_shards.py index cae882cdef..459a9e9da0 100644 --- a/policyengine_uk/tests/code_health/test_ci_test_shards.py +++ b/policyengine_uk/tests/code_health/test_ci_test_shards.py @@ -40,9 +40,9 @@ the least loaded shard, whatever order the table lists them in, so no shard's listed load exceeds another's by more than the longest test; the table parses and each entry names a test function that exists. -13. The tests of a file in ``SHARED_FIXTURE_FILES`` all run in one shard, listed - or not, at any number of shards. The deal gives the file the weight of its - longest test, and the bound in 12 holds with that weight. +13. With two or more shards, the last runs exactly the tests that need the + dataset, and ``shard_of`` over one shard fewer places the others. The + workflow numbers its shards from 1 to N and gives the last one one worker. """ import ast @@ -248,67 +248,54 @@ def test_a_missing_table_leaves_every_test_to_the_hash(conftest, tmp_path): assert conftest.read_shard_durations(tmp_path / "absent.toml") == {} -# Invariant 13: a file with a costly module fixture runs in one shard. +# Invariant 13: the last shard runs the tests that need the dataset. -CERTIFIED = "policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py" -_certified_ids = st.builds( - lambda name, params: f"{CERTIFIED}::{name}{params}", _identifier, _params -) +class _CollectedTest: + """What item_shard reads from a pytest item.""" + def __init__(self, nodeid, needs_data): + self.nodeid = nodeid + self.needs_data = needs_data -def test_the_certified_aggregate_tests_share_one_shard(conftest): - source = (REPO_ROOT / CERTIFIED).read_text(encoding="utf-8") - names = [ - node.name - for node in ast.parse(source).body - if isinstance(node, ast.FunctionDef) and node.name.startswith("test_") - ] - assert len(names) > 1 - for shards in (1, 3, 5, 7): - placed = {conftest.shard_of(f"{CERTIFIED}::{name}", shards) for name in names} - assert len(placed) == 1, (shards, placed) + def get_closest_marker(self, name): + return object() if self.needs_data and name == "microsimulation" else None @PROPERTY_SETTINGS -@given(nodeid=_certified_ids, shards=st.integers(1, 8)) -def test_any_test_in_a_shared_fixture_file_joins_the_others(conftest, nodeid, shards): - assert conftest.shard_of(nodeid, shards) == conftest.shard_of( - f"{CERTIFIED}::test_anything", shards - ) +@given( + collection=st.dictionaries(nodeids, st.booleans(), max_size=40), + shards=st.integers(2, 8), +) +def test_the_last_shard_runs_exactly_the_tests_that_need_the_dataset( + conftest, collection, shards +): + for nodeid, needs_data in collection.items(): + shard = conftest.item_shard(_CollectedTest(nodeid, needs_data), shards) + if needs_data: + assert shard == shards + else: + assert shard == conftest.shard_of(nodeid, shards - 1) < shards -def test_a_shared_fixture_file_hashes_as_one_without_a_table(conftest, monkeypatch): - monkeypatch.setattr(conftest, "_BALANCED", {7: {}}) - assert len({conftest.shard_of(f"{CERTIFIED}::test_{k}", 7) for k in range(20)}) == 1 +@PROPERTY_SETTINGS +@given(nodeid=nodeids, needs_data=st.booleans()) +def test_one_shard_runs_every_test(conftest, nodeid, needs_data): + assert conftest.item_shard(_CollectedTest(nodeid, needs_data), 1) == 1 -@PROPERTY_SETTINGS -@given( - certified=st.dictionaries( - _certified_ids, st.floats(0, 3_000, allow_nan=False), max_size=8 - ), - others=_durations, - shards=st.integers(1, 8), - data=st.data(), -) -def test_the_deal_moves_a_shared_fixture_file_as_one( - conftest, certified, others, shards, data -): - durations = {**others, **certified} - placed = conftest.balanced_shards(durations, shards) - assert placed.keys() == durations.keys() - assert all(1 <= shard <= shards for shard in placed.values()) - assert len({placed[nodeid] for nodeid in certified}) <= 1 - load = [0.0] * shards - for nodeid, seconds in others.items(): - load[placed[nodeid] - 1] += seconds - if certified: - load[placed[next(iter(certified))] - 1] += max(certified.values()) - assert max(load) - min(load) <= max(durations.values(), default=0) + 1e-6 - # The same placement whatever order the table lists the tests in. - reordered = dict(data.draw(st.permutations(list(durations.items())))) - assert conftest.balanced_shards(reordered, shards) == placed +def test_the_workflow_gives_the_last_shard_one_worker(): + import yaml + + workflow = GITHUB_DIR / "workflows" / "test-suite.yaml" + job = yaml.safe_load(workflow.read_text(encoding="utf-8"))["jobs"]["Python"] + shards = job["strategy"]["matrix"]["shard"] + assert shards == list(range(1, len(shards) + 1)) + [step] = [step for step in job["steps"] if step.get("name") == "Run the tests"] + assert step["env"]["WORKERS"] == ( + "${{ matrix.shard == strategy.job-total && 1 || 4 }}" + ) + assert '-n "$WORKERS"' in step["run"] # Invariant 4: the collection listing. @@ -611,6 +598,15 @@ def test_never_runs(): raise AssertionError """ +SAMPLE_DATA_TEST = """ +import pytest + + +@pytest.mark.microsimulation +def test_needs_the_dataset(): + pass +""" + FAILING_SUITE = """ import pytest @@ -674,6 +670,11 @@ def test_passes(): nodeid = "suite/test_skipped_module.py" outcome = "collect-skipped" why = "sample" + +[[allow]] +nodeid = "suite/test_latest_data_smoke.py::test_needs_the_dataset" +outcome = "skipped" +why = "sample" """ @@ -794,19 +795,32 @@ def test_a_skipped_subtest_is_not_hidden_by_its_tests_pass(tmp_path): @pytest.mark.parametrize("xdist", [(), ("-n", "2", "--dist", "worksteal")]) def test_real_pytest_shards_pass_the_check_only_when_allowed(tmp_path, xdist): - _suite(tmp_path, test_sample=SAMPLE_SUITE, test_skipped_module=SKIPPED_MODULE) + # conftest.py verifies no dataset for a test in this file (UNPINNED_BY_DESIGN), + # so the run downloads nothing. + _suite( + tmp_path, + test_sample=SAMPLE_SUITE, + test_skipped_module=SKIPPED_MODULE, + test_latest_data_smoke=SAMPLE_DATA_TEST, + ) listing = _pytest(tmp_path, "suite", "--collect-only", "-q") assert listing.returncode == 0, listing.stdout + listing.stderr collected = tmp_path / "collected.txt" collected.write_text(listing.stdout, encoding="utf-8") - assert len(checker.read_collected(listing.stdout)) == 43 + assert len(checker.read_collected(listing.stdout)) == 44 logs = _run_shards(tmp_path, 3, *xdist) sizes = [ sum(not r.is_collector for r in checker.read_outcomes(log.read_text())) for log in logs ] - assert sum(sizes) == 43 and all(sizes) + assert sum(sizes) == 44 and all(sizes) + # The last shard runs the test that needs the dataset and nothing else. + assert [ + r.nodeid + for r in checker.read_outcomes(logs[-1].read_text()) + if not r.is_collector + ] == ["suite/test_latest_data_smoke.py::test_needs_the_dataset"] allowlist = tmp_path / "allow.toml" allowlist.write_text(SAMPLE_ALLOWLIST, encoding="utf-8") diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index 9476822850..3511547f5e 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -175,7 +175,7 @@ def pytest_report_header(config): ] shard = config.getoption("shard", default=None) if shard is not None: - lines.append(f"shard: {shard[0]} of {shard[1]}, chosen by node id") + lines.append(f"shard: {shard[0]} of {shard[1]}, by marker and node id") return lines @@ -193,12 +193,17 @@ def cloned_uk_tax_benefit_system(): # CI splits the suite over several runners with ``--shard K/N``. The shard of a -# test is a function of its node id alone (and of the checked-in table below), -# so the shards partition whatever is collected: every test runs in exactly -# one of them, including a test added later. The Test job checks that on each -# run (.github/check_test_shards.py). +# test is a function of its marker and node id alone (and of the checked-in +# table below), so the shards partition whatever is collected: every test runs +# in exactly one of them, including a test added later. The Test job checks that +# on each run (.github/check_test_shards.py). # -# Most tests are placed by a hash of their node id. A hash spreads thousands +# The last shard runs the tests that need the dataset, which build simulations +# of all of it, and the other shards share the rest. CI gives the last shard +# one worker, so only one of those simulations runs at a time +# (.github/workflows/test-suite.yaml has the measurements). +# +# Most other tests are placed by a hash of their node id. A hash spreads thousands # of short tests evenly, but not a few long ones: on 24 October 2026 one # property test took 22 minutes, and by chance it and several others of 4 to # 7 minutes hashed to shard 2 of 5, which then took 57 minutes on average @@ -206,17 +211,6 @@ def cloned_uk_tax_benefit_system(): # slowest on CI, are instead dealt out by duration, longest first, each to the # shard with the least so far. A test missing from the table, or the table # missing, falls back to the hash. -# -# The tests of a file in SHARED_FIXTURE_FILES all run in one shard, because each -# shard that holds one of them builds that file's module fixture. In merge queue -# run 42212587066 the fixture in test_aggregate_impact_certified.py held one -# worker for 8 to 9 minutes at the start of five shards, which peaked at 14.2 to -# 15.3 GB on 16 GB runners, against 10.9 and 12.6 GB for the two without it. Runs -# 42218280596 and 42218280615 had those tests on all seven shards and lost eight -# of their 14 shard jobs at 15.4 to 15.7 GB. -SHARED_FIXTURE_FILES = frozenset( - {"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py"} -) SHARD_DURATIONS = ( Path(importlib.util.find_spec("policyengine_uk").origin).parent / "tests" @@ -236,36 +230,19 @@ def read_shard_durations(path: Path = SHARD_DURATIONS) -> dict[str, float]: } -def placement_unit(nodeid: str) -> str: - """What the deal and the hash place: a shared-fixture file, or else the test.""" - path = nodeid.split("::", 1)[0] - return path if path in SHARED_FIXTURE_FILES else nodeid - - -def deal_units(durations: Mapping[str, float], shards: int) -> dict[str, int]: - """Deal placement units to shards, longest first, each to the least loaded. +def balanced_shards(durations: Mapping[str, float], shards: int) -> dict[str, int]: + """Deal tests to shards, longest first, each to the least loaded shard. - A shared-fixture file weighs as much as its longest test, since the file's - long entries each measured the same fixture. Ties go to the lower name and - the lower shard, so every process gives the same answer. + Ties go to the lower node id and the lower shard, so every process gives + the same answer. """ - weights: dict[str, float] = {} - for nodeid, seconds in durations.items(): - unit = placement_unit(nodeid) - weights[unit] = max(seconds, weights.get(unit, seconds)) load = [0.0] * shards - dealt = {} - for unit, seconds in sorted(weights.items(), key=lambda kv: (-kv[1], kv[0])): + placed = {} + for nodeid, seconds in sorted(durations.items(), key=lambda kv: (-kv[1], kv[0])): shard = min(range(shards), key=lambda k: (load[k], k)) load[shard] += seconds - dealt[unit] = shard + 1 - return dealt - - -def balanced_shards(durations: Mapping[str, float], shards: int) -> dict[str, int]: - """The shard the deal gives each test in ``durations``.""" - dealt = deal_units(durations, shards) - return {nodeid: dealt[placement_unit(nodeid)] for nodeid in durations} + placed[nodeid] = shard + 1 + return placed _BALANCED: dict[int, dict[str, int]] = {} @@ -278,14 +255,24 @@ def shard_of(nodeid: str, shards: int) -> int: runner and every pytest-xdist worker must give the same answer. """ if shards not in _BALANCED: - _BALANCED[shards] = deal_units(read_shard_durations(), shards) - unit = placement_unit(nodeid) - if unit in _BALANCED[shards]: - return _BALANCED[shards][unit] - digest = hashlib.sha256(unit.encode("utf-8")).digest() + _BALANCED[shards] = balanced_shards(read_shard_durations(), shards) + if nodeid in _BALANCED[shards]: + return _BALANCED[shards][nodeid] + digest = hashlib.sha256(nodeid.encode("utf-8")).digest() return int.from_bytes(digest[:8], "big") % shards + 1 +def item_shard(item, shards: int) -> int: + """The shard, from 1 to ``shards``, that runs a collected test. + + The last of two or more shards runs the tests that need the dataset, and + ``shard_of`` places the others on the remaining shards. + """ + if shards == 1: + return 1 + return shards if needs_data(item) else shard_of(item.nodeid, shards - 1) + + def parse_shard(value: str) -> tuple[int, int]: """Parse ``K/N`` into ``(K, N)``, requiring 1 <= K <= N.""" match = re.fullmatch(r"([0-9]+)/([0-9]+)", value) @@ -315,7 +302,7 @@ def pytest_collection_modifyitems(config, items): this_shard, shards = shard selected, deselected = [], [] for item in items: - if shard_of(item.nodeid, shards) == this_shard: + if item_shard(item, shards) == this_shard: selected.append(item) else: deselected.append(item) diff --git a/policyengine_uk/tests/shard_durations.toml b/policyengine_uk/tests/shard_durations.toml index 5e04dd9d37..d4d48e2694 100644 --- a/policyengine_uk/tests/shard_durations.toml +++ b/policyengine_uk/tests/shard_durations.toml @@ -11,7 +11,8 @@ # test_ci_test_shards.py fails if an entry names a test that no longer # exists, so a pull request that renames or removes a listed test should # rename or delete its line here. Regenerate when a shard drifts well above -# the others. +# the others. The last shard runs the tests that need the dataset, so this +# table leaves them out: regenerate it from the other shards' logs. [seconds] "policyengine_uk/tests/test_capital_share_components.py::test_capital_matches_the_original_lists" = 1299 "policyengine_uk/tests/test_statutory_pension_edges.py::test_nics_cap_preserves_aa_inputs_and_charge_with_fixed_ordinary_tax[2029]" = 761 @@ -21,14 +22,10 @@ "policyengine_uk/tests/test_dated_reform_routes.py::test_routes_agree_on_start_stop_keys" = 392 "policyengine_uk/tests/test_reform_representations.py::test_parametric_reform_representations_agree" = 368 "policyengine_uk/tests/test_tax_credits_pension_credit_passport_properties.py::test_results_do_not_depend_on_request_order" = 340 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_accounting_identities_and_closure_hold" = 338 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_household_detail_is_complete" = 338 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_every_tax_component_is_counted_exactly_once" = 333 "policyengine_uk/tests/test_childcare_entitlement_floor_properties.py::test_more_hours_used_never_lowers_childcare_support" = 330 "policyengine_uk/tests/test_pension_credit_earnings_disregard_properties.py::test_statute_bounds_differential_and_monotonicity" = 330 "policyengine_uk/tests/test_tax_credits_pension_credit_passport_properties.py::test_without_awards_pension_credit_alone_decides_the_passport" = 320 "policyengine_uk/tests/test_brma_region_default.py::test_brma_with_sparse_regions_follows_the_rule_and_keeps_regions" = 302 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_declared_double_counts_are_still_double_counted_by_the_model" = 301 "policyengine_uk/tests/test_housing_benefit_universal_credit_passport_properties.py::test_universal_credit_limb_dates" = 296 "policyengine_uk/tests/test_from_reform_modifier.py::test_sequence_modifier_matches_apply_with_repeated_and_reordered_writes[True]" = 288 "policyengine_uk/tests/test_pension_credit_child_amount_tax_credit_properties.py::test_results_do_not_depend_on_request_order" = 286 @@ -359,8 +356,6 @@ "policyengine_uk/tests/test_housing_benefit_savings_credit_only_properties.py::test_income_follows_the_route_for_the_award" = 50 "policyengine_uk/tests/test_housing_benefit_specified_accommodation_disregard_properties.py::test_income_support_disregards_all_earnings" = 49 "policyengine_uk/tests/test_pension_age_severe_disability_premium_properties.py::test_net_income_does_not_fall_where_guarantee_credit_ends" = 49 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise personal allowance by ~800GBP/year]" = 49 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise basic rate by 1pp]" = 49 "policyengine_uk/tests/test_scenario_sum_timing.py::test_a_simulation_runs_each_part_once_at_its_own_time" = 48 "policyengine_uk/tests/test_uc_statutory_pay_properties.py::test_statutory_pay_counts_as_wages_and_earned_income_adds_up" = 48 "policyengine_uk/tests/test_uc_work_related_groups_properties.py::test_thresholds_follow_the_group_and_expected_hours" = 48 @@ -380,7 +375,6 @@ "policyengine_uk/tests/test_variable_module_release.py::test_dead_systems_modules_are_released_while_a_test_runs" = 47 "policyengine_uk/tests/test_housing_benefit_pension_age_cohorts.py::test_council_tax_reduction_splits_only_in_england" = 47 "policyengine_uk/tests/test_capital_gains_tax_components.py::TestEqualisationReforms::test_main_rate_reform_leaves_the_other_schedules_at_current_law" = 47 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise Class 1 main employee NICs rate to 10%]" = 47 "policyengine_uk/tests/test_uc_property_capital.py::test_financial_allocation_conserves_pool_with_reports_and_complete_overrides" = 46 "policyengine_uk/tests/test_housing_benefit_specified_accommodation_disregard_properties.py::test_no_earner_has_no_para_18_amount" = 46 "policyengine_uk/tests/test_housing_benefit_earnings_disregard_properties.py::test_disregard_lowers_applicable_income_by_no_more_than_earnings" = 46 @@ -405,8 +399,6 @@ "policyengine_uk/tests/test_pension_credit_war_pension_properties.py::test_guarantee_credit_does_not_rise_with_an_afcs_payment" = 44 "policyengine_uk/tests/test_ni_class_2.py::test_ni_class_2_properties" = 44 "policyengine_uk/tests/test_statutory_paternity_pay_income_tax_properties.py::test_statutory_paternity_pay_is_treated_as_statutory_sick_pay" = 44 -"policyengine_uk/tests/microsimulation/test_salary_sacrifice_cap_reform.py::test_salary_sacrifice_cap_revenue_impact" = 44 -"policyengine_uk/tests/microsimulation/test_state_pension_reform.py::test_relative_state_pension_reform_affects_budget" = 44 "policyengine_uk/tests/test_uc_childcare_work_condition_properties.py::test_condition_matches_the_regulation_32_reference" = 43 "policyengine_uk/tests/test_housing_benefit_specified_accommodation_disregard_properties.py::test_bounds" = 43 "policyengine_uk/tests/test_housing_benefit_universal_credit_passport_properties.py::test_universal_credit_passport_ignores_income_and_capital" = 43 @@ -421,11 +413,8 @@ "policyengine_uk/tests/test_pension_annual_allowance_properties.py::test_charge_matches_band_by_band_reference" = 43 "policyengine_uk/tests/test_severe_disability_premium_properties.py::test_premium_matches_independent_oracle_and_range" = 43 "policyengine_uk/tests/test_uc_company_owner_properties.py::test_property_company_gives_no_earnings" = 43 -"policyengine_uk/tests/test_scenario_baseline_parameters.py::test_before_load_spending_scenario_constructs_on_the_dataset" = 43 "policyengine_uk/tests/test_uc_disabled_child_addition_properties.py::test_exact_award_categories_and_blindness_select_one_tier" = 42 "policyengine_uk/tests/test_lha_published_rates.py::test_northern_ireland_maxima_and_minimum_change_only_northern_ireland" = 42 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_head_totals_agree_with_the_model_weighted_sums" = 42 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise higher rate by 1pp]" = 42 "policyengine_uk/tests/test_salary_sacrifice_ni_only.py::test_explicit_actual_return_matches_actual_pay_and_member_inputs[employee_pension_contributions]" = 42 "policyengine_uk/tests/test_dataframe_dataset.py::test_an_exported_frame_gives_the_results_of_its_dataset[default export]" = 42 "policyengine_uk/tests/test_ni_class_4_profits_properties.py::test_losses_carry_forward_over_many_years" = 42 @@ -462,12 +451,10 @@ "policyengine_uk/tests/test_uc_childcare_work_condition_properties.py::test_a_couple_both_in_paid_work_always_meets_the_condition" = 37 "policyengine_uk/tests/test_state_pension_components.py::test_add_on_follows_september_cpi_and_nothing_else_changes" = 37 "policyengine_uk/tests/test_property_income_band_base_properties.py::test_non_savings_taxable_income_is_the_rest_after_allowances" = 37 -"policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py::test_spending_never_rises_by_more_than_is_added_except_known_defects" = 37 "policyengine_uk/tests/test_rent_of_sharers_boarders_and_lodgers_properties.py::test_non_dependants" = 37 "policyengine_uk/tests/test_statutory_paternity_pay_income_tax_properties.py::test_statutory_paternity_pay_is_taxed_as_wages" = 36 "policyengine_uk/tests/test_trading_loss_properties.py::test_class_4_profits_net_of_the_loss_balance" = 36 "policyengine_uk/tests/test_dataframe_dataset.py::test_an_exported_frame_gives_the_results_of_its_dataset[export of every stored value]" = 36 -"policyengine_uk/tests/microsimulation/test_salary_sacrifice_cap_reform.py::test_decile_impact_negative_for_higher_earners" = 36 "policyengine_uk/tests/test_dated_reform_routes.py::test_a_fresh_process_agrees_with_a_warm_one" = 36 "policyengine_uk/tests/test_statutory_pension_edges.py::test_trade_losses_do_not_leak_nics_reclassification_into_aa[DB-2030]" = 35 "policyengine_uk/tests/test_trading_loss_properties.py::test_relief_is_the_loss_within_net_income_and_the_cap" = 35 @@ -484,19 +471,14 @@ "policyengine_uk/tests/test_council_tax_reduction_child_benefit_properties.py::test_pensioner_ctr_is_invariant_to_actual_child_benefit_claim" = 33 "policyengine_uk/tests/test_child_benefit_charge_rounding_properties.py::test_simulated_charge_matches_the_statute" = 33 "policyengine_uk/tests/code_health/test_cached_arrays_not_written_in_place.py::test_branching_runs_with_read_only_cache[labour_supply_responses]" = 33 -"policyengine_uk/tests/microsimulation/test_salary_sacrifice_cap_reform.py::test_income_tax_impact" = 33 "policyengine_uk/tests/test_disable_simulated_benefits_winter_heating.py::test_reported_winter_heating_is_conserved" = 33 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Reduce Universal Credit taper rate to 20%]" = 33 "policyengine_uk/tests/test_capital_gains_tax_components.py::TestEqualisationReforms::test_setting_every_schedule_taxes_every_gain_alike" = 32 "policyengine_uk/tests/test_triple_lock_statutory_inputs.py::test_policy_levers_apply_from_the_april_of_the_year_they_name" = 32 "policyengine_uk/tests/test_disable_simulated_benefits_winter_heating.py::test_reported_winter_heating_follows_each_years_country" = 32 "policyengine_uk/tests/test_uc_health_element_freeze.py::test_lifting_the_freeze_reaches_a_simulated_new_claimant" = 31 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise child benefit by 25GBP/week per additional child]" = 31 "policyengine_uk/tests/test_state_pension_age.py::test_extracted_household_keeps_its_birthday" = 30 "policyengine_uk/tests/test_benefit_cap_specified_benefits_properties.py::test_an_element_a_reform_gives_is_decided_by_its_value[uc_carer_element]" = 30 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise VAT standard rate by 2pp]" = 30 "policyengine_uk/tests/test_benefit_cap_specified_benefits_properties.py::test_an_element_a_reform_gives_is_decided_by_its_value[uc_LCWRA_element]" = 29 -"policyengine_uk/tests/microsimulation/test_reform_impacts.py::test_reform_fiscal_impacts[Raise additional rate by 3pp]" = 28 "policyengine_uk/tests/test_dated_reform_routes.py::test_a_scenario_keeps_its_modifiers_parameters_and_its_parameter_changes[False]" = 28 "policyengine_uk/tests/test_targeted_childcare_guarantee_credit_properties.py::test_known_families_detect_the_earlier_routes" = 27 "policyengine_uk/tests/test_data_built_imputations.py::test_imputations_do_not_depend_on_total_weight" = 27 @@ -521,7 +503,6 @@ "policyengine_uk/tests/test_from_reform_modifier.py::test_modifier_matches_scenario_apply_on_built_parameters" = 24 "policyengine_uk/tests/test_legacy_award_readers_properties.py::test_an_outside_adults_work_keeps_their_non_dependant_deduction" = 24 "policyengine_uk/tests/test_savings_credit_only_receipt_properties.py::test_stopping_pension_credit_payments_ends_savings_credit_only_receipt" = 24 -"policyengine_uk/tests/microsimulation/test_salary_sacrifice_cap_reform.py::test_ni_increases_with_reform" = 24 "policyengine_uk/tests/test_dataframe_dataset.py::test_a_reform_on_an_exported_frame_gives_the_results_of_its_dataset" = 24 "policyengine_uk/tests/test_capital_gains_responses.py::test_realisations_fall_when_rates_rise" = 24 "policyengine_uk/tests/test_lha_freeze.py::test_great_britain_lha_maxima_and_minimum_leave_northern_ireland_unchanged" = 24 @@ -561,10 +542,8 @@ "policyengine_uk/tests/test_marginal_rates_hold_behavioural_responses.py::test_marginal_rates_hold_the_labour_supply_response[marginal_tax_rate_on_capital_gains-22000-0-changes0]" = 21 "policyengine_uk/tests/test_claims_all_entitled_benefits.py::TestPerBenefitUnit::test_reporters_of_any_of_the_seven_are_false" = 21 "policyengine_uk/tests/test_marginal_rates_hold_behavioural_responses.py::test_marginal_tax_rate_holds_the_capital_gains_response[13000]" = 21 -"policyengine_uk/tests/test_latest_data_smoke.py::test_universal_credit_aggregate_in_range" = 21 "policyengine_uk/tests/test_capital_gains_response_default.py::test_pay_rise_branches_hold_the_realisation_response" = 21 "policyengine_uk/tests/code_health/test_cached_arrays_not_written_in_place.py::test_branching_runs_with_read_only_cache[capital_gains_responses]" = 21 -"policyengine_uk/tests/microsimulation/test_state_pension_reform.py::test_basic_state_pension_responds_to_reform" = 21 "policyengine_uk/tests/test_council_tax_reduction_pensioner_earnings_disregard_properties.py::test_disregard_is_the_statutory_weekly_sum_in_every_year" = 21 "policyengine_uk/tests/test_marginal_rates_hold_behavioural_responses.py::test_marginal_rates_hold_the_labour_supply_response[marginal_tax_rate_wrt_employer_cost-0-10000-changes2]" = 21 "policyengine_uk/tests/test_disable_simulated_benefits_winter_heating.py::test_reform_sets_reported_benefits[Simulation]" = 21 From ca2ea8da10120b7db84febd0b805523928fe4e6b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 18:33:32 -0500 Subject: [PATCH 3/3] Report CI memory in MiB, the unit free -m counts The Test job's memory log printed free -m's figures, which count MiB, under the label MB, and the workflow's comments called its peaks GB, some of them in thousands of MiB. The log now labels its figures MiB, .github/test_durations.py reads lines with either label, and the comments give each peak in GiB: - merge queue run 42212587066: 14.2 to 15.3 GiB on the five shards that built the certified fixture, 10.9 and 12.6 GiB on the two without it; - runs 42218280596 and 42218280615: 15.1 to 15.4 GiB on the eight shard jobs they lost within minutes, and 15.2 GiB on the shard the first lost 51 minutes in; - run 42231098419: 15.2 GiB on the certified shard it lost; - run 38156967136: 15.3 GiB of the 15.6 GiB the runner reports as used and available. On two shard logs from 595fd3fdf, the parser gives the same durations as before, and the same again when each memory line says MiB. Co-Authored-By: Claude Opus 5.5 --- .github/test_durations.py | 5 +++-- .github/workflows/test-suite.yaml | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/test_durations.py b/.github/test_durations.py index 9f46bbbf36..b5d7e9e954 100644 --- a/.github/test_durations.py +++ b/.github/test_durations.py @@ -28,8 +28,9 @@ LINE = re.compile( r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?)Z " r"\[(gw\d+)\] \[\s*\d+%\] (?:PASSED|FAILED|SKIPPED|XFAIL|XPASS|ERROR) (.+?)" - # The job's once-a-minute memory report can land on the same line. - r"(?: memory: \d+ MB used, \d+ MB available)? *$" + # The job's once-a-minute memory report can land on the same line. It + # counts MiB; logs from before November 2026 label it MB. + r"(?: memory: \d+ Mi?B used, \d+ Mi?B available)? *$" ) # pytest-xdist logs this once its workers are up; a worker's first test is # timed from here. diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 840c36e072..c834d29968 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -188,13 +188,13 @@ jobs: # dataset, with one worker, so only one simulation of the dataset runs at # a time, and the other seven run none of them. In merge queue run # 42212587066 the five shards that built the fixture in - # test_aggregate_impact_certified.py peaked at 14.2 to 15.3 GB, and the two - # without it at 10.9 and 12.6 GB. On a merge of 28 pull requests with those - # tests on all seven shards, runs 42218280596 and 42218280615 lost eight of - # their 14 shard jobs within minutes at 15.4 to 15.7 GB, and shard 6/7 of - # the first lost its runner at 15.6 GB 51 minutes in, with + # test_aggregate_impact_certified.py peaked at 14.2 to 15.3 GiB, and the + # two without it at 10.9 and 12.6 GiB. On a merge of 28 pull requests with + # those tests on all seven shards, runs 42218280596 and 42218280615 lost + # eight of their 14 shard jobs within minutes at 15.1 to 15.4 GiB, and + # shard 6/7 of the first lost its runner at 15.2 GiB 51 minutes in, with # test_latest_data_smoke.py running. With the certified tests on one shard - # of seven, run 42231098419 lost that shard at 15.6 GB. + # of seven, run 42231098419 lost that shard at 15.2 GiB. name: Python ${{ matrix.shard }}/${{ strategy.job-total }} if: ${{ !inputs.skip }} runs-on: ${{ vars.CI_RUNNER_ARM != '' && (vars.CI_RUNNER_EVENTS == '' || contains(format(',{0},', vars.CI_RUNNER_EVENTS), format(',{0},', github.event_name))) && vars.CI_RUNNER_ARM || 'ubuntu-24.04-arm' }} @@ -252,9 +252,10 @@ jobs: export HUGGING_FACE_TOKEN="$HF_TOKEN_VALUE" fi # Log memory each minute, so a runner lost to memory pressure - # ("The runner has received a shutdown signal") shows it. + # ("The runner has received a shutdown signal") shows it. free -m + # counts MiB. (while sleep 60; do - free -m | awk 'NR == 2 { print "memory: " $3 " MB used, " $7 " MB available" }' + free -m | awk 'NR == 2 { print "memory: " $3 " MiB used, " $7 " MiB available" }' done) & trap 'kill $! 2>/dev/null' EXIT coverage=() @@ -266,8 +267,8 @@ jobs: # when its test passes, so a fixed bug cannot hide behind it. # Four pytest-xdist workers, one per core, and one on the last shard. # Their memory grows with each test they run (#2330): with three - # shards they peaked at 15.6 GB of the runner's 16 GB (run - # 38156967136). + # shards they peaked at 15.3 GiB of the 15.6 GiB the runner reports + # as used and available (run 38156967136). PYTHONPATH="$GITHUB_WORKSPACE/.github" pytest "$TESTS" \ --rootdir "$ROOTDIR" -c pyproject.toml -p no:cacheprovider \ -p pytest_outcome_log --outcome-log="$OUTCOMES" \