diff --git a/Makefile b/Makefile index c4a8a7fd82c..031b882e1d2 100644 --- a/Makefile +++ b/Makefile @@ -175,12 +175,9 @@ test-yaml-structural-heavy-shard-4: test-yaml-structural-other: # Per-subdir so every remaining contrib folder runs in its own subprocess, # instead of stacking ~20 light files into one ~13-min catch-all batch that - # risked the 30-min per-batch timeout on slow runners. SSA runs separately - # per-file below because its reform combinations retain large model copies. + # risked the 30-min per-batch timeout on slow runners. SSA runs per-file in + # shard 3 because its reform combinations retain large model copies. $(BATCH) $(TESTS)/policy/contrib --exclude states,ctc,ubi_center,federal,harris,treasury,crfb,congress,refundable_credit_conversion,ssa --mode per-subdir --workers 1 - # SSA's COLA file measured about 11 GB locally. Override CI's worker setting - # as well as the CLI default so each file exits before the next one starts. - POLICYENGINE_BATCH_WORKERS=1 $(BATCH) $(TESTS)/policy/contrib/ssa --mode per-file --workers 1 test-yaml-structural-other-shard-2a: # ctc is microsim-heavy: per-file isolation frees each ~5 GB peak between # files, and --workers 1 keeps only one peak resident at a time. @@ -201,6 +198,12 @@ test-yaml-structural-other-shard-3: # file). Per-file isolation frees each peak between files; --workers 1 keeps # one peak resident at a time. $(BATCH) $(TESTS)/policy/contrib/refundable_credit_conversion --mode per-file --workers 1 + # SSA's COLA file measured about 11 GB locally. Override CI's worker setting + # as well as the CLI default so each file exits before the next one starts. + # SSA moved here from shard 1, which also runs the contrib Python tests: + # with 12 files it took 27 min there, and the shard hit CI's 60-min timeout + # (release run 41123038987). + POLICYENGINE_BATCH_WORKERS=1 $(BATCH) $(TESTS)/policy/contrib/ssa --mode per-file --workers 1 test-yaml-structural-congress: # One subprocess per congress proposal, split when a proposal's reform # combos exceed the batcher's budget; new proposals auto-route. diff --git a/changelog.d/ci-move-contrib-ssa-to-shard-3.fixed.md b/changelog.d/ci-move-contrib-ssa-to-shard-3.fixed.md new file mode 100644 index 00000000000..7b1c2d013cb --- /dev/null +++ b/changelog.d/ci-move-contrib-ssa-to-shard-3.fixed.md @@ -0,0 +1 @@ +Run the contrib SSA YAML tests in Contrib shard 3 instead of shard 1, which also runs the contrib Python tests and was hitting CI's 60-minute timeout. diff --git a/policyengine_us/tests/code_health/test_make_test_matches_ci.py b/policyengine_us/tests/code_health/test_make_test_matches_ci.py index 68437fd99b5..4cf4b8fb683 100644 --- a/policyengine_us/tests/code_health/test_make_test_matches_ci.py +++ b/policyengine_us/tests/code_health/test_make_test_matches_ci.py @@ -119,28 +119,42 @@ def test_contrib_ssa_yaml_has_a_dedicated_single_file_serial_route(): Its large reform combinations need separate files and an effective one-worker limit even when CI exports POLICYENGINE_BATCH_WORKERS=2. + The route runs in shard 3, not shard 1, which also runs the contrib Python + tests: with 12 files SSA took 27 minutes there, and shard 1 hit CI's + 60-minute timeout (release run 41123038987). """ root = "policyengine_us/tests/policy/contrib" ssa = f"{root}/ssa" + shared, shard = "test-yaml-structural-other", "test-yaml-structural-other-shard-3" assert any((REPO_ROOT / ssa).rglob("*.yaml")), "SSA YAML coverage is absent" - commands = make_commands("test-yaml-structural-other") batcher = "policyengine_us/tests/test_batched.py" - batch_commands = [ - shlex.split(command) for command in commands if batcher in command - ] - dedicated = [arguments for arguments in batch_commands if ssa in arguments] + + def batch_commands(target): + return [ + shlex.split(command) + for command in make_commands(target) + if batcher in command + ] + + assert not [ + arguments for arguments in batch_commands(shared) if ssa in arguments + ], "SSA must not share shard 1 with the contrib Python tests" + dedicated = [arguments for arguments in batch_commands(shard) if ssa in arguments] assert len(dedicated) == 1, "SSA must have one dedicated YAML invocation" (arguments,) = dedicated assert arguments[arguments.index("--mode") + 1] == "per-file" assert arguments[arguments.index("--workers") + 1] == "1" assert "POLICYENGINE_BATCH_WORKERS=1" in arguments - generic = [arguments for arguments in batch_commands if root in arguments] + generic = [arguments for arguments in batch_commands(shared) if root in arguments] assert len(generic) == 1 (arguments,) = generic excluded = arguments[arguments.index("--exclude") + 1].split(",") assert "ssa" in excluded, "SSA YAML must not also enter a shared subdir batch" - assert commands <= make_commands("test"), "The full suite must retain SSA's route" + assert shard in ci_suites()[0], "CI must run SSA's route" + assert make_commands(shard) <= make_commands("test"), ( + "The full suite must retain SSA's route" + ) @pytest.mark.parametrize("workflow_name", ("python-tests.yaml", "push.yaml"))