From fd8d4297ce6d3490a39366683c05e9d1790f2730 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:06:33 +0000 Subject: [PATCH 1/5] Add pytest-xdist to the dev extra Lock pytest-xdist 3.8.0 and its dependency execnet 2.1.2. No other locked package moves; pexpect's ptyprocess marker keeps the form main locked, which uv lock --check accepts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6J8V2hdzByTC5Q7Fnu --- pyproject.toml | 1 + uv.lock | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 60347795..64c95e11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dev = [ "hypothesis>=6.100", "pytest>=8.0", "pytest-cov>=5.0", + "pytest-xdist>=3.6", "ruff>=0.4", ] # The paper render environment (paper/render_paper.py, .github/workflows/paper.yml): diff --git a/uv.lock b/uv.lock index d7da6bde..4cf961e3 100644 --- a/uv.lock +++ b/uv.lock @@ -755,6 +755,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -2321,6 +2330,7 @@ dev = [ { name = "hypothesis" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, ] docs = [ @@ -2345,6 +2355,7 @@ requires-dist = [ { name = "policyengine-us", specifier = "==2.60.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, ] provides-extras = ["dev", "docs"] @@ -2761,6 +2772,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From a79e57d1f8d05d459cce17306a4b8915ab6bd412 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:08:01 +0000 Subject: [PATCH 2/5] Parametrize the audit-pin refusal tests in a fixed order test_any_refused_input_stops_the_impact_script_before_it_scores and test_any_refused_input_stops_the_direct_script_before_it_writes built their parameters by iterating over sets of path strings, whose order follows the per-process string hash seed. pytest-xdist requires every worker to collect the same tests in the same order, so the workers refused to start. Iterate over the sorted paths instead; the collected node ids are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6J8V2hdzByTC5Q7Fnu --- tests/test_reference_audit_pins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reference_audit_pins.py b/tests/test_reference_audit_pins.py index 9499e212..db82bfe8 100644 --- a/tests/test_reference_audit_pins.py +++ b/tests/test_reference_audit_pins.py @@ -513,7 +513,7 @@ def call(*args, **kwargs): @pytest.mark.parametrize( ("audit", "path"), - [(audit, path) for audit in IMPACT for path in IMPACT[audit].inputs], + [(audit, path) for audit in IMPACT for path in sorted(IMPACT[audit].inputs)], ) def test_any_refused_input_stops_the_impact_script_before_it_scores( scratch, monkeypatch, audit, path @@ -533,7 +533,7 @@ def test_any_refused_input_stops_the_impact_script_before_it_scores( @pytest.mark.parametrize( ("script", "path"), - [(script, path) for script in DIRECT for path in DIRECT[script].inputs], + [(script, path) for script in DIRECT for path in sorted(DIRECT[script].inputs)], ) def test_any_refused_input_stops_the_direct_script_before_it_writes( scratch, monkeypatch, script, path From 4491519e39d6049253197edafc0f111eaa087d3f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:47:41 +0000 Subject: [PATCH 3/5] Run the CI test job in parallel with pytest-xdist Run pytest with -n auto --dist loadfile, one worker per runner core, and restate the timeout comment from the new measurements: the three latest serial test jobs on main took 17.3, 15.3 and 19.9 minutes; on a 4-core VM the suite took 34.4 minutes serially and 9.2 to 9.7 minutes in parallel. Keep timeout-minutes at 30. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6J8V2hdzByTC5Q7Fnu --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 597d540d..f43440c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,8 +30,11 @@ jobs: test: runs-on: ubuntu-latest - # Slowest of 12 successful runs, 2026-10-05..08: 7.2 minutes. Fail a hung - # job instead of holding a shared runner for GitHub's 6-hour default. + # Run serially, the three latest test jobs on main took 17.3, 15.3 and + # 19.9 minutes (2026-10-31, 10-26, 10-25). On a 4-core VM on 2026-11-02 + # the suite took 34.4 minutes serially and 9.2 to 9.7 minutes in three + # runs with -n 4 --dist loadfile. Fail a hung job instead of holding a + # shared runner for GitHub's 6-hour default. timeout-minutes: 30 steps: - uses: actions/checkout@v6 @@ -44,7 +47,9 @@ jobs: python-version: "3.12" - run: pip install uv - run: uv sync --locked --extra dev - - run: uv run pytest -m "not slow" --tb=short -q + # One worker per core. --dist loadfile keeps each test file on one + # worker; on the 4-core VM it ran faster than --dist load (10.1 minutes). + - run: uv run pytest -m "not slow" -n auto --dist loadfile --tb=short -q app: runs-on: ubuntu-latest From 525d167c28a94d1fa6532479474ede5cc815bc21 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 16:12:47 -0500 Subject: [PATCH 4/5] Start one CI test worker per logical CPU and log the count With psutil installed, as the lock installs it, -n auto starts one worker per physical core, which can be fewer than the runner's CPUs. -n logical starts one per logical CPU, and without -q pytest prints its "created: N/N workers" line. The timeout comment now cites main's serial job, which hit the 30-minute limit at ec5413c, and the parallel job's times. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f43440c5..fe0a46a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,10 @@ jobs: test: runs-on: ubuntu-latest - # Run serially, the three latest test jobs on main took 17.3, 15.3 and - # 19.9 minutes (2026-10-31, 10-26, 10-25). On a 4-core VM on 2026-11-02 - # the suite took 34.4 minutes serially and 9.2 to 9.7 minutes in three - # runs with -n 4 --dist loadfile. Fail a hung job instead of holding a - # shared runner for GitHub's 6-hour default. + # Serially, the test job on main took 29.5 minutes at b0c8f2e and hit + # this limit at ec5413c (2026-11-02). In parallel, with -n auto, it took + # 12.7, 12.5 and 16.2 minutes (run 41490222217). Fail a hung job instead + # of holding a shared runner for GitHub's 6-hour default. timeout-minutes: 30 steps: - uses: actions/checkout@v6 @@ -47,9 +46,11 @@ jobs: python-version: "3.12" - run: pip install uv - run: uv sync --locked --extra dev - # One worker per core. --dist loadfile keeps each test file on one - # worker; on the 4-core VM it ran faster than --dist load (10.1 minutes). - - run: uv run pytest -m "not slow" -n auto --dist loadfile --tb=short -q + # One worker per logical CPU (-n auto would count physical cores, since + # psutil is installed); the "created: N/N workers" line gives the count. + # --dist loadfile keeps each test file on one worker; on the 4-core VM it + # ran faster than --dist load (10.1 minutes). + - run: uv run pytest -m "not slow" -n logical --dist loadfile --tb=short app: runs-on: ubuntu-latest From efcecfc5deda5654f09424fb9332551cb0762d5f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 16:30:44 -0500 Subject: [PATCH 5/5] Cite the -n logical run in the timeout comment and note parallel local runs The comment on the test job's timeout now gives the parallel time with -n logical (4 workers, 12.3 minutes in run 41526298328) next to the three -n auto attempts. CLAUDE.md gives the four-worker pytest command and says to run the suite from a full clone, since some tests read git history. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 ++++--- CLAUDE.md | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe0a46a9..959fbe5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,10 @@ jobs: test: runs-on: ubuntu-latest # Serially, the test job on main took 29.5 minutes at b0c8f2e and hit - # this limit at ec5413c (2026-11-02). In parallel, with -n auto, it took - # 12.7, 12.5 and 16.2 minutes (run 41490222217). Fail a hung job instead - # of holding a shared runner for GitHub's 6-hour default. + # this limit at ec5413c (2026-11-02). In parallel it took 12.3 minutes + # with -n logical (4 workers, run 41526298328), and 12.5 to 16.2 minutes + # in three attempts with -n auto (run 41490222217). Fail a hung job + # instead of holding a shared runner for GitHub's 6-hour default. timeout-minutes: 30 steps: - uses: actions/checkout@v6 diff --git a/CLAUDE.md b/CLAUDE.md index d812a296..e33cbcc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,8 @@ ruff format . # Format ## Testing - All tests mock external calls (LiteLLM, PE-US API) - `pytest -m "not slow"` to skip slow tests +- `pytest -m "not slow" -n 4 --dist loadfile` to split them across four workers (CI starts one per CPU) +- Some tests read git history, so run the suite from a full clone (`git fetch --unshallow`) - Full benchmark runs are manual and expensive ## Formatting