diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8d17d56fa..126db3e30 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -171,10 +171,6 @@ jobs: needs: workspace runs-on: ${{ matrix.os }} timeout-minutes: 20 - concurrency: - # Both OS variants use the same workspace; avoid overlapping same-agent runs. - group: integration-smoke-${{ github.run_id }}-${{ matrix.agent }} - cancel-in-progress: false strategy: fail-fast: false matrix: @@ -311,16 +307,13 @@ jobs: if: ${{ !cancelled() && needs.workspace.result == 'success' }} needs: [workspace, smoke] runs-on: ${{ matrix.os }} - concurrency: - # Keep the previous per-agent serialization across the new OS matrix. - group: integration-full-${{ github.run_id }}-${{ matrix.agent }} - cancel-in-progress: false # The runner enforces its own 3600s pytest deadline; keep the job above it # so a slow run fails through the runner's report instead of a job kill. timeout-minutes: 70 strategy: fail-fast: false - # One runner per OS variant; concurrency above serializes variants per agent. + # One runner per agent and OS, all in parallel. Cases share no workspace state, + # only inference capacity. max-parallel: 4 matrix: os: [ubuntu-22.04, windows-latest] @@ -331,7 +324,8 @@ jobs: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} DEPENDENCY: ${{ inputs.dependency }} AGENT: ${{ matrix.agent }} - TEST_MARKER: live and ${{ matrix.agent }} and not soft_fail + # Smoke cases already ran on this OS in the smoke job; don't repeat them here. + TEST_MARKER: live and ${{ matrix.agent }} and not soft_fail and not smoke ARTIFACT_NAME: integration-full-${{ matrix.agent }}-${{ matrix.os }} steps: *live-steps diff --git a/tests/integration/README.md b/tests/integration/README.md index c0c147c75..f7a9bd322 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -989,13 +989,13 @@ each test; only explicit-model scenarios choose and record a discovered Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, -two agent jobs). Full runs all 88 live cases, including those smoke cases, in two -disjoint agent lanes: +two agent jobs per OS). Full runs every other live case in two disjoint agent lanes; +it excludes the smoke cases, which smoke already ran on the same OS: -| Agent lane | Marker | Cases | -| --- | --- | --- | -| Claude | `live and claude` | 40 | -| Codex | `live and codex` | 46 | +| Agent lane | Marker | +| --- | --- | +| Claude | `live and claude and not soft_fail and not smoke` | +| Codex | `live and codex and not soft_fail and not smoke` | A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with `continue-on-error` and is not part of the required `cujs` gate until it is stable. @@ -1010,11 +1010,10 @@ Each lane installs only the agent CLIs it needs (one, except the managed OpenCod three), once, and runs all its configure, headless, commands, lifecycle, and applicable app-server journeys. Cases remain serial inside each fresh VM because configure/revert can touch machine-level settings; -separate runners isolate those writes as well as the PTYs. Claude and Codex run -in parallel, with at most one full-suite job per agent in a workflow run. This -avoids six serial job startups without overlapping same-agent shards. The two -lanes still share workspace capacity with the concurrently running agent e2e -shards and other PRs; this limit does not guarantee freedom from rate limits. +separate runners isolate those writes as well as the PTYs. All four agent × OS +Full jobs run in parallel: cases share no workspace state, only inference capacity. +They share that capacity with each other, the concurrently running agent e2e shards, +and other PRs, so rate limits remain possible. No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 01f9eb06d..0b654c838 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -104,26 +104,7 @@ def test_soft_fail_marks_only_the_affected_claude_journey(): assert marked == [("test_ug_smart_routing_hooks.py", name) for name in CLAUDE_TOGGLE_TESTS] -@pytest.mark.parametrize( - ("job_name", "agent", "retained_test"), - [ - ("smoke", "claude", "test_ug_claude_headless_prompt_argument"), - ("smoke", "codex", "test_ug_codex_headless_prompt_argument"), - ("full", "claude", "test_smart_routing_claude_route_subagent_hook"), - ("full", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), - ("managed", "claude", "test_ug_configure_managed_claude"), - ("managed", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), - ("soft-fail", "claude", None), - ], -) -def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test): - job = _integration_jobs()[job_name] - if job_name == "managed": - marker = re.search(r'-- -m "([^"]+)"', job) - else: - marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", job) - assert marker is not None - expression = marker.group(1).replace("${{ matrix.agent }}", agent).replace("$AGENT", agent) +def _collect_integration_cases(expression): result = subprocess.run( [ sys.executable, @@ -144,6 +125,55 @@ def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test assert result.returncode == 0, result.stdout + result.stderr cases = {line for line in result.stdout.splitlines() if line.startswith("test_")} assert cases, result.stdout + return cases + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_smoke_and_full_together_cover_every_blocking_live_case(agent): + jobs = _integration_jobs() + selected = [] + for name in ("smoke", "full"): + marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", jobs[name]) + assert marker is not None + selected.append( + _collect_integration_cases(marker.group(1).replace("${{ matrix.agent }}", agent)) + ) + smoke, full = selected + # Full skips smoke cases only because smoke runs them on the same OS matrix. + assert "os: [ubuntu-22.04, windows-latest]" in jobs["smoke"] + assert "os: [ubuntu-22.04, windows-latest]" in jobs["full"] + assert not smoke & full + assert smoke | full == _collect_integration_cases(f"live and {agent} and not soft_fail") + + +def test_smoke_and_full_journeys_run_every_agent_and_os_in_parallel(): + jobs = _integration_jobs() + assert "concurrency:" not in jobs["smoke"] + assert "concurrency:" not in jobs["full"] + assert "max-parallel: 4" in jobs["full"] + + +@pytest.mark.parametrize( + ("job_name", "agent", "retained_test"), + [ + ("smoke", "claude", "test_ug_claude_headless_prompt_argument"), + ("smoke", "codex", "test_ug_codex_headless_prompt_argument"), + ("full", "claude", "test_smart_routing_claude_route_subagent_hook"), + ("full", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), + ("managed", "claude", "test_ug_configure_managed_claude"), + ("managed", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), + ("soft-fail", "claude", None), + ], +) +def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test): + job = _integration_jobs()[job_name] + if job_name == "managed": + marker = re.search(r'-- -m "([^"]+)"', job) + else: + marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", job) + assert marker is not None + expression = marker.group(1).replace("${{ matrix.agent }}", agent).replace("$AGENT", agent) + cases = _collect_integration_cases(expression) if retained_test is not None: assert any(f"::{retained_test}" in case for case in cases), cases assert not any(f"::{toggle}[" in case for case in cases for toggle in CLAUDE_TOGGLE_TESTS)