From c101d551d73ae555564897703ff9edcb27c964d9 Mon Sep 17 00:00:00 2001 From: Rohit Agrawal Date: Wed, 28 Oct 2026 14:35:48 -0400 Subject: [PATCH 1/3] Run Full journeys in parallel and trim their runtime Run all four agent x OS Full jobs at once instead of serializing Linux and Windows per agent. Cases share no workspace state, only inference capacity. Exclude smoke cases from Full; the smoke job already runs them on the same OS matrix. A contract test checks smoke + Full still cover every blocking live case exactly once. Launch the Codex smart-router toggle journey's parent with low reasoning effort; parent turns dominated its runtime and routing never reads it. Co-authored-by: Isaac --- .github/workflows/integration.yml | 10 ++- tests/README.md | 2 +- tests/integration/README.md | 21 +++--- .../test_ug_smart_routing_hooks.py | 18 ++++- tests/test_integration_contract.py | 69 +++++++++++++------ 5 files changed, 79 insertions(+), 41 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8d17d56fa..ccb4125a7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -311,16 +311,13 @@ jobs: if: ${{ !cancelled() && needs.workspace.result == 'success' }} needs: [workspace, smoke] runs-on: ${{ matrix.os }} - concurrency: - # Keep the previous per-agent serialization across the new OS matrix. - group: integration-full-${{ github.run_id }}-${{ matrix.agent }} - cancel-in-progress: false # The runner enforces its own 3600s pytest deadline; keep the job above it # so a slow run fails through the runner's report instead of a job kill. timeout-minutes: 70 strategy: fail-fast: false - # One runner per OS variant; concurrency above serializes variants per agent. + # One runner per agent and OS, all in parallel. Cases share no workspace state, + # only inference capacity. max-parallel: 4 matrix: os: [ubuntu-22.04, windows-latest] @@ -331,7 +328,8 @@ jobs: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} DEPENDENCY: ${{ inputs.dependency }} AGENT: ${{ matrix.agent }} - TEST_MARKER: live and ${{ matrix.agent }} and not soft_fail + # Smoke cases already ran on this OS in the smoke job; don't repeat them here. + TEST_MARKER: live and ${{ matrix.agent }} and not soft_fail and not smoke ARTIFACT_NAME: integration-full-${{ matrix.agent }}-${{ matrix.os }} steps: *live-steps diff --git a/tests/README.md b/tests/README.md index f453583df..65714d20a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -607,7 +607,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_codex_app_reports_unknown_argument` | Pass an invalid option directly to `ug codex app` with legacy smart routing off/on | Real Codex parser error and status preserved | | `test_ug_codex_app_server_client_initializes` | Connect a stdio client, direct/`--` separator, with legacy smart routing off/on | Actual JSON-RPC initialize response; no non-JSON stdout; no routing | | `test_smart_routing_claude_route_subagent_hook`, `test_smart_routing_codex_route_subagent_hook` | Pipe a native spawn payload to the installed Claude mod command or Codex PreToolUse hook with legacy subagent routing or each supported routing preset | Claude returns model/rationale/updatedInput JSON naming the routed agent the mod labels; Codex returns an allow-hook decision; requested model replaced by a routed agent definition (Claude) or bundled catalog slug (Codex) from the offered models; one audited decision matching the session and task | -| `test_smart_router_routes_{claude,codex}_subagent_at_launch`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_off`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_back_on` | Configure, launch a real subagent-only TUI with legacy routing-only/orchestration flags or the three subagent-only presets, each in its own fresh session: spawn a tagged child at launch; invoke the installed Smart Router skill to turn routing off then spawn a child; or spawn a routed child, turn routing off and spawn an unrouted child, then turn routing back on and spawn a routed child | Routing-only cases install `smart-router`; orchestration cases also install `smart-router-orchestrator`; each tagged native child completes; routed children (launch, before off and after back on) show a native model/rationale row (Claude) or subagent banner (Codex) and produce a live routing decision correlated with the child, while each child after off shows neither and has no decision; Claude's recorded task inference payloads carry `smart_router_recipe_name` as the launch recipe (`at_launch`, `back_on`, where the second child must match the launch child) or DISABLED (after off), omit the recipe header, and receive successful gateway responses; opted-in Codex banners include `[orchestrator on]`; no first-prompt routing wrapper; the customer full-mode preset is intentionally outside this journey; normal exit | +| `test_smart_router_routes_{claude,codex}_subagent_at_launch`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_off`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_back_on` | Configure, launch a real subagent-only TUI with legacy routing-only/orchestration flags or the three subagent-only presets, each in its own fresh session: spawn a tagged child at launch; invoke the installed Smart Router skill to turn routing off then spawn a child; or spawn a routed child, turn routing off and spawn an unrouted child, then turn routing back on and spawn a routed child | Routing-only cases install `smart-router`; orchestration cases also install `smart-router-orchestrator`; each tagged native child completes; routed children (launch, before off and after back on) show a native model/rationale row (Claude) or subagent banner (Codex) and produce a live routing decision correlated with the child, while each child after off shows neither and has no decision; Claude's recorded task inference payloads carry `smart_router_recipe_name` as the launch recipe (`at_launch`, `back_on`, where the second child must match the launch child) or DISABLED (after off), omit the recipe header, and receive successful gateway responses; opted-in Codex banners include `[orchestrator on]`; the Codex parent launches with `model_reasoning_effort="low"` for speed; no first-prompt routing wrapper; the customer full-mode preset is intentionally outside this journey; normal exit | | `test_ug_configure_claude_repeat_and_revert`, `test_ug_configure_codex_repeat_and_revert` | Configure twice over user settings; complete a task; revert twice | Settings preserved; no bearer in ug state; generated config removed; status unconfigured | | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | diff --git a/tests/integration/README.md b/tests/integration/README.md index 54e671721..15416ccd8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -987,13 +987,13 @@ each test; only explicit-model scenarios choose and record a discovered Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, -two agent jobs). Full runs all 88 live cases, including those smoke cases, in two -disjoint agent lanes: +two agent jobs per OS). Full runs every other live case in two disjoint agent lanes; +it excludes the smoke cases, which smoke already ran on the same OS: -| Agent lane | Marker | Cases | -| --- | --- | --- | -| Claude | `live and claude` | 40 | -| Codex | `live and codex` | 46 | +| Agent lane | Marker | +| --- | --- | +| Claude | `live and claude and not soft_fail and not smoke` | +| Codex | `live and codex and not soft_fail and not smoke` | A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with `continue-on-error` and is not part of the required `cujs` gate until it is stable. @@ -1008,11 +1008,10 @@ Each lane installs only the agent CLIs it needs (one, except the managed OpenCod three), once, and runs all its configure, headless, commands, lifecycle, and applicable app-server journeys. Cases remain serial inside each fresh VM because configure/revert can touch machine-level settings; -separate runners isolate those writes as well as the PTYs. Claude and Codex run -in parallel, with at most one full-suite job per agent in a workflow run. This -avoids six serial job startups without overlapping same-agent shards. The two -lanes still share workspace capacity with the concurrently running agent e2e -shards and other PRs; this limit does not guarantee freedom from rate limits. +separate runners isolate those writes as well as the PTYs. All four agent × OS +Full jobs run in parallel: cases share no workspace state, only inference capacity. +They share that capacity with each other, the concurrently running agent e2e shards, +and other PRs, so rate limits remain possible. No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. diff --git a/tests/integration/test_ug_smart_routing_hooks.py b/tests/integration/test_ug_smart_routing_hooks.py index 067e70476..ef1d9809b 100644 --- a/tests/integration/test_ug_smart_routing_hooks.py +++ b/tests/integration/test_ug_smart_routing_hooks.py @@ -32,6 +32,9 @@ SMART_ROUTING_SUBAGENT_NOTICE = "Using Unity Gateway Smart Router - Subagent" # No selector or legacy flag: the managed config's smart_routing.enabled alone turns routing on. MANAGED_DEFAULT = "managed-default" +# The parent only delegates and invokes the skill; its default reasoning dominates the +# journey's runtime. Routing never reads the parent's effort, so this is launch-only speed. +CODEX_PARENT_EFFORT = ["-c", 'model_reasoning_effort="low"'] # Use the same task_v3-compatible menu for direct hooks and managed TUI journeys. @@ -732,7 +735,10 @@ def test_smart_router_routes_codex_subagent_at_launch( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "smart-router-launch" + session, + "codex", + [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], + "smart-router-launch", ) as tui: tui.boot() _run_calculation(tui, session, "codex", "1+1", "2", orchestration_enabled, routed=True) @@ -773,7 +779,10 @@ def test_smart_router_skill_turns_codex_subagent_routing_off( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "smart-router-skill-off" + session, + "codex", + [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], + "smart-router-skill-off", ) as tui: tui.boot() _toggle_with_skill( @@ -819,7 +828,10 @@ def test_smart_router_skill_turns_codex_subagent_routing_back_on( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "smart-router-skill-back-on" + session, + "codex", + [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], + "smart-router-skill-back-on", ) as tui: tui.boot() _run_calculation(tui, session, "codex", "1+1", "2", orchestration_enabled, routed=True) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 01f9eb06d..ed1f8f4ca 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -104,26 +104,7 @@ def test_soft_fail_marks_only_the_affected_claude_journey(): assert marked == [("test_ug_smart_routing_hooks.py", name) for name in CLAUDE_TOGGLE_TESTS] -@pytest.mark.parametrize( - ("job_name", "agent", "retained_test"), - [ - ("smoke", "claude", "test_ug_claude_headless_prompt_argument"), - ("smoke", "codex", "test_ug_codex_headless_prompt_argument"), - ("full", "claude", "test_smart_routing_claude_route_subagent_hook"), - ("full", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), - ("managed", "claude", "test_ug_configure_managed_claude"), - ("managed", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), - ("soft-fail", "claude", None), - ], -) -def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test): - job = _integration_jobs()[job_name] - if job_name == "managed": - marker = re.search(r'-- -m "([^"]+)"', job) - else: - marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", job) - assert marker is not None - expression = marker.group(1).replace("${{ matrix.agent }}", agent).replace("$AGENT", agent) +def _collect_integration_cases(expression): result = subprocess.run( [ sys.executable, @@ -144,6 +125,54 @@ def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test assert result.returncode == 0, result.stdout + result.stderr cases = {line for line in result.stdout.splitlines() if line.startswith("test_")} assert cases, result.stdout + return cases + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_smoke_and_full_together_cover_every_blocking_live_case(agent): + jobs = _integration_jobs() + selected = [] + for name in ("smoke", "full"): + marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", jobs[name]) + assert marker is not None + selected.append( + _collect_integration_cases(marker.group(1).replace("${{ matrix.agent }}", agent)) + ) + smoke, full = selected + # Full skips smoke cases only because smoke runs them on the same OS matrix. + assert "os: [ubuntu-22.04, windows-latest]" in jobs["smoke"] + assert "os: [ubuntu-22.04, windows-latest]" in jobs["full"] + assert not smoke & full + assert smoke | full == _collect_integration_cases(f"live and {agent} and not soft_fail") + + +def test_full_journeys_run_every_agent_and_os_in_parallel(): + job = _integration_jobs()["full"] + assert "concurrency:" not in job + assert "max-parallel: 4" in job + + +@pytest.mark.parametrize( + ("job_name", "agent", "retained_test"), + [ + ("smoke", "claude", "test_ug_claude_headless_prompt_argument"), + ("smoke", "codex", "test_ug_codex_headless_prompt_argument"), + ("full", "claude", "test_smart_routing_claude_route_subagent_hook"), + ("full", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), + ("managed", "claude", "test_ug_configure_managed_claude"), + ("managed", "codex", "test_smart_router_skill_turns_codex_subagent_routing_back_on"), + ("soft-fail", "claude", None), + ], +) +def test_soft_fail_ci_selectors_preserve_coverage(job_name, agent, retained_test): + job = _integration_jobs()[job_name] + if job_name == "managed": + marker = re.search(r'-- -m "([^"]+)"', job) + else: + marker = re.search(r"(?m)^ TEST_MARKER: (.+)$", job) + assert marker is not None + expression = marker.group(1).replace("${{ matrix.agent }}", agent).replace("$AGENT", agent) + cases = _collect_integration_cases(expression) if retained_test is not None: assert any(f"::{retained_test}" in case for case in cases), cases assert not any(f"::{toggle}[" in case for case in cases for toggle in CLAUDE_TOGGLE_TESTS) From a13e9a42ed54770cf02fca9462a8f538f442d3ba Mon Sep 17 00:00:00 2001 From: Rohit Agrawal Date: Wed, 28 Oct 2026 14:44:42 -0400 Subject: [PATCH 2/3] Run Smoke journeys in parallel across OSes too Smoke had the same per-agent concurrency group as Full, so each agent's Linux job waited for its Windows job. Drop it for the same reason: cases share no workspace state, only inference capacity. Co-authored-by: Isaac --- .github/workflows/integration.yml | 4 ---- tests/test_integration_contract.py | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ccb4125a7..126db3e30 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -171,10 +171,6 @@ jobs: needs: workspace runs-on: ${{ matrix.os }} timeout-minutes: 20 - concurrency: - # Both OS variants use the same workspace; avoid overlapping same-agent runs. - group: integration-smoke-${{ github.run_id }}-${{ matrix.agent }} - cancel-in-progress: false strategy: fail-fast: false matrix: diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index ed1f8f4ca..0b654c838 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -146,10 +146,11 @@ def test_smoke_and_full_together_cover_every_blocking_live_case(agent): assert smoke | full == _collect_integration_cases(f"live and {agent} and not soft_fail") -def test_full_journeys_run_every_agent_and_os_in_parallel(): - job = _integration_jobs()["full"] - assert "concurrency:" not in job - assert "max-parallel: 4" in job +def test_smoke_and_full_journeys_run_every_agent_and_os_in_parallel(): + jobs = _integration_jobs() + assert "concurrency:" not in jobs["smoke"] + assert "concurrency:" not in jobs["full"] + assert "max-parallel: 4" in jobs["full"] @pytest.mark.parametrize( From b0199ee13a322a5c077571f293177348a528e56f Mon Sep 17 00:00:00 2001 From: Rohit Agrawal Date: Wed, 28 Oct 2026 15:19:36 -0400 Subject: [PATCH 3/3] Revert low reasoning effort for the Codex Smart Router journeys With `-c model_reasoning_effort="low"` forwarded to the parent, Codex still spawned the child but the Smart Router subagent banner never appeared, so all 18 Codex TUI journeys timed out in Managed config and Full. main passes them without the flag. Co-authored-by: Isaac --- tests/README.md | 2 +- .../integration/test_ug_smart_routing_hooks.py | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/README.md b/tests/README.md index 65714d20a..f453583df 100644 --- a/tests/README.md +++ b/tests/README.md @@ -607,7 +607,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_codex_app_reports_unknown_argument` | Pass an invalid option directly to `ug codex app` with legacy smart routing off/on | Real Codex parser error and status preserved | | `test_ug_codex_app_server_client_initializes` | Connect a stdio client, direct/`--` separator, with legacy smart routing off/on | Actual JSON-RPC initialize response; no non-JSON stdout; no routing | | `test_smart_routing_claude_route_subagent_hook`, `test_smart_routing_codex_route_subagent_hook` | Pipe a native spawn payload to the installed Claude mod command or Codex PreToolUse hook with legacy subagent routing or each supported routing preset | Claude returns model/rationale/updatedInput JSON naming the routed agent the mod labels; Codex returns an allow-hook decision; requested model replaced by a routed agent definition (Claude) or bundled catalog slug (Codex) from the offered models; one audited decision matching the session and task | -| `test_smart_router_routes_{claude,codex}_subagent_at_launch`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_off`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_back_on` | Configure, launch a real subagent-only TUI with legacy routing-only/orchestration flags or the three subagent-only presets, each in its own fresh session: spawn a tagged child at launch; invoke the installed Smart Router skill to turn routing off then spawn a child; or spawn a routed child, turn routing off and spawn an unrouted child, then turn routing back on and spawn a routed child | Routing-only cases install `smart-router`; orchestration cases also install `smart-router-orchestrator`; each tagged native child completes; routed children (launch, before off and after back on) show a native model/rationale row (Claude) or subagent banner (Codex) and produce a live routing decision correlated with the child, while each child after off shows neither and has no decision; Claude's recorded task inference payloads carry `smart_router_recipe_name` as the launch recipe (`at_launch`, `back_on`, where the second child must match the launch child) or DISABLED (after off), omit the recipe header, and receive successful gateway responses; opted-in Codex banners include `[orchestrator on]`; the Codex parent launches with `model_reasoning_effort="low"` for speed; no first-prompt routing wrapper; the customer full-mode preset is intentionally outside this journey; normal exit | +| `test_smart_router_routes_{claude,codex}_subagent_at_launch`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_off`, `test_smart_router_skill_turns_{claude,codex}_subagent_routing_back_on` | Configure, launch a real subagent-only TUI with legacy routing-only/orchestration flags or the three subagent-only presets, each in its own fresh session: spawn a tagged child at launch; invoke the installed Smart Router skill to turn routing off then spawn a child; or spawn a routed child, turn routing off and spawn an unrouted child, then turn routing back on and spawn a routed child | Routing-only cases install `smart-router`; orchestration cases also install `smart-router-orchestrator`; each tagged native child completes; routed children (launch, before off and after back on) show a native model/rationale row (Claude) or subagent banner (Codex) and produce a live routing decision correlated with the child, while each child after off shows neither and has no decision; Claude's recorded task inference payloads carry `smart_router_recipe_name` as the launch recipe (`at_launch`, `back_on`, where the second child must match the launch child) or DISABLED (after off), omit the recipe header, and receive successful gateway responses; opted-in Codex banners include `[orchestrator on]`; no first-prompt routing wrapper; the customer full-mode preset is intentionally outside this journey; normal exit | | `test_ug_configure_claude_repeat_and_revert`, `test_ug_configure_codex_repeat_and_revert` | Configure twice over user settings; complete a task; revert twice | Settings preserved; no bearer in ug state; generated config removed; status unconfigured | | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | diff --git a/tests/integration/test_ug_smart_routing_hooks.py b/tests/integration/test_ug_smart_routing_hooks.py index ef1d9809b..067e70476 100644 --- a/tests/integration/test_ug_smart_routing_hooks.py +++ b/tests/integration/test_ug_smart_routing_hooks.py @@ -32,9 +32,6 @@ SMART_ROUTING_SUBAGENT_NOTICE = "Using Unity Gateway Smart Router - Subagent" # No selector or legacy flag: the managed config's smart_routing.enabled alone turns routing on. MANAGED_DEFAULT = "managed-default" -# The parent only delegates and invokes the skill; its default reasoning dominates the -# journey's runtime. Routing never reads the parent's effort, so this is launch-only speed. -CODEX_PARENT_EFFORT = ["-c", 'model_reasoning_effort="low"'] # Use the same task_v3-compatible menu for direct hooks and managed TUI journeys. @@ -735,10 +732,7 @@ def test_smart_router_routes_codex_subagent_at_launch( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, - "codex", - [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], - "smart-router-launch", + session, "codex", [str(session.binary), "codex"], "smart-router-launch" ) as tui: tui.boot() _run_calculation(tui, session, "codex", "1+1", "2", orchestration_enabled, routed=True) @@ -779,10 +773,7 @@ def test_smart_router_skill_turns_codex_subagent_routing_off( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, - "codex", - [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], - "smart-router-skill-off", + session, "codex", [str(session.binary), "codex"], "smart-router-skill-off" ) as tui: tui.boot() _toggle_with_skill( @@ -828,10 +819,7 @@ def test_smart_router_skill_turns_codex_subagent_routing_back_on( "--disable-databricks-ai-tools", ) with AgentTerminal( - session, - "codex", - [str(session.binary), "codex", "--", *CODEX_PARENT_EFFORT], - "smart-router-skill-back-on", + session, "codex", [str(session.binary), "codex"], "smart-router-skill-back-on" ) as tui: tui.boot() _run_calculation(tui, session, "codex", "1+1", "2", orchestration_enabled, routed=True)