diff --git a/src/ucode/smart_routing/routing.py b/src/ucode/smart_routing/routing.py index 6286c767d..2356fe881 100644 --- a/src/ucode/smart_routing/routing.py +++ b/src/ucode/smart_routing/routing.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import logging import os import re import textwrap @@ -309,8 +310,11 @@ def resolve_spawn_route( ), default_task_label, ) - decision, _ = decision_fn(task) + decision, reason = decision_fn(task) if decision is None: + logging.getLogger(__name__).warning( + "Smart Router could not route subagent: %s", reason or "no routing decision" + ) return None routed_model = model_id_mapper(decision.model) return SpawnRoute(tool_input, task, decision, routed_model) diff --git a/tests/README.md b/tests/README.md index 01c727a6f..2496c0baa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -741,3 +741,27 @@ Codex retains its CLI control-file checks. No extra TUI sessions or tasks are ad Clear/reload, compaction, and denied-write rollback are not independently checked. A native process check runs the real disabled routing CLI from Claude with inherited launch flags; it makes no gateway request. + +Subagent-routing failures have shared warning coverage in `test_codex_routing.py`; +the live hook journeys retain their routing assertions and include stderr when no +decision is returned. This diagnostic coverage does not prove routing service health. + +Smart-routing scenarios use checked-in model menus assigned to the explicit CI +workspace. US `task_v3` uses GPT-6 Sol / Luna; the Canadian managed workspace uses +GPT-5.6 Sol / Terra / Luna, since it has no GPT-6 services. Unknown Codex workspaces +fail until a fixture is assigned. Claude uses Opus 4.8 / Sonnet 4.6 / Haiku 4.5. +Direct hook inputs share the US fixture; managed TUI inputs use their workspace's +fixture. No workspace is auto-selected or modified. All real routing, native notice, +audit correlation, and child completion assertions remain required. + +Routing-toggle journeys wait for the child's own persisted assistant answer, in addition +to the parent answer. A parent reporting the expected value while the child is still +running cannot satisfy child completion. The wait is bounded and still fails on timeout. + +Codex routing-display unit coverage accepts a generic child label only after the tagged +prompt and before the next prompt, requires the expected orchestration state, and +rejects stale or unrelated panels. Live audit correlation and child answers remain required. + +After Codex compaction removes a prompt, display correlation may instead use the +native Spawned row immediately following the routing box. Unit checks reject later +answer text, intervening prompts, and nonadjacent or unrelated spawn rows. diff --git a/tests/fixtures/managed_config/claude_smart_routing.json b/tests/fixtures/managed_config/claude_smart_routing.json index dbb0bfb36..da17c57a6 100644 --- a/tests/fixtures/managed_config/claude_smart_routing.json +++ b/tests/fixtures/managed_config/claude_smart_routing.json @@ -7,13 +7,13 @@ "config": { "models": { "model_services": [ - "system.ai.claude-sonnet-5", - "system.ai.claude-haiku-4-5", - "system.ai.claude-opus-4-8" + "system.ai.claude-opus-4-8", + "system.ai.claude-sonnet-4-6", + "system.ai.claude-haiku-4-5" ] }, "default_models": { - "default_model": "system.ai.claude-sonnet-5" + "default_model": "system.ai.claude-sonnet-4-6" }, "smart_routing": { "enabled": true diff --git a/tests/fixtures/managed_config/codex_smart_routing.json b/tests/fixtures/managed_config/codex_smart_routing.json index 32d41ff1a..728bfe519 100644 --- a/tests/fixtures/managed_config/codex_smart_routing.json +++ b/tests/fixtures/managed_config/codex_smart_routing.json @@ -7,13 +7,12 @@ "config": { "models": { "model_services": [ - "system.ai.gpt-5-6-sol", - "system.ai.gpt-5-6-terra", - "system.ai.gpt-5-6-luna" + "system.ai.gpt-6-sol", + "system.ai.gpt-6-luna" ] }, "default_models": { - "default_model": "system.ai.gpt-5-6-sol" + "default_model": "system.ai.gpt-6-sol" }, "smart_routing": { "enabled": true diff --git a/tests/fixtures/managed_config/codex_smart_routing_ca.json b/tests/fixtures/managed_config/codex_smart_routing_ca.json new file mode 100644 index 000000000..32d41ff1a --- /dev/null +++ b/tests/fixtures/managed_config/codex_smart_routing_ca.json @@ -0,0 +1,24 @@ +{ + "spec_version": 1, + "default_agent": "CODING_AGENT_CODEX", + "enabled_agents": [ + { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": { + "model_services": [ + "system.ai.gpt-5-6-sol", + "system.ai.gpt-5-6-terra", + "system.ai.gpt-5-6-luna" + ] + }, + "default_models": { + "default_model": "system.ai.gpt-5-6-sol" + }, + "smart_routing": { + "enabled": true + } + } + } + ] +} diff --git a/tests/integration/README.md b/tests/integration/README.md index 88658dd34..0bfff317d 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1248,3 +1248,32 @@ recipe change does not require editing the test. Local Claude mod confirmation replaces the old saved-control-file/assistant-turn wait; Codex retains its CLI control-file checks. No extra TUI sessions or tasks are added. Clear/reload, compaction, and denied-write rollback are not independently checked. + +Subagent-routing failures have shared warning coverage in `test_codex_routing.py`; +the live hook journeys retain their routing assertions and include stderr when no +decision is returned. This diagnostic coverage does not prove routing service health. + +Smart-routing scenarios use checked-in model menus assigned to the explicit CI +workspace. US `task_v3` uses GPT-6 Sol / Luna; the Canadian managed workspace uses +GPT-5.6 Sol / Terra / Luna, since it has no GPT-6 services. Unknown Codex workspaces +fail until a fixture is assigned. Claude uses Opus 4.8 / Sonnet 4.6 / Haiku 4.5. +Direct hook inputs share the US fixture; managed TUI inputs use their workspace's +fixture. No workspace is auto-selected or modified. All real routing, native notice, +audit correlation, and child completion assertions remain required. + +Routing-toggle journeys wait for the child's own persisted assistant answer, in addition +to the parent answer. A parent reporting the expected value while the child is still +running cannot satisfy child completion. The wait is bounded and still fails on timeout. + +Codex routing-display unit coverage accepts a generic child label only after the tagged +prompt and before the next prompt, requires the expected orchestration state, and +rejects stale or unrelated panels. Live audit correlation and child answers remain required. + +After Codex compaction removes a prompt, display correlation may instead use the +native Spawned row immediately following the routing box. Unit checks reject later +answer text, intervening prompts, and nonadjacent or unrelated spawn rows. + +For a managed workspace other than the two CI workspaces, set +`UCODE_TEST_CODEX_ROUTING_FIXTURE` to the basename of a checked-in fixture whose model menu has +been validated in that workspace. The suite never guesses a model menu or silently changes the +workspace. diff --git a/tests/integration/test_ug_configure_managed_models.py b/tests/integration/test_ug_configure_managed_models.py index ca65c0cd9..1f0c766d1 100644 --- a/tests/integration/test_ug_configure_managed_models.py +++ b/tests/integration/test_ug_configure_managed_models.py @@ -11,7 +11,7 @@ import pytest from utils.constants import MANAGED_CLAUDE_PROVIDER_SERVICE from utils.evidence import FileTask -from utils.managed import use_managed_config_fixture +from utils.managed import use_managed_config_fixture, use_smart_routing_fixture from utils.provider_catalog import fetch_anthropic_parent_catalog, fetch_anthropic_provider_catalog from utils.terminal import AgentTerminal, TerminalProcess @@ -245,7 +245,7 @@ def test_managed_fixture_claude_smart_routing_banner( """ session = live_session task = FileTask(session) - use_managed_config_fixture(session, "claude_smart_routing") + use_smart_routing_fixture(session, "claude", workspace) result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) assert "Select coding agents to configure:" not in result.stdout, result.stdout if SMART_ROUTER_CONFIG_VERSION is not None: @@ -291,7 +291,7 @@ def test_managed_fixture_codex_smart_routing_banner( """ session = live_session task = FileTask(session) - use_managed_config_fixture(session, "codex_smart_routing") + use_smart_routing_fixture(session, "codex", workspace) result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) assert "Select coding agents to configure:" not in result.stdout, result.stdout if SMART_ROUTER_CONFIG_VERSION is not None: diff --git a/tests/integration/test_ug_smart_routing_hooks.py b/tests/integration/test_ug_smart_routing_hooks.py index b20a928bc..aec370358 100644 --- a/tests/integration/test_ug_smart_routing_hooks.py +++ b/tests/integration/test_ug_smart_routing_hooks.py @@ -17,11 +17,12 @@ agent_sessions, assert_subagent_routed, assistant_answers, + codex_routing_display_for_task, is_child_session, read_jsonl, tool_outputs, ) -from utils.managed import use_managed_config_fixture +from utils.managed import MANAGED_CONFIG_FIXTURES, use_smart_routing_fixture from utils.request_recorder import TuiRequestRecorder, served_inference_request from utils.terminal import AgentTerminal @@ -29,33 +30,19 @@ SMART_ROUTING_BANNER = "Using Unity Gateway Smart Router." SMART_ROUTING_SUBAGENT_NOTICE = "Using Unity Gateway Smart Router - Subagent" -CLAUDE_MODELS = [ - "system.ai.claude-opus-5", - "system.ai.claude-sonnet-5", - "system.ai.claude-haiku-4-5", - "system.ai.glm-5-3", - "system.ai.kimi-k3", -] -CODEX_MODELS = [ - "system.ai.gpt-6-astra", - "system.ai.gpt-5-6-sol", - "system.ai.gpt-5-6-terra", - "system.ai.gpt-5-6-luna", - "system.ai.gpt-5-5", - "system.ai.glm-5-3", - "system.ai.kimi-k3", -] -# Codex launches subagents on its bundled catalog slugs, not the workspace model id, so -# the routed model in the hook response must be one of these slugs. -CODEX_MODEL_SLUGS = { - "gpt-6-astra", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5", - "glm-5-3", - "kimi-k3", -} + + +# Use the same task_v3-compatible menu for direct hooks and managed TUI journeys. +# Direct hooks run in the US workspace; managed TUI inputs are assigned by workspace. +def _fixture_routing_models(agent: str) -> list[str]: + config = json.loads((MANAGED_CONFIG_FIXTURES / f"{agent}_smart_routing.json").read_text()) + return config["enabled_agents"][0]["config"]["models"]["model_services"] + + +CLAUDE_MODELS = _fixture_routing_models("claude") +CODEX_MODELS = _fixture_routing_models("codex") +# Expected native Codex slugs, independently of the application's model mapper. +CODEX_MODEL_SLUGS = {"gpt-6-sol", "gpt-6-luna"} SKILL_ROOTS = {"claude": ".claude/skills", "codex": ".codex/skills"} @@ -91,20 +78,7 @@ def _routing_display_for_task( lines = screen.splitlines() if agent == "claude": return _claude_routing_row_for_task(lines, marker) - for index, line in enumerate(lines): - if SMART_ROUTING_SUBAGENT_NOTICE not in line: - continue - panel = [] - for panel_line in lines[index : index + 12]: - panel.append(panel_line) - if "└" in panel_line: - break - # Rich can wrap the marker between any two characters in a narrow TUI. - # Compare without rendered whitespace so the banner remains attributable. - text = "".join("\n".join(panel).split()) - if marker in text and (not orchestrator_on or "[orchestratoron]" in text): - return True - return False + return codex_routing_display_for_task(screen, marker, orchestrator_on=orchestrator_on) def _run_calculation( @@ -133,6 +107,13 @@ def _run_calculation( timeout=120, ) tui.wait_for_task(task, timeout=180) + # A parent may report the requested value while its background child is still + # running. Require the child's own persisted answer before asserting completion. + tui.wait_for( + lambda _: task.completed(session, agent, child=True), + f"the child assistant answer for {task.marker}", + timeout=180, + ) task.assert_completed(session, agent) task.assert_completed(session, agent, child=True) @@ -302,6 +283,7 @@ def test_smart_routing_claude_route_subagent_hook( input_text=json.dumps(payload), timeout=60, ) + assert result.stdout.strip(), f"Routing hook returned no decision: {result.stderr}" output = json.loads(result.stdout) assert output["model"] in CLAUDE_MODELS, output assert isinstance(output["rationale"], str), output @@ -379,6 +361,7 @@ def test_smart_routing_codex_route_subagent_hook( input_text=json.dumps(payload), timeout=60, ) + assert result.stdout.strip(), f"Routing hook returned no decision: {result.stderr}" output = json.loads(result.stdout) hook = output["hookSpecificOutput"] assert hook["hookEventName"] == "PreToolUse", output @@ -446,7 +429,7 @@ def test_smart_router_skill_toggles_claude_subagent_routing( session.env["ENABLE_SMART_ROUTER_ORCHESTRATOR"] = "1" else: session.env["SMART_ROUTER_CONFIG_VERSION"] = SMART_ROUTER_CONFIG_VERSION - use_managed_config_fixture(session, "claude_smart_routing") + use_smart_routing_fixture(session, "claude", workspace) with TuiRequestRecorder(workspace) as recorder: # Configure through the forwarding URL using the public CLI, keeping ug state real. session.run( @@ -581,7 +564,7 @@ def test_smart_router_skill_toggles_codex_subagent_routing( session.env["ENABLE_SMART_ROUTER_ORCHESTRATOR"] = "1" else: session.env["SMART_ROUTER_CONFIG_VERSION"] = SMART_ROUTER_CONFIG_VERSION - use_managed_config_fixture(session, "codex_smart_routing") + use_smart_routing_fixture(session, "codex", workspace) session.run( "configure", "--workspace", diff --git a/tests/integration/utils/evidence.py b/tests/integration/utils/evidence.py index 0e6e32c55..ab5ad1f0a 100644 --- a/tests/integration/utils/evidence.py +++ b/tests/integration/utils/evidence.py @@ -361,3 +361,45 @@ def assert_subagent_routed( # Some agent versions omit the child's model from SubagentStart. The report # preserves that unknown value; this test claims decision + spawn + task, # not model-identity verification when the agent did not expose it. + + +def codex_routing_display_for_task(screen: str, marker: str, *, orchestrator_on: bool) -> bool: + """Match a routing box to its tagged prompt or immediately following native spawn. + + Compaction can remove the prompt; the adjacent Spawned row still identifies + the child. Live audit and child-answer assertions verify the actual decision. + """ + lines = screen.splitlines() + prompts = [index for index, line in enumerate(lines) if line.lstrip().startswith("›")] + for index, line in enumerate(lines): + if "Using Unity Gateway Smart Router - Subagent" not in line: + continue + end = index + 1 + while end < min(index + 12, len(lines)) and "└" not in lines[end]: + end += 1 + if end >= len(lines) or "└" not in lines[end]: + continue + text = "".join("".join(lines[index : end + 1]).split()) + if "SelectedModel:" not in text or ("[orchestratoron]" in text) != orchestrator_on: + continue + preceding = [start for start in prompts if start < index] + if preceding: + start = preceding[-1] + prompt_end = start + 1 + while prompt_end < index and lines[prompt_end].strip(): + prompt_end += 1 + if marker in "".join("".join(lines[start:prompt_end]).split()): + return True + # A compacted prompt is unavailable, but its native spawn remains directly + # after the routing box. Do not accept later echoed answers or tool results. + spawn = end + 1 + while spawn < len(lines) and not lines[spawn].strip(): + spawn += 1 + if spawn >= len(lines) or not lines[spawn].lstrip().startswith("• Spawned "): + continue + spawn_end = spawn + 1 + while spawn_end < len(lines) and lines[spawn_end].strip(): + spawn_end += 1 + if marker in "".join("".join(lines[spawn:spawn_end]).split()): + return True + return False diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py index 2eec1ac78..e9bf94c79 100644 --- a/tests/integration/utils/managed.py +++ b/tests/integration/utils/managed.py @@ -6,7 +6,9 @@ enforces that boundary). """ +import os from pathlib import Path +from urllib.parse import urlparse MANAGED_CONFIGS_PATH = "/api/ai-gateway/v2/coding-agent-configs" # One JSON CodingAgentConfig per scenario, in the GET shape `ug configure --file` will accept. @@ -38,3 +40,28 @@ def assert_no_managed_config(payload: object) -> None: def is_managed_config_control_plane_cache(home: Path, path: Path) -> bool: """Whether ``path`` is ug's expected fetched-config cache, not agent-owned state.""" return path == home / ".ucode" / "managed-config.json" + + +def use_smart_routing_fixture(session, agent: str, workspace: str) -> Path: + """Select the checked-in scenario input for the two configured CI workspaces. + + The regions publish different GPT generations. This chooses no workspace and + retries no request; unsupported workspaces need an explicit fixture assignment. + """ + if agent == "claude": + name = "claude_smart_routing" + elif agent == "codex": + override = os.environ.get("UCODE_TEST_CODEX_ROUTING_FIXTURE", "").strip() + fixtures = { + "eng-ml-inference-team-us-east-1.cloud.databricks.com": "codex_smart_routing", + "eng-ml-inference-team-ca-central-1.cloud.databricks.com": "codex_smart_routing_ca", + } + hostname = urlparse(workspace).hostname + name = override or fixtures.get(hostname) + assert name, ( + f"No Codex routing fixture assigned to {hostname}; set " + "UCODE_TEST_CODEX_ROUTING_FIXTURE to a fixture basename validated for that workspace" + ) + else: + raise AssertionError(f"No routing fixture for {agent}") + return use_managed_config_fixture(session, name) diff --git a/tests/test_codex_routing.py b/tests/test_codex_routing.py index 2cda5c09c..ea3668f96 100644 --- a/tests/test_codex_routing.py +++ b/tests/test_codex_routing.py @@ -504,3 +504,21 @@ def test_decision_record_persists_rationale(tmp_path, monkeypatch): # returned none" from a display-placement bug. record = json.loads(decisions.read_text().strip()) assert record["rationale"] == "Cross-cutting refactor needs the strongest model." + + +def test_failed_subagent_routing_reports_reason_without_hook_response(monkeypatch, caplog): + monkeypatch.setattr( + codex_routing, + "request_routing_decision", + lambda *args, **kwargs: (None, "router returned HTTP 400: unsupported model"), + ) + output = codex_routing.route_pre_tool_use( + {"tool_name": "spawn_agent", "tool_input": {"message": "Review the parser"}}, + workspace=WS, + token="private-bearer", + available_models=["system.ai.gpt-5-6-sol"], + ) + assert output is None + assert "router returned HTTP 400: unsupported model" in caplog.text + assert "private-bearer" not in caplog.text + assert "Review the parser" not in caplog.text diff --git a/tests/test_integration_evidence.py b/tests/test_integration_evidence.py index e0c22846d..dd5626706 100644 --- a/tests/test_integration_evidence.py +++ b/tests/test_integration_evidence.py @@ -304,3 +304,59 @@ def test_tool_outputs_exclude_user_echoes_and_assistant_claims(agent): }, ] assert tool_outputs(agent, records) == [] + + +@pytest.mark.parametrize("orchestrator_on", [False, True]) +def test_codex_generic_routing_label_correlates_to_tagged_prompt(orchestrator_on): + suffix = " [orchestrator on]" if orchestrator_on else "" + panel = ( + "│ Using Unity Gateway Smart Router - Subagent │\n" + f"│ Subagent : subagent{suffix} │\n" + "│ Selected Model : gpt-6-luna │\n└──────────┘\n" + ) + prompt = '› Spawn a child with task name "ug_subagent_ab12"\n and return its answer.\n\n' + screen = prompt + panel + "› Ask Codex to do anything\n" + assert evidence.codex_routing_display_for_task( + screen, "ug_subagent_ab12", orchestrator_on=orchestrator_on + ) + assert not evidence.codex_routing_display_for_task( + screen, "ug_subagent_other", orchestrator_on=orchestrator_on + ) + assert not evidence.codex_routing_display_for_task( + panel + prompt, "ug_subagent_ab12", orchestrator_on=orchestrator_on + ) + assert not evidence.codex_routing_display_for_task( + prompt + "› Another task\n\n" + panel, + "ug_subagent_ab12", + orchestrator_on=orchestrator_on, + ) + assert not evidence.codex_routing_display_for_task( + screen, "ug_subagent_ab12", orchestrator_on=not orchestrator_on + ) + + +def test_codex_compacted_prompt_uses_adjacent_native_spawn(): + panel = ( + "• Context compacted\n\n" + "│ Using Unity Gateway Smart Router - Subagent │\n" + "│ Subagent : subagent [orchestrator on] │\n" + "│ Selected Model : gpt-6-luna │\n└──────────┘\n\n" + ) + spawn = "• Spawned child-id (gpt-6-luna)\n └ Task name: ug_subagent_ab12. Calculate 2+2.\n\n" + assert evidence.codex_routing_display_for_task( + panel + spawn, "ug_subagent_ab12", orchestrator_on=True + ) + assert not evidence.codex_routing_display_for_task( + panel + spawn, "ug_subagent_other", orchestrator_on=True + ) + assert not evidence.codex_routing_display_for_task( + panel + "• Answer: ug_subagent_ab12=4\n", "ug_subagent_ab12", orchestrator_on=True + ) + assert not evidence.codex_routing_display_for_task( + panel + "› New prompt\n\n" + spawn, "ug_subagent_ab12", orchestrator_on=True + ) + assert not evidence.codex_routing_display_for_task( + panel + "• Spawned other-child\n └ Another task\n\n" + spawn, + "ug_subagent_ab12", + orchestrator_on=True, + )