diff --git a/apps/backend/config/model_routing.yaml b/apps/backend/config/model_routing.yaml index afa3f53f5..e99b28e20 100644 --- a/apps/backend/config/model_routing.yaml +++ b/apps/backend/config/model_routing.yaml @@ -43,9 +43,13 @@ routing: provider: openai model: gpt-4o max_tokens: 8192 + # Trivial subtasks route to Haiku by default: it stays inside the Claude + # runtime (works under AUTO_CODE_AUTONOMY=claude, no second provider needed) + # and is far cheaper than the Sonnet fallback. Override to gpt-4o-mini via + # MODEL_ROUTER_LOW_PROVIDER=openai + MODEL_ROUTER_LOW_MODEL=gpt-4o-mini. low: - provider: openai - model: gpt-4o-mini + provider: claude + model: claude-haiku-4-5-20251001 max_tokens: 4096 # Default estimated tokens for cost calculation diff --git a/apps/backend/core/providers/task_router.py b/apps/backend/core/providers/task_router.py index 744693bf2..917cd6790 100644 --- a/apps/backend/core/providers/task_router.py +++ b/apps/backend/core/providers/task_router.py @@ -21,6 +21,16 @@ logger = logging.getLogger(__name__) +# Cheap "Haiku-equivalent" model per provider. Used when the low (trivial) +# tier can't reach its configured provider and must fall back: without this, +# the fallback would use the provider's *default* (expensive) model, silently +# upgrading trivial subtasks and inverting the low < medium cost ordering. +CHEAP_MODEL_BY_PROVIDER: dict[str, str] = { + "claude": "claude-haiku-4-5-20251001", + "openai": "gpt-4o-mini", + "google": "gemini-1.5-flash", +} + @dataclass class TaskRoute: @@ -175,7 +185,12 @@ def _resolve_available_route_config( ) return configured_route - fallback_model = provider_config.get_model_for(fallback_provider) + # For the cheap (low) tier, keep the fallback cheap: use the + # provider's Haiku-equivalent rather than its expensive default model. + if complexity == "low" and fallback_provider in CHEAP_MODEL_BY_PROVIDER: + fallback_model = CHEAP_MODEL_BY_PROVIDER[fallback_provider] + else: + fallback_model = provider_config.get_model_for(fallback_provider) if not fallback_model: configured_route["fallback_reason"] = ( f"configured provider {configured_provider} is {unavailable_reason}; " diff --git a/apps/backend/core/providers/task_router_config.py b/apps/backend/core/providers/task_router_config.py index 263cc1808..a33cfc97d 100644 --- a/apps/backend/core/providers/task_router_config.py +++ b/apps/backend/core/providers/task_router_config.py @@ -73,9 +73,12 @@ "model": "gpt-4o", "max_tokens": 8192, }, + # Trivial subtasks default to Haiku (a cheap Claude model that works + # under the default claude-only runtime); override to gpt-4o-mini via + # MODEL_ROUTER_LOW_* env vars or the routing YAML. "low": { - "provider": "openai", - "model": "gpt-4o-mini", + "provider": "claude", + "model": "claude-haiku-4-5-20251001", "max_tokens": 4096, }, }, diff --git a/tests/test_task_router.py b/tests/test_task_router.py index d90982697..e9cfc4af2 100644 --- a/tests/test_task_router.py +++ b/tests/test_task_router.py @@ -130,8 +130,8 @@ def test_route_maps_score_to_configured_low_medium_high_models(): ) assert low_route.complexity == "low" - assert low_route.provider == "openai" - assert low_route.model == "gpt-4o-mini" + assert low_route.provider == "claude" + assert low_route.model == "claude-haiku-4-5-20251001" assert medium_route.complexity == "medium" assert medium_route.model == "gpt-4o" assert high_route.complexity == "high" @@ -145,8 +145,9 @@ def test_route_maps_score_to_configured_low_medium_high_models(): ) -def test_route_falls_back_to_available_provider_when_route_provider_unavailable(): - """Routing should not select OpenAI when only Claude auth is configured.""" +def test_low_complexity_routes_to_haiku_under_claude_only(): + """Trivial subtasks route to Haiku, which needs no fallback under a + claude-only runtime — the P5.T5 cost-saving default.""" router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml")) router.risk_analyzer = MagicMock() router.risk_analyzer.analyze_subtask_risks.return_value = [] @@ -158,10 +159,61 @@ def test_route_falls_back_to_available_provider_when_route_provider_unavailable( anthropic_api_key="test-anthropic-key", claude_model="claude-sonnet-4-5-20250929", ), + allowed_providers={"claude"}, ) assert route.complexity == "low" assert route.provider == "claude" + assert route.model == "claude-haiku-4-5-20251001" + # Haiku is already claude, so no fallback/compat downgrade fires. + assert "unavailable" not in route.reasoning + assert "not compatible" not in route.reasoning + + +def test_low_complexity_fallback_uses_cheap_model_not_provider_default(): + """When the low tier can't use Claude, the fallback must pick a CHEAP model + of the available provider (gpt-4o-mini), not that provider's expensive + default — otherwise trivial subtasks silently upgrade and low >= medium.""" + router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml")) + router.risk_analyzer = MagicMock() + router.risk_analyzer.analyze_subtask_risks.return_value = [] + + route = router.route( + {"description": "Fix typo", "files_to_modify": []}, + provider_config=ProviderConfig( + provider="openai", + openai_api_key="test-openai-key", + anthropic_api_key="test-anthropic-key", # Claude available, but... + ), + allowed_providers={"openai"}, # ...the runtime only allows OpenAI. + ) + + assert route.complexity == "low" + assert route.provider == "openai" + assert route.model == "gpt-4o-mini" + assert "not compatible" in route.reasoning + + +def test_route_falls_back_to_available_provider_when_route_provider_unavailable(): + """A medium (OpenAI) route falls back to Claude when only Claude auth is set.""" + router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml")) + router.risk_analyzer = MagicMock() + router.risk_analyzer.analyze_subtask_risks.return_value = [_issue("medium")] * 2 + + route = router.route( + { + "description": "Add pagination endpoint", + "files_to_modify": ["api/users.py", "tests/test_users.py"], + }, + provider_config=ProviderConfig( + provider="claude", + anthropic_api_key="test-anthropic-key", + claude_model="claude-sonnet-4-5-20250929", + ), + ) + + assert route.complexity == "medium" + assert route.provider == "claude" assert route.model == "claude-sonnet-4-5-20250929" assert "configured provider openai is unavailable" in route.reasoning @@ -170,10 +222,13 @@ def test_route_respects_runtime_provider_allowlist(): """Runtime compatibility should override an otherwise available provider.""" router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml")) router.risk_analyzer = MagicMock() - router.risk_analyzer.analyze_subtask_risks.return_value = [] + router.risk_analyzer.analyze_subtask_risks.return_value = [_issue("medium")] * 2 route = router.route( - {"description": "Fix typo", "files_to_modify": []}, + { + "description": "Add pagination endpoint", + "files_to_modify": ["api/users.py", "tests/test_users.py"], + }, provider_config=ProviderConfig( provider="openai", anthropic_api_key="test-anthropic-key", @@ -183,7 +238,7 @@ def test_route_respects_runtime_provider_allowlist(): allowed_providers={"claude"}, ) - assert route.complexity == "low" + assert route.complexity == "medium" assert route.provider == "claude" assert route.model == "claude-sonnet-4-5-20250929" assert "not compatible with the selected runtime" in route.reasoning @@ -227,7 +282,8 @@ def test_missing_yaml_uses_defaults(): assert config["routing"]["high"]["provider"] == "claude" assert config["routing"]["medium"]["model"] == "gpt-4o" - assert config["routing"]["low"]["model"] == "gpt-4o-mini" + assert config["routing"]["low"]["provider"] == "claude" + assert config["routing"]["low"]["model"] == "claude-haiku-4-5-20251001" def test_invalid_yaml_model_fails_validation(tmp_path):