Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions apps/backend/config/model_routing.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,13 @@ routing:
provider: openai
model: gpt-4o
max_tokens: 8192
# Trivial subtasks route to Haiku by default: it stays inside the Claude
# runtime (works under AUTO_CODE_AUTONOMY=claude, no second provider needed)
# and is far cheaper than the Sonnet fallback. Override to gpt-4o-mini via
# MODEL_ROUTER_LOW_PROVIDER=openai + MODEL_ROUTER_LOW_MODEL=gpt-4o-mini.
low:
provider: openai
model: gpt-4o-mini
provider: claude
model: claude-haiku-4-5-20251001
max_tokens: 4096

# Default estimated tokens for cost calculation
Expand Down
17 changes: 16 additions & 1 deletion apps/backend/core/providers/task_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,16 @@

logger = logging.getLogger(__name__)

# Cheap "Haiku-equivalent" model per provider. Used when the low (trivial)
# tier can't reach its configured provider and must fall back: without this,
# the fallback would use the provider's *default* (expensive) model, silently
# upgrading trivial subtasks and inverting the low < medium cost ordering.
CHEAP_MODEL_BY_PROVIDER: dict[str, str] = {
"claude": "claude-haiku-4-5-20251001",
"openai": "gpt-4o-mini",
"google": "gemini-1.5-flash",
}

Comment on lines +24 to +33

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🔵 Trivial | 🏗️ Heavy lift

Hardcoded model map duplicates the config-driven default and can silently drift.

CHEAP_MODEL_BY_PROVIDER is a second, code-level source of truth for "the cheap model per provider," parallel to model_routing.yaml/task_router_config.py which already define the low-tier default (claude/claude-haiku-4-5-20251001) and its overrides (MODEL_ROUTER_LOW_PROVIDER/MODEL_ROUTER_LOW_MODEL). If the config-driven low-tier default is changed later (e.g., via env var or YAML), this hardcoded map won't follow, so a claude → openai fallback could route to a stale gpt-4o-mini while the "primary" low tier has moved on. Consider deriving per-provider cheap models from configuration (e.g., a cheap_models section in model_routing.yaml) rather than a separate Python constant.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@apps/backend/core/providers/task_router.py` around lines 24 - 33, The
hardcoded CHEAP_MODEL_BY_PROVIDER map in task_router.py duplicates the
config-driven low-tier model settings and can drift from
model_routing.yaml/task_router_config.py. Update the fallback routing logic in
the task router to derive cheap models from the existing configuration source
used for MODEL_ROUTER_LOW_PROVIDER and MODEL_ROUTER_LOW_MODEL, or add a
config-backed cheap_models section and load it there. Keep the provider fallback
behavior in sync with the config so changes to the low-tier default
automatically apply across all providers.


@dataclass
class TaskRoute:
Expand Down Expand Up @@ -175,7 +185,12 @@ def _resolve_available_route_config(
)
return configured_route

fallback_model = provider_config.get_model_for(fallback_provider)
# For the cheap (low) tier, keep the fallback cheap: use the
# provider's Haiku-equivalent rather than its expensive default model.
if complexity == "low" and fallback_provider in CHEAP_MODEL_BY_PROVIDER:
fallback_model = CHEAP_MODEL_BY_PROVIDER[fallback_provider]
else:
fallback_model = provider_config.get_model_for(fallback_provider)
if not fallback_model:
configured_route["fallback_reason"] = (
f"configured provider {configured_provider} is {unavailable_reason}; "
Expand Down
7 changes: 5 additions & 2 deletions apps/backend/core/providers/task_router_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,12 @@
"model": "gpt-4o",
"max_tokens": 8192,
},
# Trivial subtasks default to Haiku (a cheap Claude model that works
# under the default claude-only runtime); override to gpt-4o-mini via
# MODEL_ROUTER_LOW_* env vars or the routing YAML.
"low": {
"provider": "openai",
"model": "gpt-4o-mini",
"provider": "claude",
"model": "claude-haiku-4-5-20251001",
"max_tokens": 4096,
},
},
Expand Down
72 changes: 64 additions & 8 deletions tests/test_task_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ def test_route_maps_score_to_configured_low_medium_high_models():
)

assert low_route.complexity == "low"
assert low_route.provider == "openai"
assert low_route.model == "gpt-4o-mini"
assert low_route.provider == "claude"
assert low_route.model == "claude-haiku-4-5-20251001"
assert medium_route.complexity == "medium"
assert medium_route.model == "gpt-4o"
assert high_route.complexity == "high"
Expand All @@ -145,8 +145,9 @@ def test_route_maps_score_to_configured_low_medium_high_models():
)


def test_route_falls_back_to_available_provider_when_route_provider_unavailable():
"""Routing should not select OpenAI when only Claude auth is configured."""
def test_low_complexity_routes_to_haiku_under_claude_only():
"""Trivial subtasks route to Haiku, which needs no fallback under a
claude-only runtime — the P5.T5 cost-saving default."""
router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml"))
router.risk_analyzer = MagicMock()
router.risk_analyzer.analyze_subtask_risks.return_value = []
Expand All @@ -158,10 +159,61 @@ def test_route_falls_back_to_available_provider_when_route_provider_unavailable(
anthropic_api_key="test-anthropic-key",
claude_model="claude-sonnet-4-5-20250929",
),
allowed_providers={"claude"},
)

assert route.complexity == "low"
assert route.provider == "claude"
assert route.model == "claude-haiku-4-5-20251001"
# Haiku is already claude, so no fallback/compat downgrade fires.
assert "unavailable" not in route.reasoning
assert "not compatible" not in route.reasoning


def test_low_complexity_fallback_uses_cheap_model_not_provider_default():
"""When the low tier can't use Claude, the fallback must pick a CHEAP model
of the available provider (gpt-4o-mini), not that provider's expensive
default — otherwise trivial subtasks silently upgrade and low >= medium."""
router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml"))
router.risk_analyzer = MagicMock()
router.risk_analyzer.analyze_subtask_risks.return_value = []

route = router.route(
{"description": "Fix typo", "files_to_modify": []},
provider_config=ProviderConfig(
provider="openai",
openai_api_key="test-openai-key",
anthropic_api_key="test-anthropic-key", # Claude available, but...
),
allowed_providers={"openai"}, # ...the runtime only allows OpenAI.
)

assert route.complexity == "low"
assert route.provider == "openai"
assert route.model == "gpt-4o-mini"
assert "not compatible" in route.reasoning


def test_route_falls_back_to_available_provider_when_route_provider_unavailable():
"""A medium (OpenAI) route falls back to Claude when only Claude auth is set."""
router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml"))
router.risk_analyzer = MagicMock()
router.risk_analyzer.analyze_subtask_risks.return_value = [_issue("medium")] * 2

route = router.route(
{
"description": "Add pagination endpoint",
"files_to_modify": ["api/users.py", "tests/test_users.py"],
},
provider_config=ProviderConfig(
provider="claude",
anthropic_api_key="test-anthropic-key",
claude_model="claude-sonnet-4-5-20250929",
),
)

assert route.complexity == "medium"
assert route.provider == "claude"
assert route.model == "claude-sonnet-4-5-20250929"
assert "configured provider openai is unavailable" in route.reasoning

Expand All @@ -170,10 +222,13 @@ def test_route_respects_runtime_provider_allowlist():
"""Runtime compatibility should override an otherwise available provider."""
router = TaskComplexityRouter(config_path=Path("/missing/model_routing.yaml"))
router.risk_analyzer = MagicMock()
router.risk_analyzer.analyze_subtask_risks.return_value = []
router.risk_analyzer.analyze_subtask_risks.return_value = [_issue("medium")] * 2

route = router.route(
{"description": "Fix typo", "files_to_modify": []},
{
"description": "Add pagination endpoint",
"files_to_modify": ["api/users.py", "tests/test_users.py"],
},
provider_config=ProviderConfig(
provider="openai",
anthropic_api_key="test-anthropic-key",
Expand All @@ -183,7 +238,7 @@ def test_route_respects_runtime_provider_allowlist():
allowed_providers={"claude"},
)

assert route.complexity == "low"
assert route.complexity == "medium"
assert route.provider == "claude"
assert route.model == "claude-sonnet-4-5-20250929"
assert "not compatible with the selected runtime" in route.reasoning
Expand Down Expand Up @@ -227,7 +282,8 @@ def test_missing_yaml_uses_defaults():

assert config["routing"]["high"]["provider"] == "claude"
assert config["routing"]["medium"]["model"] == "gpt-4o"
assert config["routing"]["low"]["model"] == "gpt-4o-mini"
assert config["routing"]["low"]["provider"] == "claude"
assert config["routing"]["low"]["model"] == "claude-haiku-4-5-20251001"


def test_invalid_yaml_model_fails_validation(tmp_path):
Expand Down
Loading