diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 253f602b3..76c7ed378 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -119,6 +119,7 @@ def _apt_install(*packages: str) -> str: # OpenCode routes through the chat-completions path. Shared with # ``benchflow.acp.runtime._format_acp_model`` so set_model targets the same id. OPENCODE_PROXY_PROVIDER_ID = "benchflow" +_CLAUDE_AGENT_ACP_PACKAGE = "@agentclientprotocol/claude-agent-acp@0.73.0" _OPENHANDS_CLI_GIT_REV = "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271" _OPENHANDS_SDK_VERSION = "1.28.1" _OPENHANDS_TOOLS_VERSION = "1.28.1" @@ -517,14 +518,15 @@ class AgentConfig: description="Claude Code via ACP (Anthropic's Agent Client Protocol)", skill_paths=["$HOME/.claude/skills"], home_dirs=[".claude"], - # Pinned to 0.40.0: the config-option wiring below (set_config_option + - # the "model"/"effort" ids) targets this version's ACP protocol (sdk - # 0.24, which dropped session/set_model). The option ids are coupled to - # this pin — re-verify them when bumping. runtime.py uses - # capability-first dispatch for the rest of the family. - install_cmd=_js_agent_install( - "claude-agent-acp", "@agentclientprotocol/claude-agent-acp@0.40.0" - ), + # Pinned to 0.73.0 (bundles @anthropic-ai/claude-agent-sdk 0.3.257): + # claude-fable-5-1 rejects Claude Code < 2.1.251 with + # `claude_code_version_too_old` (HTTP 400), so the previous 0.40.0 pin + # (sdk 0.3.160) cannot run that model at all. The config-option wiring + # below (set_config_option + the "model"/"effort" ids) was re-verified + # against 0.73.0 with tests/test_acp_pinned_protocol_guard.py; the ids + # stay coupled to this pin — re-run that guard when bumping. runtime.py + # uses capability-first dispatch for the rest of the family. + install_cmd=_js_agent_install("claude-agent-acp", _CLAUDE_AGENT_ACP_PACKAGE), launch_cmd=_js_agent_launch("claude-agent-acp"), protocol="acp", requires_env=["ANTHROPIC_API_KEY"], diff --git a/tests/test_acp_pinned_protocol_guard.py b/tests/test_acp_pinned_protocol_guard.py index 76a8e20d0..6cb884cd0 100644 --- a/tests/test_acp_pinned_protocol_guard.py +++ b/tests/test_acp_pinned_protocol_guard.py @@ -1,5 +1,4 @@ -"""Gated live guard: the pinned claude-agent-acp advertises the config option -ids the registry wires up. +"""Gated live guard for the pinned claude-agent-acp config-option contract. Skipped by default. Run with ``RUN_ACP_DEP_GUARD=1`` (needs ``npm`` + ``node`` + network): @@ -7,13 +6,11 @@ RUN_ACP_DEP_GUARD=1 uv run --extra dev python -m pytest \ tests/test_acp_pinned_protocol_guard.py -q -It installs the pinned ``claude-agent-acp@0.40.0``, starts it over ACP stdio, -runs ``initialize`` + ``session/new``, and asserts the advertised config option -ids include ``{"model", "effort"}`` — the ids ``benchflow.agents.registry`` -hard-codes for model and reasoning-effort selection. If a future pin keeps -``session/set_config_option`` but renames an id, this fails (a plain SDK method -grep would not). ``session/new`` advertises the options without auth, so no -credentials are needed. Re-run when bumping the ``@agentclientprotocol`` pin. +It installs the exact package selected by ``benchflow.agents.registry``, starts +it over ACP stdio, and proves the complete Fable model + effort path works: +``initialize``, ``session/new``, and both ``session/set_config_option`` calls. +The adapter exposes and accepts these options without auth, so no credentials +are needed. Re-run when bumping the ``@agentclientprotocol`` pin. """ import asyncio @@ -25,13 +22,16 @@ import pytest +from benchflow.agents.registry import _CLAUDE_AGENT_ACP_PACKAGE + pytestmark = pytest.mark.skipif( os.environ.get("RUN_ACP_DEP_GUARD") != "1", reason="gated live ACP guard; set RUN_ACP_DEP_GUARD=1 (needs npm + node + network)", ) -PINNED_CLAUDE = "@agentclientprotocol/claude-agent-acp@0.40.0" EXPECTED_OPTION_IDS = {"model", "effort"} +FABLE_MODEL = "claude-fable-5-1" +FABLE_EFFORT = "xhigh" def _tool_or_skip(name: str) -> str: @@ -41,7 +41,7 @@ def _tool_or_skip(name: str) -> str: return path -async def _advertised_option_ids(entry: Path) -> set[str]: +async def _exercise_config_options(entry: Path) -> tuple[set[str], dict[str, str]]: from benchflow.acp.client import ACPClient from benchflow.acp.transport import StdioTransport @@ -51,23 +51,39 @@ async def _advertised_option_ids(entry: Path) -> set[str]: await asyncio.wait_for(client.initialize(), timeout=60) await asyncio.wait_for(client.session_new(cwd="/tmp"), timeout=90) opts = client.session.config_options or [] - return { + ids = { o["id"] for o in opts if isinstance(o, dict) and isinstance(o.get("id"), str) } + if ids >= EXPECTED_OPTION_IDS: + await asyncio.wait_for( + client.set_config_option("model", FABLE_MODEL), timeout=60 + ) + await asyncio.wait_for( + client.set_config_option("effort", FABLE_EFFORT), timeout=60 + ) + current = { + o["id"]: o["currentValue"] + for o in client.session.config_options or [] + if isinstance(o, dict) + and o.get("id") in EXPECTED_OPTION_IDS + and isinstance(o.get("currentValue"), str) + } + return ids, current finally: with contextlib.suppress(Exception): await client.close() -def test_pinned_claude_acp_advertises_model_and_effort_options(tmp_path): +def test_pinned_claude_acp_supports_fable_model_and_effort(tmp_path): + """Guards PR #1086's Fable-compatible adapter and ACP config contract.""" npm = _tool_or_skip("npm") _tool_or_skip("node") prefix = tmp_path / "claude" prefix.mkdir() subprocess.run( - [npm, "install", "--prefix", str(prefix), PINNED_CLAUDE], + [npm, "install", "--prefix", str(prefix), _CLAUDE_AGENT_ACP_PACKAGE], check=True, capture_output=True, text=True, @@ -83,11 +99,13 @@ def test_pinned_claude_acp_advertises_model_and_effort_options(tmp_path): ) assert entry.is_file(), f"pinned agent entry not found: {entry}" - ids = asyncio.run(_advertised_option_ids(entry)) + ids, current = asyncio.run(_exercise_config_options(entry)) missing = EXPECTED_OPTION_IDS - ids assert not missing, ( - f"pinned {PINNED_CLAUDE} no longer advertises config option(s) " + f"pinned {_CLAUDE_AGENT_ACP_PACKAGE} no longer advertises config option(s) " f"{sorted(missing)!r} (advertised: {sorted(ids)!r}); the registry " f"model/effort wiring is stale — re-verify acp_model_config_id / " f"acp_effort_config_id" ) + assert current.get("model", "").split("[", 1)[0] == FABLE_MODEL, current + assert current.get("effort") == FABLE_EFFORT, current diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index f0fb03abe..3ae2d3266 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -26,6 +26,7 @@ class TestEnvMappingField: """env_mapping exists on AgentConfig and is populated for known agents.""" def test_claude_agent_has_mapping(self): + """Guards PR #1086's Fable-compatible adapter pin and ACP config ids.""" cfg = AGENTS["claude-agent-acp"] assert "BENCHFLOW_PROVIDER_BASE_URL" in cfg.env_mapping assert cfg.env_mapping["BENCHFLOW_PROVIDER_BASE_URL"] == "ANTHROPIC_BASE_URL" @@ -33,7 +34,7 @@ def test_claude_agent_has_mapping(self): assert cfg.supports_acp_set_model is False assert cfg.acp_model_config_id == "model" assert cfg.acp_effort_config_id == "effort" - assert "@agentclientprotocol/claude-agent-acp@0.40.0" in cfg.install_cmd + assert "@agentclientprotocol/claude-agent-acp@0.73.0" in cfg.install_cmd def test_pi_acp_no_static_mapping(self): """pi-acp is multi-protocol — launch wrapper handles env translation."""