diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c471a9..1b3cbfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,16 @@ The version is carried in `pyproject.toml` and `src/assistant/__init__.py` - Add an attention overview with compact task cards, expandable return notes, and reminders to finish older pending work before starting another task. +### Changed +- Move the `balanced` model tier from Sonnet 4.6 to Sonnet 5.5 on Bedrock, the + Anthropic API, and Vertex. The warm Slack session, the Observer, the brief + narrator, and the strategist use this tier. Restart + `com.assistant.assistant-comms` to pick it up; the restarted daemon replaces + a warm session on the old model before its next Slack message. The Observer + and fleet evals now default to this tier's model, and their qualification + hashes include `model_tiers.py`. Cost estimates price Sonnet 5 models at $2 + and $10 per million input and output tokens. + ### Fixed - Bring the warm Slack session up again on Claude Code 2.1.303 and cmux 0.65. Claude Code now writes several hundred KB of records right after each prompt, diff --git a/bin/metering.py b/bin/metering.py index a16fba2..e97bc82 100644 --- a/bin/metering.py +++ b/bin/metering.py @@ -64,6 +64,7 @@ _PRICE_PER_MTOK = ( ("opus", (5.0, 25.0)), ("haiku", (1.0, 5.0)), + ("sonnet-5", (2.0, 10.0)), ("sonnet", (3.0, 15.0)), ("glm-5.2", (1.40, 4.40)), ) diff --git a/docs/assistant-comms-onboarding.md b/docs/assistant-comms-onboarding.md index 0b425a9..2f72234 100644 --- a/docs/assistant-comms-onboarding.md +++ b/docs/assistant-comms-onboarding.md @@ -79,7 +79,7 @@ To stop: `launchctl bootout gui/$UID/com.assistant.assistant-comms`. |---|---|---| | `SLACK_BOT_TOKEN` | ✅ | `xoxb-` bot token (from `~/.zprofile`). Never stored in config.json. | | `SLACK_PING_TARGET` | optional | `C…` private channel (recommended) or `U…` user (DMed); overrides `config.slack.target`. | -| `COMMS_MODEL` | optional | warm-session model (default Sonnet 4.6 1M). | +| `COMMS_MODEL` | optional | warm-session model (default Sonnet 5.5 1M). | | `COMMS_SLACK_POLL_SEC` | optional | inbound poll interval (default 3s). | | `COMMS_SUBMIT_ATTEMPTS` / `COMMS_SUBMIT_WAIT_SEC` | optional | Enter presses to try when a typed prompt doesn't reach the transcript, and seconds to wait after each (default 3 and 15s). | | `COMMS_LIVENESS_ATTEMPTS` / `COMMS_LIVENESS_RETRY_SEC` | optional | looks at a workspace cmux didn't answer about before calling it unknown, and seconds between them (default 3 and 3s). | diff --git a/evals/fleet/run.py b/evals/fleet/run.py index 242c713..af195eb 100644 --- a/evals/fleet/run.py +++ b/evals/fleet/run.py @@ -16,16 +16,19 @@ EVAL_DIR = Path(__file__).resolve().parent REPO = EVAL_DIR.parents[1] BIN = REPO / "bin" -if str(BIN) not in sys.path: - sys.path.insert(0, str(BIN)) +SRC = REPO / "src" +for path in (BIN, SRC): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) import llm_runner +from assistant import model_tiers CLAUDE_BIN = os.environ.get( "CLAUDE_BIN", str(Path.home() / ".local/bin/claude")) DROID_BIN = os.environ.get( "DROID_BIN", str(Path.home() / ".local/bin/droid")) CLAUDE_MODEL = os.environ.get( - "EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]") + "EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True)) DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2") TIMEOUT = int(os.environ.get("EVAL_TIMEOUT_SEC", "240")) REPORT = EVAL_DIR / "last-run.json" @@ -35,6 +38,7 @@ def qualification_input_hashes() -> dict[str, str]: paths = [ Path(__file__), BIN / "llm_runner.py", + SRC / "assistant/model_tiers.py", REPO / "prompts/triage-batch-prompt.md", REPO / "prompts/strategist-draft-prompt.md", REPO / "prompts/brief-narrator-prompt.md", diff --git a/evals/observer/README.md b/evals/observer/README.md index 58e5b9b..c4de9ad 100644 --- a/evals/observer/README.md +++ b/evals/observer/README.md @@ -9,7 +9,7 @@ workspace states. Each fixture is one test case. ./run.py # all fixtures ./run.py 01-ws97-trap-no-pr-mid-audit ./run.py --list # list fixture names -EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py +EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude ``` `run.py` spawns a one-shot headless `claude --print` per fixture, feeding diff --git a/evals/observer/run.py b/evals/observer/run.py index abef6ed..174971c 100755 --- a/evals/observer/run.py +++ b/evals/observer/run.py @@ -14,7 +14,7 @@ Usage: ./run.py # all fixtures ./run.py ws97-trap # one fixture - EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py + EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude """ from __future__ import annotations @@ -33,14 +33,17 @@ REPO = EVAL_DIR.parents[1] OBSERVER_PROMPT = REPO / "prompts/observer-batch-prompt.md" BIN = REPO / "bin" -if str(BIN) not in sys.path: - sys.path.insert(0, str(BIN)) +SRC = REPO / "src" +for path in (BIN, SRC): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) import llm_runner +from assistant import model_tiers CLAUDE_BIN = os.environ.get("CLAUDE_BIN", str(Path.home() / ".local/bin/claude")) DROID_BIN = os.environ.get("DROID_BIN", str(Path.home() / ".local/bin/droid")) CLAUDE_MODEL = os.environ.get( - "EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]") + "EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True)) DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2") TIMEOUT_SEC = int(os.environ.get("EVAL_TIMEOUT_SEC", "180")) REPORT_PATH = EVAL_DIR / "last-run.json" @@ -76,6 +79,7 @@ def qualification_input_hashes() -> dict[str, str]: Path(__file__), BIN / "llm_runner.py", BIN / "pulse.py", + SRC / "assistant/model_tiers.py", *sorted(path for path in FIXTURES_DIR.rglob("*") if path.is_file()), ] return { diff --git a/src/assistant/model_tiers.py b/src/assistant/model_tiers.py index 53df83a..c99e266 100644 --- a/src/assistant/model_tiers.py +++ b/src/assistant/model_tiers.py @@ -41,17 +41,17 @@ _DEFAULTS = { "bedrock": { "frontier": "us.anthropic.claude-opus-4-8", - "balanced": "us.anthropic.claude-sonnet-4-6", + "balanced": "us.anthropic.claude-sonnet-5-5", "cheap": "us.anthropic.claude-haiku-4-5", }, "anthropic": { "frontier": "claude-opus-4-8", - "balanced": "claude-sonnet-4-6", + "balanced": "claude-sonnet-5-5", "cheap": "claude-haiku-4-5", }, "vertex": { "frontier": "claude-opus-4-8", - "balanced": "claude-sonnet-4-6", + "balanced": "claude-sonnet-5-5", "cheap": "claude-haiku-4-5", }, } diff --git a/tests/test_comms_session.py b/tests/test_comms_session.py index 38a0524..f5946d8 100644 --- a/tests/test_comms_session.py +++ b/tests/test_comms_session.py @@ -68,7 +68,7 @@ def test_warm_launch_declares_bedrock_backend_and_keeps_1m_context(monkeypatch): monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") cmd = cs._warm_launch(ag.CLAUDE) assert cmd.startswith("CLAUDE_CODE_USE_BEDROCK=1 ") - assert "us.anthropic.claude-sonnet-4-6[1m]" in cmd + assert "us.anthropic.claude-sonnet-5-5[1m]" in cmd def test_warm_launch_declares_non_bedrock_backend_and_still_gets_1m_context(monkeypatch): @@ -120,13 +120,13 @@ def test_resolve_warm_model_and_backend_respects_env(monkeypatch): monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") model, backend, pinned = cs._resolve_warm_model_and_backend() assert backend == "bedrock" - assert model == "us.anthropic.claude-sonnet-4-6[1m]" + assert model == "us.anthropic.claude-sonnet-5-5[1m]" assert pinned is False monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "0") model, backend, pinned = cs._resolve_warm_model_and_backend() assert backend == "anthropic" - assert model == "claude-sonnet-4-6[1m]" + assert model == "claude-sonnet-5-5[1m]" assert pinned is False monkeypatch.setenv("COMMS_MODEL", "my-pinned-id") diff --git a/tests/test_metering.py b/tests/test_metering.py index 34d0fc7..e8ab58b 100644 --- a/tests/test_metering.py +++ b/tests/test_metering.py @@ -218,6 +218,11 @@ def test_observer_usage_falls_back_to_estimate(self): # 1000 in @ $3/M + tokens_out @ $15/M — sonnet rates self.assertGreater(u["cost_usd"], 0) + def test_sonnet_5_is_priced_below_sonnet_4(self): + self.assertEqual(self.mod.price_per_mtok("claude-sonnet-5-5[1m]"), (2.0, 10.0)) + self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-5-5"), (2.0, 10.0)) + self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-4-6[1m]"), (3.0, 15.0)) + def test_sum_usage_source_mixing(self): cli = {"tokens_in": 10, "tokens_out": 1, "cost_usd": 0.1, "source": "cli"} est = {"tokens_in": 20, "tokens_out": 2, "cost_usd": 0.2, "source": "estimated"} diff --git a/tests/test_model_tiers.py b/tests/test_model_tiers.py index 8477ccc..8e2458d 100644 --- a/tests/test_model_tiers.py +++ b/tests/test_model_tiers.py @@ -92,7 +92,7 @@ def test_bedrock_ids(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1" # frontier is a CURRENT Opus (not the deprecated 4-1 the review flagged) self.assertEqual(mt.model_for("frontier"), "us.anthropic.claude-opus-4-8") - self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-4-6") + self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-5-5") self.assertEqual(mt.model_for("cheap"), "us.anthropic.claude-haiku-4-5") def test_anthropic_ids_are_bare(self): @@ -113,7 +113,7 @@ def test_provider_hint_pins_id_format_independent_of_detection(self): def test_long_context_suffix_is_opt_in_on_bedrock_and_anthropic(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1" self.assertEqual(mt.model_for("balanced", long_context=True), - "us.anthropic.claude-sonnet-4-6[1m]") + "us.anthropic.claude-sonnet-5-5[1m]") # not requested → no suffix (the mem0 path stays clean) self.assertNotIn("[1m]", mt.model_for("cheap")) # direct Anthropic ALSO gets it when requested — verified live: the @@ -121,7 +121,7 @@ def test_long_context_suffix_is_opt_in_on_bedrock_and_anthropic(self): # (2026-09-05 — an earlier version wrongly called this Bedrock-only). os.environ["MODEL_PROVIDER"] = "anthropic" self.assertEqual(mt.model_for("balanced", long_context=True), - "claude-sonnet-4-6[1m]") + "claude-sonnet-5-5[1m]") def test_per_tier_override_is_verbatim(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1"