From fe95d83e20486f9f86aa0c95b32b536b61398d6c Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Thu, 1 Oct 2026 10:55:58 -0700 Subject: [PATCH 1/4] chore(model-tiers): bump balanced tier from sonnet-4-6 to sonnet-5-5 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/assistant/model_tiers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/assistant/model_tiers.py b/src/assistant/model_tiers.py index 53df83a..c99e266 100644 --- a/src/assistant/model_tiers.py +++ b/src/assistant/model_tiers.py @@ -41,17 +41,17 @@ _DEFAULTS = { "bedrock": { "frontier": "us.anthropic.claude-opus-4-8", - "balanced": "us.anthropic.claude-sonnet-4-6", + "balanced": "us.anthropic.claude-sonnet-5-5", "cheap": "us.anthropic.claude-haiku-4-5", }, "anthropic": { "frontier": "claude-opus-4-8", - "balanced": "claude-sonnet-4-6", + "balanced": "claude-sonnet-5-5", "cheap": "claude-haiku-4-5", }, "vertex": { "frontier": "claude-opus-4-8", - "balanced": "claude-sonnet-4-6", + "balanced": "claude-sonnet-5-5", "cheap": "claude-haiku-4-5", }, } From ef9eea53985704a43fde683d3f38c96668c16a75 Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Thu, 1 Oct 2026 11:04:05 -0700 Subject: [PATCH 2/4] test(model-tiers): update balanced-tier assertions to sonnet-5-5 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- tests/test_comms_session.py | 6 +++--- tests/test_model_tiers.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_comms_session.py b/tests/test_comms_session.py index 38a0524..f5946d8 100644 --- a/tests/test_comms_session.py +++ b/tests/test_comms_session.py @@ -68,7 +68,7 @@ def test_warm_launch_declares_bedrock_backend_and_keeps_1m_context(monkeypatch): monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") cmd = cs._warm_launch(ag.CLAUDE) assert cmd.startswith("CLAUDE_CODE_USE_BEDROCK=1 ") - assert "us.anthropic.claude-sonnet-4-6[1m]" in cmd + assert "us.anthropic.claude-sonnet-5-5[1m]" in cmd def test_warm_launch_declares_non_bedrock_backend_and_still_gets_1m_context(monkeypatch): @@ -120,13 +120,13 @@ def test_resolve_warm_model_and_backend_respects_env(monkeypatch): monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") model, backend, pinned = cs._resolve_warm_model_and_backend() assert backend == "bedrock" - assert model == "us.anthropic.claude-sonnet-4-6[1m]" + assert model == "us.anthropic.claude-sonnet-5-5[1m]" assert pinned is False monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "0") model, backend, pinned = cs._resolve_warm_model_and_backend() assert backend == "anthropic" - assert model == "claude-sonnet-4-6[1m]" + assert model == "claude-sonnet-5-5[1m]" assert pinned is False monkeypatch.setenv("COMMS_MODEL", "my-pinned-id") diff --git a/tests/test_model_tiers.py b/tests/test_model_tiers.py index 8477ccc..8e2458d 100644 --- a/tests/test_model_tiers.py +++ b/tests/test_model_tiers.py @@ -92,7 +92,7 @@ def test_bedrock_ids(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1" # frontier is a CURRENT Opus (not the deprecated 4-1 the review flagged) self.assertEqual(mt.model_for("frontier"), "us.anthropic.claude-opus-4-8") - self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-4-6") + self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-5-5") self.assertEqual(mt.model_for("cheap"), "us.anthropic.claude-haiku-4-5") def test_anthropic_ids_are_bare(self): @@ -113,7 +113,7 @@ def test_provider_hint_pins_id_format_independent_of_detection(self): def test_long_context_suffix_is_opt_in_on_bedrock_and_anthropic(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1" self.assertEqual(mt.model_for("balanced", long_context=True), - "us.anthropic.claude-sonnet-4-6[1m]") + "us.anthropic.claude-sonnet-5-5[1m]") # not requested → no suffix (the mem0 path stays clean) self.assertNotIn("[1m]", mt.model_for("cheap")) # direct Anthropic ALSO gets it when requested — verified live: the @@ -121,7 +121,7 @@ def test_long_context_suffix_is_opt_in_on_bedrock_and_anthropic(self): # (2026-09-05 — an earlier version wrongly called this Bedrock-only). os.environ["MODEL_PROVIDER"] = "anthropic" self.assertEqual(mt.model_for("balanced", long_context=True), - "claude-sonnet-4-6[1m]") + "claude-sonnet-5-5[1m]") def test_per_tier_override_is_verbatim(self): os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1" From 4ca1eed6a3f827068d99978eff63cc5d0f8fefc4 Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Sun, 18 Oct 2026 23:01:38 -0700 Subject: [PATCH 3/4] docs(changelog): note the balanced tier move to Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c471a9..f8a44d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,12 @@ The version is carried in `pyproject.toml` and `src/assistant/__init__.py` - Add an attention overview with compact task cards, expandable return notes, and reminders to finish older pending work before starting another task. +### Changed +- Move the `balanced` model tier from Sonnet 4.6 to Sonnet 5.5 on Bedrock, the + Anthropic API, and Vertex. The warm Slack session, the Observer, the brief + narrator, and the strategist use this tier. A live warm session on the old + model is replaced before it handles its next Slack message. + ### Fixed - Bring the warm Slack session up again on Claude Code 2.1.303 and cmux 0.65. Claude Code now writes several hundred KB of records right after each prompt, From 7888d98dc8600b64f8c19e13733d074ff3f3ffcc Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Sun, 18 Oct 2026 23:42:08 -0700 Subject: [PATCH 4/4] fix(model-tiers): price Sonnet 5, point evals at the balanced tier, note the comms restart Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++-- bin/metering.py | 1 + docs/assistant-comms-onboarding.md | 2 +- evals/fleet/run.py | 10 +++++++--- evals/observer/README.md | 2 +- evals/observer/run.py | 12 ++++++++---- tests/test_metering.py | 5 +++++ 7 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a44d6..1b3cbfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,12 @@ The version is carried in `pyproject.toml` and `src/assistant/__init__.py` ### Changed - Move the `balanced` model tier from Sonnet 4.6 to Sonnet 5.5 on Bedrock, the Anthropic API, and Vertex. The warm Slack session, the Observer, the brief - narrator, and the strategist use this tier. A live warm session on the old - model is replaced before it handles its next Slack message. + narrator, and the strategist use this tier. Restart + `com.assistant.assistant-comms` to pick it up; the restarted daemon replaces + a warm session on the old model before its next Slack message. The Observer + and fleet evals now default to this tier's model, and their qualification + hashes include `model_tiers.py`. Cost estimates price Sonnet 5 models at $2 + and $10 per million input and output tokens. ### Fixed - Bring the warm Slack session up again on Claude Code 2.1.303 and cmux 0.65. diff --git a/bin/metering.py b/bin/metering.py index a16fba2..e97bc82 100644 --- a/bin/metering.py +++ b/bin/metering.py @@ -64,6 +64,7 @@ _PRICE_PER_MTOK = ( ("opus", (5.0, 25.0)), ("haiku", (1.0, 5.0)), + ("sonnet-5", (2.0, 10.0)), ("sonnet", (3.0, 15.0)), ("glm-5.2", (1.40, 4.40)), ) diff --git a/docs/assistant-comms-onboarding.md b/docs/assistant-comms-onboarding.md index 0b425a9..2f72234 100644 --- a/docs/assistant-comms-onboarding.md +++ b/docs/assistant-comms-onboarding.md @@ -79,7 +79,7 @@ To stop: `launchctl bootout gui/$UID/com.assistant.assistant-comms`. |---|---|---| | `SLACK_BOT_TOKEN` | ✅ | `xoxb-` bot token (from `~/.zprofile`). Never stored in config.json. | | `SLACK_PING_TARGET` | optional | `C…` private channel (recommended) or `U…` user (DMed); overrides `config.slack.target`. | -| `COMMS_MODEL` | optional | warm-session model (default Sonnet 4.6 1M). | +| `COMMS_MODEL` | optional | warm-session model (default Sonnet 5.5 1M). | | `COMMS_SLACK_POLL_SEC` | optional | inbound poll interval (default 3s). | | `COMMS_SUBMIT_ATTEMPTS` / `COMMS_SUBMIT_WAIT_SEC` | optional | Enter presses to try when a typed prompt doesn't reach the transcript, and seconds to wait after each (default 3 and 15s). | | `COMMS_LIVENESS_ATTEMPTS` / `COMMS_LIVENESS_RETRY_SEC` | optional | looks at a workspace cmux didn't answer about before calling it unknown, and seconds between them (default 3 and 3s). | diff --git a/evals/fleet/run.py b/evals/fleet/run.py index 242c713..af195eb 100644 --- a/evals/fleet/run.py +++ b/evals/fleet/run.py @@ -16,16 +16,19 @@ EVAL_DIR = Path(__file__).resolve().parent REPO = EVAL_DIR.parents[1] BIN = REPO / "bin" -if str(BIN) not in sys.path: - sys.path.insert(0, str(BIN)) +SRC = REPO / "src" +for path in (BIN, SRC): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) import llm_runner +from assistant import model_tiers CLAUDE_BIN = os.environ.get( "CLAUDE_BIN", str(Path.home() / ".local/bin/claude")) DROID_BIN = os.environ.get( "DROID_BIN", str(Path.home() / ".local/bin/droid")) CLAUDE_MODEL = os.environ.get( - "EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]") + "EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True)) DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2") TIMEOUT = int(os.environ.get("EVAL_TIMEOUT_SEC", "240")) REPORT = EVAL_DIR / "last-run.json" @@ -35,6 +38,7 @@ def qualification_input_hashes() -> dict[str, str]: paths = [ Path(__file__), BIN / "llm_runner.py", + SRC / "assistant/model_tiers.py", REPO / "prompts/triage-batch-prompt.md", REPO / "prompts/strategist-draft-prompt.md", REPO / "prompts/brief-narrator-prompt.md", diff --git a/evals/observer/README.md b/evals/observer/README.md index 58e5b9b..c4de9ad 100644 --- a/evals/observer/README.md +++ b/evals/observer/README.md @@ -9,7 +9,7 @@ workspace states. Each fixture is one test case. ./run.py # all fixtures ./run.py 01-ws97-trap-no-pr-mid-audit ./run.py --list # list fixture names -EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py +EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude ``` `run.py` spawns a one-shot headless `claude --print` per fixture, feeding diff --git a/evals/observer/run.py b/evals/observer/run.py index abef6ed..174971c 100755 --- a/evals/observer/run.py +++ b/evals/observer/run.py @@ -14,7 +14,7 @@ Usage: ./run.py # all fixtures ./run.py ws97-trap # one fixture - EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py + EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude """ from __future__ import annotations @@ -33,14 +33,17 @@ REPO = EVAL_DIR.parents[1] OBSERVER_PROMPT = REPO / "prompts/observer-batch-prompt.md" BIN = REPO / "bin" -if str(BIN) not in sys.path: - sys.path.insert(0, str(BIN)) +SRC = REPO / "src" +for path in (BIN, SRC): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) import llm_runner +from assistant import model_tiers CLAUDE_BIN = os.environ.get("CLAUDE_BIN", str(Path.home() / ".local/bin/claude")) DROID_BIN = os.environ.get("DROID_BIN", str(Path.home() / ".local/bin/droid")) CLAUDE_MODEL = os.environ.get( - "EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]") + "EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True)) DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2") TIMEOUT_SEC = int(os.environ.get("EVAL_TIMEOUT_SEC", "180")) REPORT_PATH = EVAL_DIR / "last-run.json" @@ -76,6 +79,7 @@ def qualification_input_hashes() -> dict[str, str]: Path(__file__), BIN / "llm_runner.py", BIN / "pulse.py", + SRC / "assistant/model_tiers.py", *sorted(path for path in FIXTURES_DIR.rglob("*") if path.is_file()), ] return { diff --git a/tests/test_metering.py b/tests/test_metering.py index 34d0fc7..e8ab58b 100644 --- a/tests/test_metering.py +++ b/tests/test_metering.py @@ -218,6 +218,11 @@ def test_observer_usage_falls_back_to_estimate(self): # 1000 in @ $3/M + tokens_out @ $15/M — sonnet rates self.assertGreater(u["cost_usd"], 0) + def test_sonnet_5_is_priced_below_sonnet_4(self): + self.assertEqual(self.mod.price_per_mtok("claude-sonnet-5-5[1m]"), (2.0, 10.0)) + self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-5-5"), (2.0, 10.0)) + self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-4-6[1m]"), (3.0, 15.0)) + def test_sum_usage_source_mixing(self): cli = {"tokens_in": 10, "tokens_out": 1, "cost_usd": 0.1, "source": "cli"} est = {"tokens_in": 20, "tokens_out": 2, "cost_usd": 0.2, "source": "estimated"}