Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ The version is carried in `pyproject.toml` and `src/assistant/__init__.py`
- Add an attention overview with compact task cards, expandable return notes,
and reminders to finish older pending work before starting another task.

### Changed
- Move the `balanced` model tier from Sonnet 4.6 to Sonnet 5.5 on Bedrock, the
Anthropic API, and Vertex. The warm Slack session, the Observer, the brief
narrator, and the strategist use this tier. Restart
`com.assistant.assistant-comms` to pick it up; the restarted daemon replaces
a warm session on the old model before its next Slack message. The Observer
and fleet evals now default to this tier's model, and their qualification
hashes include `model_tiers.py`. Cost estimates price Sonnet 5 models at $2
and $10 per million input and output tokens.

### Fixed
- Bring the warm Slack session up again on Claude Code 2.1.303 and cmux 0.65.
Claude Code now writes several hundred KB of records right after each prompt,
Expand Down
1 change: 1 addition & 0 deletions bin/metering.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
_PRICE_PER_MTOK = (
("opus", (5.0, 25.0)),
("haiku", (1.0, 5.0)),
("sonnet-5", (2.0, 10.0)),
("sonnet", (3.0, 15.0)),
("glm-5.2", (1.40, 4.40)),
)
Expand Down
2 changes: 1 addition & 1 deletion docs/assistant-comms-onboarding.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ To stop: `launchctl bootout gui/$UID/com.assistant.assistant-comms`.
|---|---|---|
| `SLACK_BOT_TOKEN` | ✅ | `xoxb-` bot token (from `~/.zprofile`). Never stored in config.json. |
| `SLACK_PING_TARGET` | optional | `C…` private channel (recommended) or `U…` user (DMed); overrides `config.slack.target`. |
| `COMMS_MODEL` | optional | warm-session model (default Sonnet 4.6 1M). |
| `COMMS_MODEL` | optional | warm-session model (default Sonnet 5.5 1M). |
| `COMMS_SLACK_POLL_SEC` | optional | inbound poll interval (default 3s). |
| `COMMS_SUBMIT_ATTEMPTS` / `COMMS_SUBMIT_WAIT_SEC` | optional | Enter presses to try when a typed prompt doesn't reach the transcript, and seconds to wait after each (default 3 and 15s). |
| `COMMS_LIVENESS_ATTEMPTS` / `COMMS_LIVENESS_RETRY_SEC` | optional | looks at a workspace cmux didn't answer about before calling it unknown, and seconds between them (default 3 and 3s). |
Expand Down
10 changes: 7 additions & 3 deletions evals/fleet/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,19 @@
EVAL_DIR = Path(__file__).resolve().parent
REPO = EVAL_DIR.parents[1]
BIN = REPO / "bin"
if str(BIN) not in sys.path:
sys.path.insert(0, str(BIN))
SRC = REPO / "src"
for path in (BIN, SRC):
if str(path) not in sys.path:
sys.path.insert(0, str(path))
import llm_runner
from assistant import model_tiers

CLAUDE_BIN = os.environ.get(
"CLAUDE_BIN", str(Path.home() / ".local/bin/claude"))
DROID_BIN = os.environ.get(
"DROID_BIN", str(Path.home() / ".local/bin/droid"))
CLAUDE_MODEL = os.environ.get(
"EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]")
"EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True))
DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2")
TIMEOUT = int(os.environ.get("EVAL_TIMEOUT_SEC", "240"))
REPORT = EVAL_DIR / "last-run.json"
Expand All @@ -35,6 +38,7 @@ def qualification_input_hashes() -> dict[str, str]:
paths = [
Path(__file__),
BIN / "llm_runner.py",
SRC / "assistant/model_tiers.py",
REPO / "prompts/triage-batch-prompt.md",
REPO / "prompts/strategist-draft-prompt.md",
REPO / "prompts/brief-narrator-prompt.md",
Expand Down
2 changes: 1 addition & 1 deletion evals/observer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ workspace states. Each fixture is one test case.
./run.py # all fixtures
./run.py 01-ws97-trap-no-pr-mid-audit
./run.py --list # list fixture names
EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py
EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude
```

`run.py` spawns a one-shot headless `claude --print` per fixture, feeding
Expand Down
12 changes: 8 additions & 4 deletions evals/observer/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
Usage:
./run.py # all fixtures
./run.py ws97-trap # one fixture
EVAL_MODEL=us.anthropic.claude-sonnet-4-6[1m] ./run.py
EVAL_CLAUDE_MODEL='claude-sonnet-5-5[1m]' ./run.py --provider claude
"""
from __future__ import annotations

Expand All @@ -33,14 +33,17 @@
REPO = EVAL_DIR.parents[1]
OBSERVER_PROMPT = REPO / "prompts/observer-batch-prompt.md"
BIN = REPO / "bin"
if str(BIN) not in sys.path:
sys.path.insert(0, str(BIN))
SRC = REPO / "src"
for path in (BIN, SRC):
if str(path) not in sys.path:
sys.path.insert(0, str(path))
import llm_runner
from assistant import model_tiers

CLAUDE_BIN = os.environ.get("CLAUDE_BIN", str(Path.home() / ".local/bin/claude"))
DROID_BIN = os.environ.get("DROID_BIN", str(Path.home() / ".local/bin/droid"))
CLAUDE_MODEL = os.environ.get(
"EVAL_CLAUDE_MODEL", "us.anthropic.claude-sonnet-4-6[1m]")
"EVAL_CLAUDE_MODEL", model_tiers.model_for("balanced", long_context=True))
DROID_MODEL = os.environ.get("EVAL_DROID_MODEL", "glm-5.2")
TIMEOUT_SEC = int(os.environ.get("EVAL_TIMEOUT_SEC", "180"))
REPORT_PATH = EVAL_DIR / "last-run.json"
Expand Down Expand Up @@ -76,6 +79,7 @@ def qualification_input_hashes() -> dict[str, str]:
Path(__file__),
BIN / "llm_runner.py",
BIN / "pulse.py",
SRC / "assistant/model_tiers.py",
*sorted(path for path in FIXTURES_DIR.rglob("*") if path.is_file()),
]
return {
Expand Down
6 changes: 3 additions & 3 deletions src/assistant/model_tiers.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,17 @@
_DEFAULTS = {
"bedrock": {
"frontier": "us.anthropic.claude-opus-4-8",
"balanced": "us.anthropic.claude-sonnet-4-6",
"balanced": "us.anthropic.claude-sonnet-5-5",
"cheap": "us.anthropic.claude-haiku-4-5",
},
"anthropic": {
"frontier": "claude-opus-4-8",
"balanced": "claude-sonnet-4-6",
"balanced": "claude-sonnet-5-5",
"cheap": "claude-haiku-4-5",
},
"vertex": {
"frontier": "claude-opus-4-8",
"balanced": "claude-sonnet-4-6",
"balanced": "claude-sonnet-5-5",
"cheap": "claude-haiku-4-5",
},
}
Expand Down
6 changes: 3 additions & 3 deletions tests/test_comms_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def test_warm_launch_declares_bedrock_backend_and_keeps_1m_context(monkeypatch):
monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1")
cmd = cs._warm_launch(ag.CLAUDE)
assert cmd.startswith("CLAUDE_CODE_USE_BEDROCK=1 ")
assert "us.anthropic.claude-sonnet-4-6[1m]" in cmd
assert "us.anthropic.claude-sonnet-5-5[1m]" in cmd


def test_warm_launch_declares_non_bedrock_backend_and_still_gets_1m_context(monkeypatch):
Expand Down Expand Up @@ -120,13 +120,13 @@ def test_resolve_warm_model_and_backend_respects_env(monkeypatch):
monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1")
model, backend, pinned = cs._resolve_warm_model_and_backend()
assert backend == "bedrock"
assert model == "us.anthropic.claude-sonnet-4-6[1m]"
assert model == "us.anthropic.claude-sonnet-5-5[1m]"
assert pinned is False

monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "0")
model, backend, pinned = cs._resolve_warm_model_and_backend()
assert backend == "anthropic"
assert model == "claude-sonnet-4-6[1m]"
assert model == "claude-sonnet-5-5[1m]"
assert pinned is False

monkeypatch.setenv("COMMS_MODEL", "my-pinned-id")
Expand Down
5 changes: 5 additions & 0 deletions tests/test_metering.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,11 @@ def test_observer_usage_falls_back_to_estimate(self):
# 1000 in @ $3/M + tokens_out @ $15/M — sonnet rates
self.assertGreater(u["cost_usd"], 0)

def test_sonnet_5_is_priced_below_sonnet_4(self):
self.assertEqual(self.mod.price_per_mtok("claude-sonnet-5-5[1m]"), (2.0, 10.0))
self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-5-5"), (2.0, 10.0))
self.assertEqual(self.mod.price_per_mtok("us.anthropic.claude-sonnet-4-6[1m]"), (3.0, 15.0))

def test_sum_usage_source_mixing(self):
cli = {"tokens_in": 10, "tokens_out": 1, "cost_usd": 0.1, "source": "cli"}
est = {"tokens_in": 20, "tokens_out": 2, "cost_usd": 0.2, "source": "estimated"}
Expand Down
6 changes: 3 additions & 3 deletions tests/test_model_tiers.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def test_bedrock_ids(self):
os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1"
# frontier is a CURRENT Opus (not the deprecated 4-1 the review flagged)
self.assertEqual(mt.model_for("frontier"), "us.anthropic.claude-opus-4-8")
self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-4-6")
self.assertEqual(mt.model_for("balanced"), "us.anthropic.claude-sonnet-5-5")
self.assertEqual(mt.model_for("cheap"), "us.anthropic.claude-haiku-4-5")

def test_anthropic_ids_are_bare(self):
Expand All @@ -113,15 +113,15 @@ def test_provider_hint_pins_id_format_independent_of_detection(self):
def test_long_context_suffix_is_opt_in_on_bedrock_and_anthropic(self):
os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1"
self.assertEqual(mt.model_for("balanced", long_context=True),
"us.anthropic.claude-sonnet-4-6[1m]")
"us.anthropic.claude-sonnet-5-5[1m]")
# not requested → no suffix (the mem0 path stays clean)
self.assertNotIn("[1m]", mt.model_for("cheap"))
# direct Anthropic ALSO gets it when requested — verified live: the
# operator's non-Bedrock ~/.zprofile alias runs `[1m]` on a bare id
# (2026-09-05 — an earlier version wrongly called this Bedrock-only).
os.environ["MODEL_PROVIDER"] = "anthropic"
self.assertEqual(mt.model_for("balanced", long_context=True),
"claude-sonnet-4-6[1m]")
"claude-sonnet-5-5[1m]")

def test_per_tier_override_is_verbatim(self):
os.environ["CLAUDE_CODE_USE_BEDROCK"] = "1"
Expand Down
Loading