Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions tests/test_verbose_sidecall.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
"""Verbose Claude Code streams an event array, not one result object (#131).

With "verbose": true in the user's settings, `claude -p --output-format json`
prints a JSON array of events, and claude_call's data.get("result") raised
AttributeError on the list. The micro-skills proposer then let the exception
escape, aborting the whole fold. Both halves are pinned here.
"""
import json
from types import SimpleNamespace

import pytest

from tokeneconomizer.microskills import GENERIC
from tokeneconomizer.microskills import distiller as dist
from tokeneconomizer.plugin import sidecall
from fixtures import write_claude_transcript


def _verbose_stdout(result_text):
events = [
{"type": "system", "subtype": "init"},
{"type": "assistant", "message": {}},
{"type": "result", "result": result_text,
"usage": {"input_tokens": 10, "output_tokens": 5}},
]
return json.dumps(events)


@pytest.mark.real_sidecalls
def test_claude_call_accepts_verbose_event_array(monkeypatch):
def fake_run(cmd, **kw):
if "--help" in cmd:
return SimpleNamespace(stdout="")
return SimpleNamespace(stdout=_verbose_stdout("hello"), stderr="")

monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
text, eff = sidecall.claude_call("hi")
assert text == "hello"
assert eff is not None


@pytest.mark.real_sidecalls
def test_claude_call_ignores_events_without_a_result(monkeypatch):
# No result event at all: no reply, not a crash.
def fake_run(cmd, **kw):
if "--help" in cmd:
return SimpleNamespace(stdout="")
return SimpleNamespace(
stdout=json.dumps([{"type": "system", "subtype": "init"}]), stderr="")

monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
text, eff = sidecall.claude_call("hi")
assert text == ""
assert eff is None


def test_failed_proposer_call_skips_proposal_instead_of_aborting_fold(tmp_path, monkeypatch):
monkeypatch.setattr(dist, "PROPOSE_EVERY", 1)

calls = []

def runner(prompt, model):
calls.append(prompt)
raise RuntimeError("side call failed")

root = str(tmp_path / "state")
d = dist.Distiller.load("acme/tool", root, runner=runner, vocab=GENERIC)
path = write_claude_transcript(str(tmp_path / "s0.jsonl"), [
("Bash", {"command": "make build"}, "built"),
("Bash", {"command": "make test"}, "ok ./... 0.01s")])
d.update(path, iid="s0") # must not raise
d.save(root)
assert d.facts == {}
# The failed call was made once and counts as the tick: the next waits its sessions.
assert len(calls) == 1 and d.last_propose == d.now


def test_a_failed_proposer_call_is_noted_in_the_errors_log(tmp_path):
from tokeneconomizer.plugin import fold

def runner(prompt, model):
raise RuntimeError("side call failed")

call = fold._noted(str(tmp_path), runner)
with pytest.raises(RuntimeError):
call("a prompt", "agent")
log = (tmp_path / "errors.log").read_text(encoding="utf-8")
assert "(proposer side call)" in log and "RuntimeError: side call failed" in log
9 changes: 8 additions & 1 deletion tokeneconomizer/microskills/distiller.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,14 @@ def _maybe_propose(self):
{i: f.claim for i, f in self.facts.items()},
set(self.extractors), excerpts, external=external,
generic=self.generic)
raw, eff = self.runner(prompt, self.model)
# A failed side call skips the proposal instead of aborting the fold (#131). It still
# counts as a tick: a CLI that keeps failing is called every PROPOSE_EVERY sessions, as
# a working one is, not at every fold.
try:
raw, eff = self.runner(prompt, self.model)
except Exception:
self.last_propose = self.now
return
if eff is not None:
self.opt_tokens += eff
exts, facts = parse_payload(raw)
Expand Down
25 changes: 24 additions & 1 deletion tokeneconomizer/plugin/fold.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import shutil
import tempfile
import time
import traceback

from .. import trace
from . import accounting, denylist, sidecall
Expand Down Expand Up @@ -64,6 +65,28 @@ def _run_dir(workdir, trace_path):
return run


def _noted(root, runner):
"""The proposer's side call, with a failure written to `errors.log`. The distiller skips a
call that raises and the fold goes on, so nothing else would show that it failed."""
if runner is None:
return None

def call(*args, **kwargs):
try:
return runner(*args, **kwargs)
except Exception:
try:
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "errors.log"), "a", encoding="utf-8") as fh:
fh.write(f"--- {time.strftime('%Y-%m-%dT%H:%M:%S')} pid={os.getpid()} "
f"(proposer side call)\n")
fh.write(traceback.format_exc())
except Exception:
pass
raise
return call


def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported=False):
profile = ctx.config.microskills
if profile == "off":
Expand All @@ -77,7 +100,7 @@ def _fold_microskills(ctx, adapter, run_dir, session_id, injected=None, imported
# Paused once the host's CLI said it is not signed in (`sidecall.paused`): the free layer
# still learns. A session read from the agent's history never calls a model: the free
# layer learns from it, and the proposer waits for sessions the plugin sees end.
runner = (sidecall.runner_for(ctx.host.name, ctx.config)
runner = (_noted(ctx.root, sidecall.runner_for(ctx.host.name, ctx.config))
if icfg.paid and not imported and not sidecall.paused(ctx.host.name, ctx.root)
else None)
stage = PrefixStage(icfg, ctx.repo.label, ctx.mech_dir, False, runner=runner,
Expand Down
5 changes: 5 additions & 0 deletions tokeneconomizer/plugin/sidecall.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,11 @@ def claude_call(prompt, model="haiku", cli=None, timeout=TIMEOUT_S):
data = json.loads(out.stdout or "{}")
except ValueError:
data = {}
# With verbose on, Claude Code streams an array of events instead of one
# result object (#131): take the result event, as the single object would be.
if isinstance(data, list):
data = next((ev for ev in reversed(data)
if isinstance(ev, dict) and ev.get("type") == "result"), {})
if not data.get("result"):
_no_reply("claude", out)
usage = dict(data.get("usage") or {})
Expand Down
Loading