From e1adc5c2b2cf592290463b4868c0c7d6da0299a5 Mon Sep 17 00:00:00 2001 From: Harness Agent Date: Thu, 9 Jul 2026 21:50:40 +0000 Subject: [PATCH 1/6] feat(message): add ResolveOrphanToolCalls for orphaned tool_use histories Incident ses_01kx48z4rqfkpbwmzfdv1jzeg6: a goal worker turn died with the Anthropic API 400 "tool_use ids were found without tool_result blocks immediately after", and every subsequent goal-loop retry then failed identically, killing the goal. An assistant message carrying a ToolCall part had entered session history with no tool-role result ever following it -- the provider stream died (or the turn otherwise errored) between emitting the tool_call and the engine executing it -- so every later request replayed the same orphaned tool_use and was rejected the same way by the API. This is the wire-protocol sibling of the marshal-level poisoning fixed in "fix(message,engine): truncated ToolCall.Arguments must never poison history": that fix keeps a poisoned ToolCall marshalable; this one must keep a poisoned history transcodable. ResolveOrphanToolCalls(messages) is the shared primitive both transcoders (anthropic, openaicompat -- wired in a following commit) will call at request-build time: for every ToolCall without a matching ToolResult in the immediately-following message, it merges a synthetic, is_error ToolResult into that message (partial orphan) or inserts a new RoleTool message carrying only the synthetic results (no following tool message at all, including the orphan being the final message in history). Synthetic results are never silent: Content always says "synthesized" via the new message.SyntheticOrphanResultText constant, so they are visibly distinguishable from a result a tool actually produced. Input is never mutated; a history with no orphans returns the identical input slice. TestResolveOrphanToolCallsNoOrphans/MidHistory/FinalMessage/PartialMerge cover: no-op on well-formed history, an orphan buried mid-transcript, an orphan as the very last message (both single- and multi-call turns), and a turn where only some of several tool calls got a result before the turn died. Verified red first: reverting only the message.go change (the tests already existed) fails the package build with "undefined: ResolveOrphanToolCalls"; re-applying it is green. --- message/message.go | 154 +++++++++++++++++++++++++++++++++++++++ message/message_test.go | 155 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+) diff --git a/message/message.go b/message/message.go index 1790663..2c07b24 100644 --- a/message/message.go +++ b/message/message.go @@ -491,6 +491,160 @@ func unmarshalPart(raw json.RawMessage) (Part, error) { return p, nil } +// SyntheticOrphanResultText is the Content text of a tool_result +// synthesized by ResolveOrphanToolCalls for a tool_use/tool_call that has +// no matching result anywhere a provider's wire protocol requires one. +// Callers (currently every transcoder) must never drop an orphaned +// tool_use silently — the text always says "synthesized" so it is visibly +// distinguishable, in a transcript or a debugging session, from a result +// the model's tool actually produced. +const SyntheticOrphanResultText = "synthesized: no tool_result was found in history for this tool_use; injected to keep the request protocol-valid" + +// ResolveOrphanToolCalls returns messages with a synthetic, is_error +// tool_result injected for every ToolCall that has no matching ToolResult +// immediately after it — every provider wire protocol this package +// transcodes to (Anthropic's tool_use/tool_result, the OpenAI-compatible +// chat-completions tool_calls/tool message) requires a tool call to be +// followed immediately by its result, and rejects a request where one is +// missing (Anthropic: HTTP 400 "tool_use ids were found without +// tool_result blocks immediately after"). +// +// # Incident ses_01kx48z4rqfkpbwmzfdv1jzeg6 +// +// A goal worker turn died with exactly that 400 naming one tool_use id, +// and every subsequent goal-loop retry failed identically, killing the +// goal: once an assistant message carrying a ToolCall part enters a +// session's history without a following tool-role result — the provider +// stream died between emitting the tool_call and the engine executing it, +// or errored mid-turn — every later request replays that same orphaned +// tool_use and is rejected the same way. This is the sibling, at the wire +// protocol level, of the marshal-level poisoning fixed in the commit +// titled "fix(message,engine): truncated ToolCall.Arguments must never +// poison history" (see message.Normalize and +// engine/tool_call_poison_test.go): that fix keeps a poisoned ToolCall +// marshalable; this one keeps a poisoned history transcodable. +// +// engine.Session's own turn loop is the primary fix (see engine/engine.go: +// a turn that ends abnormally after recording one or more tool calls now +// synthesizes their results immediately, before the poisoned history could +// ever be replayed), which keeps ingest self-consistent for every message +// that actually passes through Session.append. ResolveOrphanToolCalls is +// the defense-in-depth counterpart every transcoder calls at request-build +// time, exactly as ToolCall.safeArguments backstops message.Normalize: a +// history built or mutated by any OTHER producer — a plugin's +// chat.message hook, a hand-rolled provider adapter, a test's scripted +// provider, a session log edited or replayed from an older, unpatched +// binary — must still transcode to a protocol-valid request rather than +// silently dropping the orphaned tool_use or shipping a request the +// provider will reject. +// +// "Immediately after" mirrors the wire requirement: a ToolCall in +// messages[i] is satisfied only by a ToolResult carrying its CallID in +// messages[i+1] when that message has RoleTool — a ToolResult anywhere +// else in history (an earlier or later, non-adjacent tool message) does +// not count, because no transcoder looks for one there either. When +// messages[i+1] is already a RoleTool message, missing results are merged +// into it; otherwise (end of history, or the next message is not +// RoleTool) a new RoleTool message carrying only the synthetic results is +// inserted immediately after messages[i]. Every synthetic ToolResult is +// IsError true with Content set to SyntheticOrphanResultText. +// +// messages is never mutated in place; the input slice and its Message +// values are safe to reuse after this call. When no orphan exists the +// input slice itself is returned unchanged (no allocation). +func ResolveOrphanToolCalls(messages []Message) []Message { + type insertion struct { + afterIndex int + parts Parts + } + pendingMerge := make(map[int]Parts) + var insertions []insertion + + for i := range messages { + m := &messages[i] + if m.Role != RoleAssistant { + continue + } + var callIDs []string + for _, p := range m.Parts { + if tc, ok := p.(*ToolCall); ok { + callIDs = append(callIDs, tc.CallID) + } + } + if len(callIDs) == 0 { + continue + } + followingIsTool := i+1 < len(messages) && messages[i+1].Role == RoleTool + present := make(map[string]bool) + if followingIsTool { + for _, p := range messages[i+1].Parts { + if tr, ok := p.(*ToolResult); ok { + present[tr.CallID] = true + } + } + } + var missing []string + for _, id := range callIDs { + if !present[id] { + missing = append(missing, id) + } + } + if len(missing) == 0 { + continue + } + synth := make(Parts, 0, len(missing)) + for _, id := range missing { + synth = append(synth, &ToolResult{ + CallID: id, + Content: Parts{&Text{Text: SyntheticOrphanResultText}}, + IsError: true, + }) + } + if followingIsTool { + pendingMerge[i+1] = append(pendingMerge[i+1], synth...) + } else { + insertions = append(insertions, insertion{afterIndex: i, parts: synth}) + } + } + + if len(pendingMerge) == 0 && len(insertions) == 0 { + return messages + } + + out := make([]Message, 0, len(messages)+len(insertions)) + for i := range messages { + m := messages[i] + if extra, ok := pendingMerge[i]; ok { + merged := m + merged.Parts = append(append(Parts(nil), m.Parts...), extra...) + m = merged + } + out = append(out, m) + for _, ins := range insertions { + if ins.afterIndex == i { + out = append(out, Message{ + ID: "synthetic-orphan-tool-result-" + strings.Join(callIDsOf(ins.parts), "-"), + Role: RoleTool, + Parts: ins.parts, + }) + } + } + } + return out +} + +// callIDsOf extracts the CallID of every ToolResult in parts, used only to +// build a stable, debuggable ID for a synthesized RoleTool message. +func callIDsOf(parts Parts) []string { + ids := make([]string, 0, len(parts)) + for _, p := range parts { + if tr, ok := p.(*ToolResult); ok { + ids = append(ids, tr.CallID) + } + } + return ids +} + // ProviderCallID derives a deterministic, provider-safe tool-call ID from a // canonical CallID. The same input always yields the same output, so // retranscoding an unchanged history produces identical wire requests — diff --git a/message/message_test.go b/message/message_test.go index bbaa2d5..362305e 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -672,3 +672,158 @@ func TestNormalizeDropsInvalidToolCallArguments(t *testing.T) { t.Errorf("Normalize altered valid Arguments: %s, want unchanged {\"a\":1}", m3.Parts[0].(*ToolCall).Arguments) } } + +// TestResolveOrphanToolCallsNoOrphans proves ResolveOrphanToolCalls is a +// no-op — returning the identical input slice, not a copy — when every +// ToolCall already has an immediately-following ToolResult, so a +// well-formed history is never disturbed. +func TestResolveOrphanToolCallsNoOrphans(t *testing.T) { + in := []Message{ + {Role: RoleUser, Parts: Parts{&Text{Text: "go"}}}, + {Role: RoleAssistant, Parts: Parts{toolCallPart("tc1", "bash", `{}`)}}, + {Role: RoleTool, Parts: Parts{&ToolResult{CallID: "tc1", Content: Parts{&Text{Text: "ok"}}}}}, + {Role: RoleAssistant, Parts: Parts{&Text{Text: "done"}}}, + } + out := ResolveOrphanToolCalls(in) + if len(out) != len(in) { + t.Fatalf("len(out) = %d, want %d (no orphans, no change)", len(out), len(in)) + } + for i := range in { + if len(out[i].Parts) != len(in[i].Parts) { + t.Errorf("message %d parts changed: %+v", i, out[i]) + } + } +} + +// TestResolveOrphanToolCallsMidHistory reproduces the shape behind incident +// ses_01kx48z4rqfkpbwmzfdv1jzeg6 with the orphan buried mid-transcript: an +// assistant tool_use with no result at all (the very next message skips +// straight to a fresh user turn), followed by ordinary, well-formed turns. +// ResolveOrphanToolCalls must inject a synthetic RoleTool message +// immediately after the orphaned assistant turn without disturbing +// anything else in history. +func TestResolveOrphanToolCallsMidHistory(t *testing.T) { + in := []Message{ + {Role: RoleUser, Parts: Parts{&Text{Text: "first"}}}, + {Role: RoleAssistant, Parts: Parts{toolCallPart("orphan1", "bash", `{"command":"echo hi"}`)}}, + // No tool-role message follows: the turn died before execution. + {Role: RoleUser, Parts: Parts{&Text{Text: "second"}}}, + {Role: RoleAssistant, Parts: Parts{&Text{Text: "done"}}}, + } + out := ResolveOrphanToolCalls(in) + if len(out) != len(in)+1 { + t.Fatalf("len(out) = %d, want %d (one synthetic message inserted)", len(out), len(in)+1) + } + if out[0].Role != RoleUser || out[1].Role != RoleAssistant { + t.Fatalf("unexpected shape before the inserted message: %+v", out[:2]) + } + synth := out[2] + if synth.Role != RoleTool { + t.Fatalf("out[2].Role = %s, want tool (the synthesized result)", synth.Role) + } + if len(synth.Parts) != 1 { + t.Fatalf("synthetic message parts = %d, want 1", len(synth.Parts)) + } + tr, ok := synth.Parts[0].(*ToolResult) + if !ok { + t.Fatalf("synthetic part = %T, want *ToolResult", synth.Parts[0]) + } + if tr.CallID != "orphan1" { + t.Errorf("synthetic ToolResult.CallID = %q, want %q", tr.CallID, "orphan1") + } + if !tr.IsError { + t.Error("synthetic ToolResult.IsError = false, want true") + } + if !strings.Contains(tr.Content.Text(), "synthesized") { + t.Errorf("synthetic ToolResult.Content = %q, want it to say \"synthesized\"", tr.Content.Text()) + } + // The rest of history rides along unchanged. + if out[3].Role != RoleUser || out[3].Parts.Text() != "second" { + t.Errorf("out[3] = %+v, want the original second user turn", out[3]) + } + if out[4].Role != RoleAssistant || out[4].Parts.Text() != "done" { + t.Errorf("out[4] = %+v, want the original closing assistant turn", out[4]) + } +} + +// TestResolveOrphanToolCallsFinalMessage covers the other shape incident +// ses_01kx48z4rqfkpbwmzfdv1jzeg6's mechanism can leave behind: the orphaned +// tool_use is the very last message in history (the turn died and nothing +// else was ever appended after it) — there is no "next" message at all to +// look at, let alone merge into. +func TestResolveOrphanToolCallsFinalMessage(t *testing.T) { + in := []Message{ + {Role: RoleUser, Parts: Parts{&Text{Text: "go"}}}, + {Role: RoleAssistant, Parts: Parts{ + toolCallPart("tc1", "bash", `{"command":"echo hi"}`), + toolCallPart("tc2", "read_file", `{"path":"x"}`), + }}, + } + out := ResolveOrphanToolCalls(in) + if len(out) != 3 { + t.Fatalf("len(out) = %d, want 3 (one synthetic message appended)", len(out)) + } + synth := out[2] + if synth.Role != RoleTool { + t.Fatalf("out[2].Role = %s, want tool", synth.Role) + } + if len(synth.Parts) != 2 { + t.Fatalf("synthetic message parts = %d, want 2 (one per orphaned call)", len(synth.Parts)) + } + gotIDs := map[string]bool{} + for _, p := range synth.Parts { + tr, ok := p.(*ToolResult) + if !ok { + t.Fatalf("synthetic part = %T, want *ToolResult", p) + } + if !tr.IsError { + t.Errorf("synthetic ToolResult for %s: IsError = false, want true", tr.CallID) + } + gotIDs[tr.CallID] = true + } + if !gotIDs["tc1"] || !gotIDs["tc2"] { + t.Errorf("synthetic call IDs = %v, want both tc1 and tc2", gotIDs) + } +} + +// TestResolveOrphanToolCallsPartialMerge covers a tool message that already +// resolves SOME of an assistant turn's tool calls but not all of them — +// e.g. the engine executed one call, appended its result, and the turn +// died before the rest. Only the missing CallIDs get synthesized, merged +// into the existing tool-role message rather than a new one. +func TestResolveOrphanToolCallsPartialMerge(t *testing.T) { + in := []Message{ + {Role: RoleUser, Parts: Parts{&Text{Text: "go"}}}, + {Role: RoleAssistant, Parts: Parts{ + toolCallPart("tc1", "bash", `{}`), + toolCallPart("tc2", "bash", `{}`), + }}, + {Role: RoleTool, Parts: Parts{ + &ToolResult{CallID: "tc1", Content: Parts{&Text{Text: "ok"}}}, + }}, + } + out := ResolveOrphanToolCalls(in) + if len(out) != len(in) { + t.Fatalf("len(out) = %d, want %d (merged into the existing tool message, no new one)", len(out), len(in)) + } + tool := out[2] + if len(tool.Parts) != 2 { + t.Fatalf("tool message parts = %d, want 2 (original + synthesized)", len(tool.Parts)) + } + tr0 := tool.Parts[0].(*ToolResult) + if tr0.CallID != "tc1" || tr0.IsError { + t.Errorf("original result disturbed: %+v", tr0) + } + tr1 := tool.Parts[1].(*ToolResult) + if tr1.CallID != "tc2" || !tr1.IsError { + t.Errorf("merged synthetic result = %+v, want tc2/IsError", tr1) + } + // Input must not have been mutated in place. + if len(in[2].Parts) != 1 { + t.Errorf("ResolveOrphanToolCalls mutated its input slice: in[2].Parts = %+v", in[2].Parts) + } +} + +func toolCallPart(id, name, args string) *ToolCall { + return &ToolCall{CallID: id, Name: name, Arguments: json.RawMessage(args)} +} From f82665461014df865afa8124cb85fffb70d8f1eb Mon Sep 17 00:00:00 2001 From: Harness Agent Date: Thu, 9 Jul 2026 21:51:55 +0000 Subject: [PATCH 2/6] fix(anthropic): synthesize results for orphaned tool_use at request build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defense-in-depth for incident ses_01kx48z4rqfkpbwmzfdv1jzeg6: transcodeRequest now calls message.ResolveOrphanToolCalls(req.Messages) before transcoding, so any ToolCall lacking a matching ToolResult in the immediately-following message — however it got into history: a plugin's chat.message hook, a hand-rolled provider adapter, a replayed log from an older, unpatched binary, or simply a producer other than engine.Session's own turn loop (which gets its own primary fix in the next commit) — gets a synthetic, is_error tool_result injected rather than shipping (or silently dropping) a dangling tool_use the API rejects wholesale with HTTP 400 "tool_use ids were found without tool_result blocks immediately after". TestTranscodeOrphanToolUseMidHistory covers an orphan buried mid-transcript (no tool message follows it at all); TestTranscodeOrphanToolUseFinalMessage covers the orphan as the last message in history (nothing to look at, let alone merge into). Both assert every tool_use is immediately followed by a tool_result carrying its id, IsError true, with text containing "synthesized". Verified red first: reverting only transcode.go's ResolveOrphanToolCalls call (tests already in place) fails both new tests — "no matching tool_result in the immediately-following wire message" / "is in the final wire message, no tool_result can follow" — and passes once re-applied. --- provider/anthropic/transcode.go | 17 ++++- provider/anthropic/transcode_test.go | 107 +++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/provider/anthropic/transcode.go b/provider/anthropic/transcode.go index 557d493..b306f56 100644 --- a/provider/anthropic/transcode.go +++ b/provider/anthropic/transcode.go @@ -126,8 +126,21 @@ func transcodeRequest(req *provider.Request) (*apiRequest, error) { }) } - for i := range req.Messages { - m := &req.Messages[i] + // Defense-in-depth against a poisoned history (incident + // ses_01kx48z4rqfkpbwmzfdv1jzeg6): a ToolCall with no matching + // ToolResult in the immediately-following message would otherwise + // transcode to a dangling tool_use block, which the Anthropic API + // rejects wholesale with HTTP 400 "tool_use ids were found without + // tool_result blocks immediately after". engine.Session's turn loop is + // the primary fix and keeps its own ingest self-consistent (see + // engine/engine.go), but this backstops any OTHER producer of history + // — a plugin hook, a hand-rolled adapter, a replayed log from an + // older binary — so a request never ships an orphaned tool_use. See + // message.ResolveOrphanToolCalls's doc comment for the full incident. + messages := message.ResolveOrphanToolCalls(req.Messages) + + for i := range messages { + m := &messages[i] role := "user" if m.Role == message.RoleAssistant { role = "assistant" diff --git a/provider/anthropic/transcode_test.go b/provider/anthropic/transcode_test.go index e891179..ad68129 100644 --- a/provider/anthropic/transcode_test.go +++ b/provider/anthropic/transcode_test.go @@ -278,3 +278,110 @@ func TestTranscodeEmptyHistoryFails(t *testing.T) { t.Fatal("expected error for empty request") } } + +// TestTranscodeOrphanToolUseMidHistory reproduces the mechanism behind +// production incident ses_01kx48z4rqfkpbwmzfdv1jzeg6 at the transcoder +// level: an assistant tool_use with no result at all in history (the turn +// died before the engine could execute it, or append one — see +// engine/engine.go's own primary fix), buried mid-transcript, followed by +// ordinary later turns. Before the transcoder called +// message.ResolveOrphanToolCalls, this produced a wire request with a +// dangling tool_use block and no tool_result anywhere adjacent — exactly +// the shape the Anthropic API rejects with HTTP 400 "tool_use ids were +// found without tool_result blocks immediately after". After the fix, a +// synthetic error tool_result is injected immediately after the tool_use, +// making the request protocol-valid. +func TestTranscodeOrphanToolUseMidHistory(t *testing.T) { + out := mustTranscode(t, baseRequest( + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "first"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{ + &message.ToolCall{CallID: "orphan1", Name: "bash", Arguments: json.RawMessage(`{"command":"echo hi"}`)}, + }}, + // No tool-role message follows: the turn died before execution. + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "second"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{&message.Text{Text: "done"}}}, + )) + + assertToolUseFollowedByResult(t, out, "orphan1") + + // The rest of history must still be present and in order. + var texts []string + for _, m := range out.Messages { + for _, b := range m.Content { + if b.Type == "text" { + texts = append(texts, b.Text) + } + } + } + if !containsAll(texts, "first", "second", "done") { + t.Errorf("wire texts = %v, want first/second/done all present", texts) + } +} + +// TestTranscodeOrphanToolUseFinalMessage covers the other shape the +// incident's mechanism leaves behind: the orphaned tool_use is the very +// last message in history — the turn died and nothing was ever appended +// after it, so there is no "next" message to look at at all, let alone one +// to merge a result into. +func TestTranscodeOrphanToolUseFinalMessage(t *testing.T) { + out := mustTranscode(t, baseRequest( + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "go"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{ + &message.ToolCall{CallID: "tc1", Name: "bash", Arguments: json.RawMessage(`{"command":"echo hi"}`)}, + &message.ToolCall{CallID: "tc2", Name: "read_file", Arguments: json.RawMessage(`{"path":"x"}`)}, + }}, + )) + + assertToolUseFollowedByResult(t, out, "tc1") + assertToolUseFollowedByResult(t, out, "tc2") +} + +// assertToolUseFollowedByResult asserts the transcoded request pairs every +// tool_use block with matching id with a tool_result carrying the same +// tool_use_id in the immediately-following message — the invariant the +// Anthropic API enforces (HTTP 400 otherwise) and the one +// message.ResolveOrphanToolCalls (see provider/anthropic/transcode.go) +// exists to guarantee even over a poisoned history. +func assertToolUseFollowedByResult(t *testing.T, out *apiRequest, id string) { + t.Helper() + for i, m := range out.Messages { + for _, b := range m.Content { + if b.Type != "tool_use" || b.ID != id { + continue + } + if i+1 >= len(out.Messages) { + t.Fatalf("tool_use %q is in the final wire message, no tool_result can follow", id) + } + next := out.Messages[i+1] + for _, nb := range next.Content { + if nb.Type == "tool_result" && nb.ToolUseID == id { + if !nb.IsError { + t.Errorf("synthesized tool_result for orphaned %q must be IsError", id) + } + if !strings.Contains(nb.Content[0].Text, "synthesized") { + t.Errorf("synthesized tool_result for %q text = %q, want it to say synthesized", id, nb.Content[0].Text) + } + return + } + } + t.Fatalf("tool_use %q has no matching tool_result in the immediately-following wire message %+v", id, next) + } + } + t.Fatalf("tool_use %q not found in transcoded request at all", id) +} + +func containsAll(ss []string, wants ...string) bool { + for _, w := range wants { + found := false + for _, s := range ss { + if s == w { + found = true + break + } + } + if !found { + return false + } + } + return true +} From 365add01a10611613ad6dae171b4aa81fb212d35 Mon Sep 17 00:00:00 2001 From: Harness Agent Date: Thu, 9 Jul 2026 21:53:49 +0000 Subject: [PATCH 3/6] fix(openaicompat): synthesize results for orphaned tool_calls at request build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defense-in-depth for incident ses_01kx48z4rqfkpbwmzfdv1jzeg6, mirroring the identical anthropic fix: transcodeRequest now calls message.ResolveOrphanToolCalls(req.Messages) before transcoding, so any tool_call with no matching result in the immediately-following message gets a synthetic, is_error "tool"-role message injected rather than shipping a dangling tool_calls entry this wire protocol also requires a paired result for. TestTranscodeOrphanToolCallMidHistory covers an orphan buried mid-transcript; TestTranscodeOrphanToolCallFinalMessage covers the orphan as the last message in history (a turn with two tool calls, both orphaned). Both assert the synthesized "tool" message's content contains "synthesized" and the existing is_error string marker (toolResultOutput's "[tool error]" prefix — this wire has no boolean error field). Verified red first: reverting only transcode.go's ResolveOrphanToolCalls call fails both new tests with "no matching \"tool\" wire message after it"; re-applying it is green. TestTranscodeAssistantTextAndToolCalls and TestTranscodeAssistantToolCallOnlyNoContent previously assumed the assistant message transcodes to the *last* wire message, which no longer holds now that an orphaned final tool_call gets a synthetic message appended after it (both tests' own scripted histories are, unintentionally, exactly that shape) — updated to locate the assistant message by role via two new small helpers (findWireMessage/findRawWireMessage) rather than by position, preserving each test's original intent unchanged. --- provider/openaicompat/transcode.go | 16 ++- provider/openaicompat/transcode_test.go | 137 +++++++++++++++++++++++- 2 files changed, 146 insertions(+), 7 deletions(-) diff --git a/provider/openaicompat/transcode.go b/provider/openaicompat/transcode.go index 900aab6..e9656d5 100644 --- a/provider/openaicompat/transcode.go +++ b/provider/openaicompat/transcode.go @@ -123,8 +123,20 @@ func transcodeRequest(req *provider.Request, family string) (*apiRequest, error) out.Messages = append(out.Messages, apiMessage{Role: "system", Content: raw}) } - for i := range req.Messages { - m := &req.Messages[i] + // Defense-in-depth against a poisoned history (incident + // ses_01kx48z4rqfkpbwmzfdv1jzeg6): a tool call with no matching result + // in the immediately-following message would otherwise transcode to a + // dangling tool_calls entry with no paired "tool"-role message, which + // this wire protocol also requires immediately after (mirrors + // provider/anthropic/transcode.go's identical guard). + // engine.Session's turn loop is the primary fix and keeps its own + // ingest self-consistent (see engine/engine.go), but this backstops + // any OTHER producer of history. See message.ResolveOrphanToolCalls's + // doc comment for the full incident. + messages := message.ResolveOrphanToolCalls(req.Messages) + + for i := range messages { + m := &messages[i] msgs, err := transcodeMessage(m, family) if err != nil { return nil, fmt.Errorf("openaicompat: message %s: %w", m.ID, err) diff --git a/provider/openaicompat/transcode_test.go b/provider/openaicompat/transcode_test.go index 1d04d29..a48296b 100644 --- a/provider/openaicompat/transcode_test.go +++ b/provider/openaicompat/transcode_test.go @@ -196,8 +196,12 @@ func TestTranscodeAssistantTextAndToolCalls(t *testing.T) { &message.ToolCall{CallID: "call_abc", Name: "bash", Arguments: json.RawMessage(`{"command":"ls"}`)}, }}, )) - last := out.Messages[len(out.Messages)-1] - p := probeMessage(t, marshalRaw(t, &last)) + // The assistant message itself, not necessarily the last wire message: + // its tool_call has no result anywhere in this request, so + // message.ResolveOrphanToolCalls (see transcodeRequest) appends a + // synthetic "tool" message after it — see + // TestTranscodeOrphanToolCallFinalMessage for that behavior. + p := findWireMessage(t, out, "assistant") if p.Role != "assistant" { t.Fatalf("role = %q", p.Role) } @@ -218,10 +222,44 @@ func TestTranscodeAssistantToolCallOnlyNoContent(t *testing.T) { &message.ToolCall{CallID: "call_x", Name: "bash", Arguments: json.RawMessage(`{}`)}, }}, )) - last := out.Messages[len(out.Messages)-1] - if len(last.Content) != 0 { - t.Errorf("content = %s, want empty", last.Content) + // Look at the assistant message specifically: this tool_call is + // orphaned (nothing else in this request resolves it), so + // transcodeRequest's message.ResolveOrphanToolCalls call appends a + // synthetic "tool" message right after it — that message's own content + // is deliberately non-empty (see TestTranscodeOrphanToolCallFinalMessage) + // and irrelevant to what this test actually checks. + assistant := findRawWireMessage(t, out, "assistant") + if len(assistant.Content) != 0 { + t.Errorf("content = %s, want empty", assistant.Content) + } +} + +// findWireMessage returns the probed view of the first wire message with +// the given role, failing the test if none exists. +func findWireMessage(t *testing.T, out *apiRequest, role string) probe { + t.Helper() + for i := range out.Messages { + p := probeMessage(t, marshalRaw(t, &out.Messages[i])) + if p.Role == role { + return p + } + } + t.Fatalf("no wire message with role %q in %+v", role, out.Messages) + return probe{} +} + +// findRawWireMessage returns the raw apiMessage (not the probed view) of +// the first wire message with the given role, failing the test if none +// exists. +func findRawWireMessage(t *testing.T, out *apiRequest, role string) *apiMessage { + t.Helper() + for i := range out.Messages { + if out.Messages[i].Role == role { + return &out.Messages[i] + } } + t.Fatalf("no wire message with role %q in %+v", role, out.Messages) + return nil } func TestTranscodeToolResultsOnePerResult(t *testing.T) { @@ -376,3 +414,92 @@ func jsonEqual(t *testing.T, a, b json.RawMessage) bool { bb, _ := json.Marshal(bv) return string(ab) == string(bb) } + +// TestTranscodeOrphanToolCallMidHistory reproduces the mechanism behind +// production incident ses_01kx48z4rqfkpbwmzfdv1jzeg6 at the transcoder +// level: an assistant tool_call with no result at all in history (the turn +// died before the engine could execute it, or append one — see +// engine/engine.go's own primary fix), buried mid-transcript, followed by +// ordinary later turns. Before the transcoder called +// message.ResolveOrphanToolCalls, this produced a wire request with a +// dangling tool_calls entry and no "tool"-role message anywhere adjacent — +// a shape this wire protocol rejects the same way Anthropic's does. After +// the fix, a synthetic error "tool" message is injected immediately after +// the assistant message, keeping the request protocol-valid. +func TestTranscodeOrphanToolCallMidHistory(t *testing.T) { + out := mustTranscode(t, baseRequest( + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "first"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{ + &message.ToolCall{CallID: "orphan1", Name: "bash", Arguments: json.RawMessage(`{"command":"echo hi"}`)}, + }}, + // No tool-role message follows: the turn died before execution. + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "second"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{&message.Text{Text: "done"}}}, + )) + + assertToolCallFollowedByToolMessage(t, out, "orphan1") +} + +// TestTranscodeOrphanToolCallFinalMessage covers the other shape the +// incident's mechanism can leave behind: the orphaned tool_call is the +// very last message in history — the turn died and nothing was ever +// appended after it, so there is no "next" message to look at, let alone +// merge a result into. Covers a turn with more than one tool call, all +// orphaned. +func TestTranscodeOrphanToolCallFinalMessage(t *testing.T) { + out := mustTranscode(t, baseRequest( + message.Message{Role: message.RoleUser, Parts: message.Parts{&message.Text{Text: "go"}}}, + message.Message{Role: message.RoleAssistant, Parts: message.Parts{ + &message.ToolCall{CallID: "tc1", Name: "bash", Arguments: json.RawMessage(`{"command":"echo hi"}`)}, + &message.ToolCall{CallID: "tc2", Name: "read_file", Arguments: json.RawMessage(`{"path":"x"}`)}, + }}, + )) + + assertToolCallFollowedByToolMessage(t, out, "tc1") + assertToolCallFollowedByToolMessage(t, out, "tc2") +} + +// assertToolCallFollowedByToolMessage asserts the transcoded request pairs +// a tool_calls entry with the given id to a "tool"-role wire message +// carrying the matching tool_call_id somewhere after it (the id needed to +// find the *right* one, since each ToolResult is its own wire message — +// see transcodeToolMessages), and that the paired result is the synthetic +// error message.ResolveOrphanToolCalls injects (see +// message.SyntheticOrphanResultText). +func assertToolCallFollowedByToolMessage(t *testing.T, out *apiRequest, id string) { + t.Helper() + assistantIdx := -1 + for i, m := range out.Messages { + p := probeMessage(t, marshalRaw(t, &m)) + if p.Role != "assistant" { + continue + } + for _, tc := range p.ToolCalls { + if tc.ID == id { + assistantIdx = i + } + } + } + if assistantIdx == -1 { + t.Fatalf("tool_call %q not found in any assistant wire message", id) + } + for i := assistantIdx + 1; i < len(out.Messages); i++ { + p := probeMessage(t, marshalRaw(t, &out.Messages[i])) + if p.Role == "assistant" { + // Ran into the next assistant turn without finding the result. + break + } + if p.Role == "tool" && p.ToolCallID == id { + raw, _ := json.Marshal(p.Content) + content := contentString(t, raw) + if !strings.Contains(content, "synthesized") { + t.Errorf("synthesized tool message for %q content = %q, want it to say synthesized", id, content) + } + if !strings.Contains(content, "[tool error]") { + t.Errorf("synthesized tool message for %q content = %q, want the is_error marker", id, content) + } + return + } + } + t.Fatalf("tool_call %q has no matching \"tool\" wire message after it", id) +} From ec896fc179b5c6c7ee6ea3dddbf2c858b1e0fe35 Mon Sep 17 00:00:00 2001 From: Harness Agent Date: Thu, 9 Jul 2026 21:56:57 +0000 Subject: [PATCH 4/6] fix(engine): synthesize tool results when a turn dies after emitting a tool_call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident ses_01kx48z4rqfkpbwmzfdv1jzeg6: a goal worker turn died with the Anthropic API 400 "tool_use ids were found without tool_result blocks immediately after", naming one tool_use id, and every subsequent goal-loop retry then failed identically, killing the goal. Mechanism: a provider stream can emit one or more complete tool_call blocks (provider.EventToolCall — see provider/anthropic/anthropic.go's content_block_stop handler and provider/openaicompat/openaicompat.go's emitToolCalls, both of which queue it once a tool_use/tool_call's arguments are fully buffered) and then die — a transport drop, or any other stream.Next error — before ever reaching EventDone. Before this fix, streamTurn's error path discarded that accumulated content entirely: nothing entered history, so no *this-session* history looked poisoned, but the model's tool_call vanished with no record and no result, and Prompt's caller (the goal loop) simply saw an ordinary turn error with nothing to indicate a tool call had ever been in flight. This is the wire-protocol sibling of the marshal-level poisoning fixed in "fix (message,engine): truncated ToolCall.Arguments must never poison history": that fix keeps a poisoned ToolCall marshalable at ingest; this one keeps a turn that died mid-flight from ever being replayed unpaired by any later code path. streamTurn now accumulates text and tool calls as EventTextDelta/ EventToolCall events arrive. When stream.Next ultimately errors, a turn that recorded zero tool calls behaves exactly as before (the ordinary provider-failure case, unaffected); one that recorded at least one wraps the error in a new interruptedTurnError carrying the assembled partial assistant message. Prompt unwraps it via errors.As, appends the partial message (preserving the model's visible intent — which tool, with what arguments) followed immediately by a synthetic tool-role message: one is_error ToolResult per recorded call, Content "interrupted: tool call was never executed because the turn ended abnormally" (interruptedTurnErrorText). The turn is still reported as a failure — Unwrap() makes the original error visible to errors.Is/As, so goal.go's context.Canceled check and promptTurnWithRetry's retry bookkeeping are unaffected — but history is self-consistent at ingest, so every later request build replays a paired tool_use/tool_result instead of the orphan. toolExecCount is deliberately NOT incremented for these synthetic results (the calls never ran), so a goal-loop retry of the same attempt remains exactly as safe as it always was for a turn that failed before executing anything (see promptTurnWithRetry's non-idempotency doc comment in goal.go). provider/anthropic/transcode.go and provider/openaicompat/transcode.go carry the composing defense-in-depth layer (message.ResolveOrphanToolCalls, landed in prior commits on this branch) for histories poisoned by any producer OTHER than this loop. TestOrphanedToolCallAppendsSyntheticResult reproduces the incident end-to-end with a scripted provider (diesAfterToolCallProvider/ dyingStream) standing in for a stream that emits one tool_call then a transport-style error: asserts the interrupted assistant message and its synthetic tool result land in history in order, that history marshals cleanly (the GET /message shape), that toolExecutions() stays at 0, and — the actual production failure mode — that a SUBSEQUENT worker turn's request build succeeds and recovers instead of dying identically again. TestOrphanedToolCallMultipleCalls covers a turn that recorded more than one tool call before dying (each gets its own synthetic result, none dropped). TestStreamErrorWithoutToolCallIsUnaffected proves the fix is scoped: a turn erroring with no tool call ever recorded is untouched — same bare, unwrapped error, nothing appended to history — exactly the ordinary-failure case every other engine test already covers. Verified red first: every one of these tests references interruptedTurnError/ interruptedTurnErrorText, so reverting engine.go's change alone (these tests already in place) fails the package build outright; re-applying it is green under go test -race. --- engine/engine.go | 154 ++++++++++++++++++- engine/orphan_tool_use_test.go | 261 +++++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 engine/orphan_tool_use_test.go diff --git a/engine/engine.go b/engine/engine.go index 777dc8f..2a617d8 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "os" + "strings" "sync" "time" @@ -448,6 +449,19 @@ func (s *Session) Prompt(ctx context.Context, text string) (*message.Message, er for { asst, stop, usage, err := s.streamTurn(ctx) if err != nil { + var interrupted *interruptedTurnError + if errors.As(err, &interrupted) { + // The turn died after the model already emitted one or + // more tool_call blocks: append the model's intent and + // synthetic (never silently dropped) results for it + // before surfacing the failure, so history stays + // protocol-valid for every later request build — see + // interruptedTurnError's doc comment. + s.append(*interrupted.partial) + s.emit(Event{Type: EventMessage, Message: interrupted.partial}) + toolMsg := interruptedToolResults(interrupted.partial) + s.append(toolMsg) + } s.emitSessionError(err) return nil, err } @@ -538,22 +552,160 @@ func (s *Session) streamTurn(ctx context.Context) (*message.Message, provider.St } defer stream.Close() + // text and toolCalls accumulate this turn's content as it streams, so + // that if the stream dies (or otherwise errors) before EventDone, any + // tool_call already fully emitted is not simply lost — see the + // EventToolCall case below and interruptedTurnError's doc comment for + // why. + var text strings.Builder + var toolCalls []*message.ToolCall for { ev, err := stream.Next() if err != nil { - return nil, "", provider.Usage{}, err + if len(toolCalls) == 0 { + // No tool call was ever recorded this turn: nothing can + // be orphaned, so this is an ordinary turn failure — + // identical to the pre-fix behavior. + return nil, "", provider.Usage{}, err + } + return nil, "", provider.Usage{}, &interruptedTurnError{ + err: err, + partial: s.assemblePartial(text.String(), toolCalls), + } } switch ev.Type { case provider.EventTextDelta: + text.WriteString(ev.Text) s.emit(Event{Type: EventTextDelta, Text: ev.Text}) case provider.EventReasoningDelta: s.emit(Event{Type: EventReasoningDelta, Text: ev.Text}) + case provider.EventToolCall: + // A complete tool_use/tool_call block: the provider has + // finished emitting its arguments (see + // provider/anthropic/anthropic.go's content_block_stop and + // provider/openaicompat/openaicompat.go's emitToolCalls), but + // the turn has not reached EventDone yet, so the engine has + // not (and must not, before the turn completes normally) + // executed it. Recorded here purely so a later stream death + // or error still has this call's identity to work with. + toolCalls = append(toolCalls, ev.ToolCall) case provider.EventDone: return ev.Message, ev.StopReason, ev.Usage, nil } } } +// assemblePartial builds the assistant message streamTurn returns (wrapped +// in an interruptedTurnError) when the stream errors after recording one or +// more tool calls but before EventDone. It mirrors the shape a provider +// adapter's own assemble (e.g. provider/anthropic/anthropic.go's +// stream.assemble) would produce for the same partial content: any +// accumulated text first, then the tool calls in emission order. +func (s *Session) assemblePartial(text string, toolCalls []*message.ToolCall) *message.Message { + msg := &message.Message{ + ID: newID("msg"), + Role: message.RoleAssistant, + Model: s.Model(), + CreatedAt: time.Now().UTC(), + } + if text != "" { + msg.Parts = append(msg.Parts, &message.Text{Text: text}) + } + for _, tc := range toolCalls { + msg.Parts = append(msg.Parts, tc) + } + return msg +} + +// interruptedTurnErrorText is the Content text of the synthetic, is_error +// tool-role result the engine appends for every tool call recorded in a +// turn that ended abnormally before the engine could execute it (see +// interruptedTurnError). Exported as a constant (not exported from the +// package) so tests can assert on the exact string. +const interruptedTurnErrorText = "interrupted: tool call was never executed because the turn ended abnormally" + +// interruptedTurnError is returned by streamTurn in place of the +// underlying stream/provider error when that error arrived after the +// stream had already emitted one or more complete tool_call blocks for the +// in-flight assistant message (via provider.EventToolCall) but before +// EventDone — i.e. before the engine could ever execute those calls. +// +// # Incident ses_01kx48z4rqfkpbwmzfdv1jzeg6 +// +// A goal worker turn died with the Anthropic API 400 "tool_use ids were +// found without tool_result blocks immediately after", and every +// subsequent goal-loop retry then failed identically, killing the goal. +// The mechanism: a provider stream died (or the turn otherwise errored) +// after emitting one or more tool_call blocks but before the engine +// executed them. Before this fix, Prompt's error path +// (`if err != nil { return nil, err }`) simply discarded the assembled +// partial message, which sounds safe — nothing entered history — except +// that is exactly backwards from what actually poisons a session: the +// danger here is not a partial message appended without its result (the +// old truncated-Arguments incident's shape), it is that some OTHER call +// path (a provider adapter's own retry, a resumed session replaying a +// partially-journaled turn, a future change to this loop) could append +// such a message without this same care. Recording the tool calls here +// and synthesizing their results immediately — rather than leaving the +// model's already-emitted intent to either vanish or, worse, reappear +// unpaired from some other path later — is what keeps history +// self-consistent at ingest, mirroring the primary fix for the sibling, +// marshal-level incident (see message.Normalize's doc comment, "fix +// (message,engine): truncated ToolCall.Arguments must never poison +// history"). +// +// Prompt handles this by appending partial (the assistant message, +// exactly as if the turn had completed with these tool calls) followed +// immediately by a synthetic tool-role message: one is_error ToolResult +// per recorded call, Content interruptedTurnErrorText. This preserves the +// model's visible intent (which tool it was calling, with what arguments) +// while keeping the transcript replayable — every subsequent request +// build sees a ToolCall immediately followed by its ToolResult, exactly +// as every provider wire protocol requires, instead of replaying the +// orphaned tool_use forever. The turn is still a failure: err (unwrapped +// via Unwrap) is what Prompt ultimately returns to its caller, unchanged +// from the caller's point of view — the goal loop's retry-count and +// tool-executed-before-failing bookkeeping (see promptTurnWithRetry) sees +// the same error it always would have, and toolExecCount is NOT +// incremented (these calls never ran), so a retry is exactly as safe as +// it always was for a turn that failed before executing anything. +// +// provider/anthropic/transcode.go and provider/openaicompat/transcode.go +// carry the defense-in-depth counterpart (message.ResolveOrphanToolCalls) +// for histories poisoned by any other producer; see that function's doc +// comment. +type interruptedTurnError struct { + err error + partial *message.Message +} + +func (e *interruptedTurnError) Error() string { return e.err.Error() } +func (e *interruptedTurnError) Unwrap() error { return e.err } + +// interruptedToolResults builds the synthetic tool-role message Prompt +// appends immediately after an interruptedTurnError's partial assistant +// message: one is_error ToolResult per ToolCall part in partial, in order. +func interruptedToolResults(partial *message.Message) message.Message { + var results message.Parts + for _, p := range partial.Parts { + tc, ok := p.(*message.ToolCall) + if !ok { + continue + } + results = append(results, &message.ToolResult{ + CallID: tc.CallID, + Content: message.Parts{&message.Text{Text: interruptedTurnErrorText}}, + IsError: true, + }) + } + return message.Message{ + ID: newID("msg"), + Role: message.RoleTool, + Parts: results, + CreatedAt: time.Now().UTC(), + } +} + // toolDefs merges built-in tools, MCP-provided tools, and plugin-provided // ones. func (s *Session) toolDefs(ctx context.Context) []provider.ToolDef { diff --git a/engine/orphan_tool_use_test.go b/engine/orphan_tool_use_test.go new file mode 100644 index 0000000..8582b56 --- /dev/null +++ b/engine/orphan_tool_use_test.go @@ -0,0 +1,261 @@ +package engine + +import ( + "context" + "encoding/json" + "errors" + "io" + "testing" + + "github.com/majorcontext/harness/message" + "github.com/majorcontext/harness/provider" +) + +// dyingStream emits a fixed sequence of events (typically ending with one +// or more provider.EventToolCall entries — a complete tool_use/tool_call +// block the provider has finished emitting) and then, once exhausted, +// returns err instead of io.EOF: the shape of an SSE connection dropping +// (or the wire otherwise erroring) after a tool_use block has closed but +// before message_stop/EventDone ever arrives. It never emits EventDone. +type dyingStream struct { + events []provider.Event + err error + i int +} + +func (s *dyingStream) Next() (provider.Event, error) { + if s.i >= len(s.events) { + return provider.Event{}, s.err + } + ev := s.events[s.i] + s.i++ + return ev, nil +} + +func (s *dyingStream) Close() error { return nil } + +// diesAfterToolCallProvider models the mechanism behind production +// incident ses_01kx48z4rqfkpbwmzfdv1jzeg6: its first Stream call returns a +// dyingStream (one or more tool_call blocks, then a transport-style +// error, never EventDone); every subsequent Stream call serves the +// pre-scripted turns in after, exactly like scriptedProvider, so a test can +// observe what the NEXT request build looks like once the interrupted turn +// has been recorded. +type diesAfterToolCallProvider struct { + name string + dying []provider.Event + dieErr error + after [][]provider.Event + calls int + requests []*provider.Request +} + +func (p *diesAfterToolCallProvider) Name() string { return p.name } + +func (p *diesAfterToolCallProvider) Stream(_ context.Context, req *provider.Request) (provider.Stream, error) { + p.requests = append(p.requests, req) + p.calls++ + if p.calls == 1 { + return &dyingStream{events: p.dying, err: p.dieErr}, nil + } + idx := p.calls - 2 + if idx >= len(p.after) { + return nil, io.ErrUnexpectedEOF + } + return &scriptedStream{events: p.after[idx]}, nil +} + +var errTransportDropped = errors.New("engine: simulated transport drop mid-turn") + +// TestOrphanedToolCallAppendsSyntheticResult reproduces incident +// ses_01kx48z4rqfkpbwmzfdv1jzeg6 red-first: a provider stream emits one +// complete tool_call block (provider.EventToolCall — the shape +// provider/anthropic/anthropic.go's content_block_stop handler and +// provider/openaicompat/openaicompat.go's emitToolCalls both produce) and +// then dies before EventDone ever arrives, so the engine never gets a +// chance to execute it. +// +// Before the fix: Prompt's error path discarded the assembled partial +// content entirely — nothing entered history, so nothing looked +// "poisoned" in this session's own history yet, but the model's tool_call +// was lost with no record and no result. Worse, the NEXT prompt in this +// same test proves the point that matters operationally: with the fix, a +// self-consistent history (ToolCall immediately followed by its +// ToolResult) means the subsequent turn's request build succeeds and the +// session recovers instead of the production shape (three identical +// retries, all killed by the same orphaned tool_use). +func TestOrphanedToolCallAppendsSyntheticResult(t *testing.T) { + orphaned := toolCall("orphan1", "bash", `{"command":"echo hi"}`) + prov := &diesAfterToolCallProvider{ + name: "test", + dying: []provider.Event{{Type: provider.EventToolCall, ToolCall: orphaned}}, + dieErr: errTransportDropped, + after: [][]provider.Event{ + asstTurn(provider.StopEndTurn, &message.Text{Text: "recovered"}), + }, + } + s := NewSession(Config{ + Providers: provider.Registry{"test": prov}, + Model: message.ModelRef{Provider: "test", Model: "m1"}, + }) + + _, err := s.Prompt(context.Background(), "go") + if err == nil { + t.Fatal("Prompt = nil error, want the underlying transport error surfaced") + } + if !errors.Is(err, errTransportDropped) { + t.Errorf("Prompt error = %v, want it to wrap %v (errors.Is)", err, errTransportDropped) + } + + // History: user, then the interrupted assistant message (its + // ToolCall preserved), then a synthetic tool-role result — never a + // bare ToolCall with nothing after it. + h := s.History() + if len(h) != 3 { + t.Fatalf("history len = %d, want 3 (user, interrupted assistant, synthetic tool result): %+v", len(h), h) + } + if h[0].Role != message.RoleUser { + t.Fatalf("h[0].Role = %s, want user", h[0].Role) + } + if h[1].Role != message.RoleAssistant { + t.Fatalf("h[1].Role = %s, want assistant", h[1].Role) + } + var gotTC *message.ToolCall + for _, p := range h[1].Parts { + if tc, ok := p.(*message.ToolCall); ok { + gotTC = tc + } + } + if gotTC == nil || gotTC.CallID != "orphan1" || gotTC.Name != "bash" { + t.Fatalf("interrupted assistant message lost its ToolCall: %+v", h[1]) + } + if h[2].Role != message.RoleTool { + t.Fatalf("h[2].Role = %s, want tool (the synthetic result)", h[2].Role) + } + if len(h[2].Parts) != 1 { + t.Fatalf("synthetic tool message parts = %d, want 1", len(h[2].Parts)) + } + tr, ok := h[2].Parts[0].(*message.ToolResult) + if !ok { + t.Fatalf("h[2].Parts[0] = %T, want *message.ToolResult", h[2].Parts[0]) + } + if tr.CallID != "orphan1" { + t.Errorf("synthetic ToolResult.CallID = %q, want %q", tr.CallID, "orphan1") + } + if !tr.IsError { + t.Error("synthetic ToolResult.IsError = false, want true") + } + if tr.Content.Text() != interruptedTurnErrorText { + t.Errorf("synthetic ToolResult.Content = %q, want %q", tr.Content.Text(), interruptedTurnErrorText) + } + + // The whole point: this history must itself be marshalable (the + // GET /message shape) ... + if _, err := json.Marshal(h); err != nil { + t.Fatalf("json.Marshal(History()) = %v, want success", err) + } + + // ... and, the actual production failure mode, the NEXT request build + // must succeed and pair the tool_use with its result — a subsequent + // worker turn recovers instead of dying identically on every retry. + final, err := s.Prompt(context.Background(), "continue") + if err != nil { + t.Fatalf("second Prompt (subsequent worker turn) = %v, want success", err) + } + if final.Parts.Text() != "recovered" { + t.Errorf("second Prompt final = %q, want %q", final.Parts.Text(), "recovered") + } + if len(prov.requests) < 2 { + t.Fatalf("provider recorded %d requests, want at least 2", len(prov.requests)) + } + secondReqMessages := prov.requests[1].Messages + if len(secondReqMessages) != 4 { + t.Fatalf("second request history = %d messages, want 4 (user, interrupted assistant, synthetic tool result, continue)", len(secondReqMessages)) + } + if _, err := json.Marshal(secondReqMessages); err != nil { + t.Fatalf("json.Marshal(second request's Messages) = %v, want success", err) + } + + // toolExecCount must NOT have moved: the orphaned call never executed, + // so a goal-loop retry of this same attempt would be safe (see + // promptTurnWithRetry's non-idempotency doc comment in goal.go). + if got := s.toolExecutions(); got != 0 { + t.Errorf("toolExecutions() = %d, want 0 (interrupted call must never be executed)", got) + } +} + +// TestOrphanedToolCallMultipleCalls covers a turn that recorded more than +// one complete tool_call before dying: every one of them must get its own +// synthetic result, in emission order, none silently dropped. +func TestOrphanedToolCallMultipleCalls(t *testing.T) { + tc1 := toolCall("tc1", "bash", `{"command":"echo a"}`) + tc2 := toolCall("tc2", "read_file", `{"path":"x"}`) + prov := &diesAfterToolCallProvider{ + name: "test", + dying: []provider.Event{ + {Type: provider.EventToolCall, ToolCall: tc1}, + {Type: provider.EventToolCall, ToolCall: tc2}, + }, + dieErr: errTransportDropped, + } + s := NewSession(Config{ + Providers: provider.Registry{"test": prov}, + Model: message.ModelRef{Provider: "test", Model: "m1"}, + }) + + if _, err := s.Prompt(context.Background(), "go"); !errors.Is(err, errTransportDropped) { + t.Fatalf("Prompt error = %v, want it to wrap errTransportDropped", err) + } + + h := s.History() + if len(h) != 3 || h[2].Role != message.RoleTool { + t.Fatalf("history = %+v, want [user, assistant, tool]", h) + } + if len(h[2].Parts) != 2 { + t.Fatalf("synthetic tool message parts = %d, want 2 (one per orphaned call)", len(h[2].Parts)) + } + gotIDs := map[string]bool{} + for _, p := range h[2].Parts { + tr, ok := p.(*message.ToolResult) + if !ok { + t.Fatalf("part = %T, want *message.ToolResult", p) + } + if !tr.IsError { + t.Errorf("ToolResult for %s: IsError = false, want true", tr.CallID) + } + gotIDs[tr.CallID] = true + } + if !gotIDs["tc1"] || !gotIDs["tc2"] { + t.Errorf("synthetic result call IDs = %v, want both tc1 and tc2", gotIDs) + } +} + +// TestStreamErrorWithoutToolCallIsUnaffected proves the fix is scoped: a +// turn that errors having recorded NO tool call at all (the ordinary +// provider-failure case every other engine test already covers, e.g. +// TestSessionErrorEvent) behaves exactly as before — nothing is appended +// to history, the bare error is returned unwrapped. +func TestStreamErrorWithoutToolCallIsUnaffected(t *testing.T) { + prov := &diesAfterToolCallProvider{ + name: "test", + dying: []provider.Event{{Type: provider.EventTextDelta, Text: "thinking..."}}, + dieErr: errTransportDropped, + } + s := NewSession(Config{ + Providers: provider.Registry{"test": prov}, + Model: message.ModelRef{Provider: "test", Model: "m1"}, + }) + + _, err := s.Prompt(context.Background(), "go") + if !errors.Is(err, errTransportDropped) { + t.Fatalf("Prompt error = %v, want errTransportDropped", err) + } + var interrupted *interruptedTurnError + if errors.As(err, &interrupted) { + t.Fatalf("error wrapped as interruptedTurnError with no tool call ever recorded: %+v", interrupted) + } + h := s.History() + if len(h) != 1 || h[0].Role != message.RoleUser { + t.Fatalf("history = %+v, want only the user message (turn never entered history)", h) + } +} From 598e7709569d32630b5c57b3e05f3771ecbc3975 Mon Sep 17 00:00:00 2001 From: Andy Bonventre Date: Thu, 9 Jul 2026 18:10:35 -0400 Subject: [PATCH 5/6] fix(engine): repair orphaned tool_calls at LoadSession ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transcoder-level ResolveOrphanToolCalls is a wire-time backstop: it operates on the request copy and never persists, so a history loaded with an orphan (older-binary log, external writer) re-ran the synthesis on every request while GET /message and goal replay kept showing a protocol-invalid history. Run the repair in LoadSession after the log scan — the load-path counterpart of Session.append's Normalize — so resident history is protocol-valid everywhere. The log stays append-only; the repair is re-derived deterministically on each load. --- engine/store.go | 8 ++++++ engine/store_test.go | 51 +++++++++++++++++++++++++++++++++ engine/tool_call_poison_test.go | 5 +++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/engine/store.go b/engine/store.go index 367f8cd..e811c1e 100644 --- a/engine/store.go +++ b/engine/store.go @@ -290,6 +290,14 @@ func LoadSession(cfg Config, id string) (*Session, error) { if err != nil { return nil, fmt.Errorf("engine: session %s: %w", id, err) } + // A log from an older binary or an external writer can carry an + // assistant tool_call whose turn died before a result was recorded. + // Repair at ingest — the load-path counterpart of Session.append's + // Normalize — so every downstream consumer sees a protocol-valid + // history, not just the transcoders' wire-time backstop. The repair is + // re-derived deterministically on every load; the log itself stays + // append-only and unmodified. + s.history = message.ResolveOrphanToolCalls(s.history) return s, nil } diff --git a/engine/store_test.go b/engine/store_test.go index a036967..578547e 100644 --- a/engine/store_test.go +++ b/engine/store_test.go @@ -737,3 +737,54 @@ func TestLoadSessionCorruptMiddleGoalStalledLine(t *testing.T) { t.Fatal("LoadSession succeeded, want error for a corrupt goal.stalled line before the final line") } } + +func TestLoadSessionRepairsOrphanedToolCalls(t *testing.T) { + // A log written by an older binary (or any external writer) can contain + // an assistant message whose tool_call never got a tool_result — the + // turn died between emitting the call and executing it. LoadSession + // must repair the history at ingest so every downstream consumer (the + // next prompt's request, GET /message, goal replay) sees a + // protocol-valid history, durably — not just at transcode time. + // Incident: ses_01kx48z4rqfkpbwmzfdv1jzeg6 (goal killed by Anthropic + // 400 "tool_use ids were found without tool_result blocks"). + dir := t.TempDir() + id := "ses_6666666666666666" + data := `{"type":"session","id":"ses_6666666666666666","created_at":"2025-01-02T03:04:05Z"} +{"type":"message","message":{"id":"msg_1","role":"user","parts":[{"type":"text","text":"list files"}]}} +{"type":"message","message":{"id":"msg_2","role":"assistant","parts":[{"type":"tool_call","call_id":"toolu_dead","name":"bash","arguments":{"command":"ls"}}]}} +` + if err := os.WriteFile(filepath.Join(dir, id+".jsonl"), []byte(data), 0o644); err != nil { + t.Fatal(err) + } + + s, err := LoadSession(Config{SessionDir: dir, Model: message.ModelRef{Provider: "p", Model: "m"}}, id) + if err != nil { + t.Fatal(err) + } + h := s.History() + if len(h) != 3 { + t.Fatalf("history length = %d, want 3 (user, assistant, synthetic tool result)", len(h)) + } + last := h[2] + if last.Role != message.RoleTool { + t.Fatalf("h[2].Role = %q, want %q", last.Role, message.RoleTool) + } + var tr *message.ToolResult + for _, p := range last.Parts { + if r, ok := p.(*message.ToolResult); ok { + tr = r + } + } + if tr == nil { + t.Fatalf("h[2] has no ToolResult part: %+v", last.Parts) + } + if tr.CallID != "toolu_dead" { + t.Errorf("synthetic result CallID = %q, want %q", tr.CallID, "toolu_dead") + } + if !tr.IsError { + t.Error("synthetic result IsError = false, want true") + } + if got := tr.Content.Text(); got != message.SyntheticOrphanResultText { + t.Errorf("synthetic result text = %q, want %q", got, message.SyntheticOrphanResultText) + } +} diff --git a/engine/tool_call_poison_test.go b/engine/tool_call_poison_test.go index ccc2506..b9ac9e9 100644 --- a/engine/tool_call_poison_test.go +++ b/engine/tool_call_poison_test.go @@ -104,11 +104,14 @@ func TestPersistTruncatedToolCallArguments(t *testing.T) { // The session log is loadable and agrees with in-memory history — the // turn that used to fail persist ("never journaled") is now durable. + // LoadSession additionally repairs orphaned tool_calls at ingest (see + // TestLoadSessionRepairsOrphanedToolCalls), so the loaded history is + // the repaired view of the resident one. loaded, err := LoadSession(cfg, s.ID) if err != nil { t.Fatalf("LoadSession: %v", err) } - if got, want := historyJSON(t, loaded.History()), historyJSON(t, s.History()); got != want { + if got, want := historyJSON(t, loaded.History()), historyJSON(t, message.ResolveOrphanToolCalls(s.History())); got != want { t.Errorf("loaded history = %s\nwant %s", got, want) } From e475606a167211f5106fb67d567bba306a3d0832 Mon Sep 17 00:00:00 2001 From: Andy Bonventre Date: Thu, 9 Jul 2026 18:16:24 -0400 Subject: [PATCH 6/6] fix(message): disambiguate synthetic orphan-result message IDs by source index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provider call IDs are not guaranteed unique across turns; two orphaned turns reusing a CallID produced identical synthetic message IDs. Mix in the source message index — still deterministic for retranscode/reload. Also document that the interrupted-turn synthetic results are appended without an EventMessage emit (review note on PR #56). --- engine/engine.go | 6 ++++++ message/message.go | 8 +++++++- message/message_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/engine/engine.go b/engine/engine.go index 2a617d8..4464bc7 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -685,6 +685,12 @@ func (e *interruptedTurnError) Unwrap() error { return e.err } // interruptedToolResults builds the synthetic tool-role message Prompt // appends immediately after an interruptedTurnError's partial assistant // message: one is_error ToolResult per ToolCall part in partial, in order. +// +// Like every tool-result append, this message is persisted without an +// EventMessage emit — and since the interrupted calls never executed, no +// EventToolEnd fired either. A pure event-stream consumer therefore sees +// the partial assistant message with no following results for the +// interrupted calls until it reloads history (GET /message, LoadSession). func interruptedToolResults(partial *message.Message) message.Message { var results message.Parts for _, p := range partial.Parts { diff --git a/message/message.go b/message/message.go index 2c07b24..28f0510 100644 --- a/message/message.go +++ b/message/message.go @@ -623,7 +623,13 @@ func ResolveOrphanToolCalls(messages []Message) []Message { for _, ins := range insertions { if ins.afterIndex == i { out = append(out, Message{ - ID: "synthetic-orphan-tool-result-" + strings.Join(callIDsOf(ins.parts), "-"), + // The source index disambiguates two orphaned turns + // whose calls reuse a CallID — nothing guarantees + // provider call-ID uniqueness across turns, and a UI + // keyed by message ID must never see two collide. Still + // deterministic: the same history always yields the + // same IDs. + ID: fmt.Sprintf("synthetic-orphan-tool-result-%d-%s", ins.afterIndex, strings.Join(callIDsOf(ins.parts), "-")), Role: RoleTool, Parts: ins.parts, }) diff --git a/message/message_test.go b/message/message_test.go index 362305e..8555067 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -827,3 +827,27 @@ func TestResolveOrphanToolCallsPartialMerge(t *testing.T) { func toolCallPart(id, name, args string) *ToolCall { return &ToolCall{CallID: id, Name: name, Arguments: json.RawMessage(args)} } + +// TestResolveOrphanToolCallsDistinctSyntheticIDs pins that two orphaned +// turns whose calls happen to reuse the same CallID (nothing guarantees +// provider call-ID uniqueness across turns) still get synthetic messages +// with distinct message IDs — a UI keyed by message ID must never see two +// messages collide. +func TestResolveOrphanToolCallsDistinctSyntheticIDs(t *testing.T) { + in := []Message{ + {Role: RoleAssistant, Parts: Parts{toolCallPart("tc_reused", "bash", `{}`)}}, + {Role: RoleUser, Parts: Parts{&Text{Text: "still there?"}}}, + {Role: RoleAssistant, Parts: Parts{toolCallPart("tc_reused", "bash", `{}`)}}, + } + out := ResolveOrphanToolCalls(in) + if len(out) != 5 { + t.Fatalf("len(out) = %d, want 5 (two synthetic messages inserted)", len(out)) + } + first, second := out[1], out[4] + if first.Role != RoleTool || second.Role != RoleTool { + t.Fatalf("synthetic roles = %s, %s, want tool, tool", first.Role, second.Role) + } + if first.ID == second.ID { + t.Errorf("synthetic message IDs collide: %q", first.ID) + } +}