diff --git a/README.md b/README.md index bf0acec5..dbdf1154 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ mcpServers: --max-steps Maximum agent steps (0 for unlimited) --stream Enable streaming output (default: true) --compact Enable compact output mode ---auto-compact Auto-compact conversation near context limit +--auto-compact Auto-compact conversation near context limit (reactive compact-and-retry on provider context-overflow errors is always on) # Extensions and tools --extension, -e Load additional extension file(s) (repeatable) @@ -718,8 +718,10 @@ host, err := kit.New(ctx, &kit.Options{ // Configuration SkipConfig: true, // Skip .kit.yml files (viper defaults + env vars still apply) - // Compaction - AutoCompact: true, // Auto-compact near context limit + // Compaction — proactive check before turns near the context limit. + // Reactive compaction (compact + replay once on provider context-overflow + // errors) is always on, independent of this setting. + AutoCompact: true, Debug: true, // Debug logging }) diff --git a/pkg/kit/README.md b/pkg/kit/README.md index aae65cc2..d42b7970 100644 --- a/pkg/kit/README.md +++ b/pkg/kit/README.md @@ -107,8 +107,10 @@ host, err := kit.New(ctx, &kit.Options{ // Configuration SkipConfig: true, // Skip .kit.yml files (viper defaults + env vars still apply) - // Compaction - AutoCompact: true, // Auto-compact near context limit + // Compaction — proactive check before turns near the context limit. + // Reactive compact-and-replay on provider context-overflow errors is + // always on, independent of this setting. + AutoCompact: true, // In-process MCP servers (map name → *kit.MCPServer) InProcessMCPServers: map[string]*kit.MCPServer{ @@ -374,7 +376,9 @@ msg := kit.ConvertFromLLMMessage(lMsg) // LLMMessage → SDK Message agent loop and surface a typed result - Provider-error sentinels - `ErrContextOverflow`, `ErrRateLimit`, `ErrAuth`, `ErrProviderUnavailable`, `ErrInvalidRequest`; classify with - `ClassifyProviderError(err)` and match via `errors.Is` + `ClassifyProviderError(err)` and match via `errors.Is`. `ErrContextOverflow` + is surfaced only after the turn loop's automatic compact-and-replay + recovery also failed ### Key Methods diff --git a/pkg/kit/errors.go b/pkg/kit/errors.go index 5e1ffb42..d54d340c 100644 --- a/pkg/kit/errors.go +++ b/pkg/kit/errors.go @@ -11,7 +11,10 @@ import ( // arbitrary provider error to one of these sentinels. var ( // ErrContextOverflow indicates the request exceeded the model's maximum - // context window. Embedders typically respond by compacting and retrying. + // context window. Kit's turn loop recovers from this automatically: + // it compacts the conversation and replays the turn once before + // surfacing the error. When this error still reaches the caller, + // compaction was impossible or insufficient. ErrContextOverflow = errors.New("context window exceeded") // ErrRateLimit indicates the provider throttled the request. Embedders diff --git a/pkg/kit/kit.go b/pkg/kit/kit.go index da57391d..9f96b059 100644 --- a/pkg/kit/kit.go +++ b/pkg/kit/kit.go @@ -1227,7 +1227,11 @@ type Options struct { InProcessMCPServers map[string]*MCPServer // Compaction - AutoCompact bool // Auto-compact when near context limit + // AutoCompact enables proactive compaction before turns that near the + // context limit. Independent of this setting, the turn loop always + // compacts reactively and replays the turn once when a provider call + // fails with a context-overflow error. + AutoCompact bool CompactionOptions *CompactionOptions // Config for auto-compaction (nil = defaults) // Debug enables debug logging for the SDK. When DebugLogger is nil this @@ -2774,6 +2778,23 @@ func (m *Kit) generate(ctx context.Context, messages []fantasy.Message) (*agent. }) } +// persistGenerationRemainder persists messages produced by generation that +// were not already persisted incrementally by the onStepMessages callback. +// sentCount is the number of input messages sent to the LLM (the prefix of +// result.ConversationMessages to skip). Safe to call with a nil result. +func (m *Kit) persistGenerationRemainder(result *agent.GenerateWithLoopResult, sentCount int) { + if result == nil || len(result.ConversationMessages) <= sentCount { + return + } + newMessages := result.ConversationMessages[sentCount:] + if result.PersistedMessageCount >= len(newMessages) { + return + } + for _, msg := range newMessages[result.PersistedMessageCount:] { + _, _ = m.session.AppendMessage(msg) + } +} + // runTurn is the shared lifecycle for every prompt mode: // 1. Run BeforeTurn hooks (can modify prompt, inject messages). // 2. Persist pre-generation messages to the tree session. @@ -2866,21 +2887,45 @@ func (m *Kit) runTurn(ctx context.Context, promptLabel string, prompt string, pr m.events.emit(MessageStartEvent{}) result, err := m.generate(ctx, messages) + + // Reactive compaction (issue #85): when the provider rejected the + // request because the context window was exceeded, compact the + // conversation and replay the turn once instead of failing. This is the + // safety net for token-estimate drift and huge mid-turn tool results + // that the proactive ShouldCompact() check cannot catch. A single-retry + // guard (no loop) prevents compact/overflow cycles. + if isContextOverflow(err) { + // Persist completed-step messages from the failed attempt first so + // compaction and the rebuilt context include them — the replay then + // resumes from where the turn overflowed rather than restarting. + m.persistGenerationRemainder(result, sentCount) + result = nil + + if retryMessages, retryErr := m.prepareOverflowRetry(ctx); retryErr != nil { + // Recovery impossible — wrap the original provider error with + // the recovery failure. ClassifyProviderError below still maps + // the chain to ErrContextOverflow, so the errors.Is contract + // holds even when the provider error was raw text. + err = fmt.Errorf("conversation too large to compact — context-overflow recovery failed (%v): %w", retryErr, err) + } else { + // Discard stream deltas captured during the failed attempt so + // TurnResult.Stream reflects only the replay. + collector.drain() + sentCount = len(retryMessages) + result, err = m.generate(ctx, retryMessages) + if isContextOverflow(err) { + err = fmt.Errorf("conversation too large to compact — still exceeds the model context window after compaction: %w", err) + } + } + } + if err != nil { // Persist any messages from completed steps that were NOT already // persisted incrementally by the onStepMessages callback. The agent // layer only includes fully-paired tool_use + tool_result messages // in completedStepMessages, so there are no orphaned entries that // would break subsequent API requests. - if result != nil { - newMessages := result.ConversationMessages[sentCount:] - alreadyPersisted := result.PersistedMessageCount - if alreadyPersisted < len(newMessages) { - for _, msg := range newMessages[alreadyPersisted:] { - _, _ = m.session.AppendMessage(msg) - } - } - } + m.persistGenerationRemainder(result, sentCount) m.events.emit(TurnEndEvent{Error: err}) // Run AfterTurn hooks even on error. m.afterTurn.run(AfterTurnHook{Error: err}) @@ -2893,15 +2938,7 @@ func (m *Kit) runTurn(ctx context.Context, promptLabel string, prompt string, pr // by the onStepMessages callback during generation. This handles the // non-streaming path (where onStepMessages is not called) and any edge // cases where the final response messages weren't covered by step callbacks. - if len(result.ConversationMessages) > sentCount { - newMessages := result.ConversationMessages[sentCount:] - alreadyPersisted := result.PersistedMessageCount - if alreadyPersisted < len(newMessages) { - for _, msg := range newMessages[alreadyPersisted:] { - _, _ = m.session.AppendMessage(msg) - } - } - } + m.persistGenerationRemainder(result, sentCount) // Store the API-reported token count so GetContextStats() matches the // built-in status bar. The context window is filled by all token diff --git a/pkg/kit/overflow.go b/pkg/kit/overflow.go new file mode 100644 index 00000000..0a5b9ce7 --- /dev/null +++ b/pkg/kit/overflow.go @@ -0,0 +1,106 @@ +package kit + +import ( + "context" + "errors" + "fmt" + + "charm.land/fantasy" +) + +// This file implements reactive compaction (issue #85): when a provider call +// fails with a context-length/overflow error, Kit automatically compacts the +// conversation and replays the turn once instead of surfacing a hard failure. +// +// Proactive compaction (ShouldCompact() before a turn) relies on token +// estimates that inevitably drift from real tokenizer counts — especially +// with tool-heavy traffic, non-English text, and images. A single huge +// mid-turn tool result can also overflow the context even when the turn +// started well under the limit. The reactive path is the safety net that +// makes an imprecise estimator acceptable in practice. + +// isContextOverflow reports whether err is (or classifies as) a provider +// context-window overflow. +func isContextOverflow(err error) bool { + return err != nil && errors.Is(ClassifyProviderError(err), ErrContextOverflow) +} + +// prepareOverflowRetry runs reactive compaction after a provider +// context-overflow error and rebuilds the request context for a single +// replay. Media attachments in the rebuilt context are replaced with text +// placeholders — they are typically the largest contributors and cannot be +// shrunk by summarisation, so dropping them from the replayed request gives +// it the best chance to fit. The session itself is not modified beyond the +// appended compaction entry. +// +// Returns the replay messages on success. On failure (compaction failed, +// was cancelled, or produced no usable context) it returns a non-nil error +// describing why recovery was impossible; callers wrap it together with the +// original provider error so the surfaced failure both explains the failed +// recovery and preserves the [ErrContextOverflow] classification. +func (m *Kit) prepareOverflowRetry(ctx context.Context) ([]fantasy.Message, error) { + if _, err := m.compactInternal(ctx, m.compactionOpts, "", true); err != nil { + return nil, fmt.Errorf("compaction failed: %w", err) + } + + // Rebuild the context from the session: the compaction summary now + // replaces the summarised prefix, and any completed steps persisted + // from the failed attempt are included so the replay resumes rather + // than restarts. + messages, _, _ := m.session.BuildContext() + messages = stripMediaParts(messages) + + // Re-run ContextPrepare hooks on the rebuilt context, mirroring the + // initial attempt so extensions observe every outgoing request. Strip + // media from the hook result too — a hook may replace the messages and + // reintroduce attachments, which would defeat the recovery. + if hookResult := m.contextPrepare.run(ContextPrepareHook{Messages: messages}); hookResult != nil && hookResult.Messages != nil { + messages = stripMediaParts(hookResult.Messages) + } + + if len(messages) == 0 { + return nil, fmt.Errorf("compaction produced an empty context") + } + return messages, nil +} + +// stripMediaParts returns a copy of messages with file/media attachments +// replaced by short text placeholders. Message slices and non-file parts are +// shared, not copied; only messages containing file parts get a new Content +// slice. +func stripMediaParts(messages []fantasy.Message) []fantasy.Message { + out := make([]fantasy.Message, len(messages)) + for i, msg := range messages { + out[i] = msg + hasFile := false + for _, part := range msg.Content { + if _, ok := part.(fantasy.FilePart); ok { + hasFile = true + break + } + } + if !hasFile { + continue + } + replaced := make([]fantasy.MessagePart, 0, len(msg.Content)) + for _, part := range msg.Content { + fp, ok := part.(fantasy.FilePart) + if !ok { + replaced = append(replaced, part) + continue + } + name := fp.Filename + if name == "" { + name = fp.MediaType + } + if name == "" { + name = "attachment" + } + replaced = append(replaced, fantasy.TextPart{ + Text: fmt.Sprintf("[attachment %q removed after context overflow]", name), + }) + } + out[i].Content = replaced + } + return out +} diff --git a/pkg/kit/overflow_test.go b/pkg/kit/overflow_test.go new file mode 100644 index 00000000..8c5fc1d3 --- /dev/null +++ b/pkg/kit/overflow_test.go @@ -0,0 +1,288 @@ +package kit + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "charm.land/fantasy" +) + +// --------------------------------------------------------------------------- +// isContextOverflow +// --------------------------------------------------------------------------- + +func TestIsContextOverflow(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"raw provider text", errors.New("error: context_length_exceeded for this model"), true}, + {"context window phrase", errors.New("the prompt is too long for the context window"), true}, + {"pre-wrapped sentinel", fmt.Errorf("%w: upstream detail", ErrContextOverflow), true}, + {"rate limit", errors.New("HTTP status 429: rate limit exceeded"), false}, + {"generic", errors.New("connection reset by peer"), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isContextOverflow(tc.err); got != tc.want { + t.Errorf("isContextOverflow(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// stripMediaParts +// --------------------------------------------------------------------------- + +func TestStripMediaParts_ReplacesFileParts(t *testing.T) { + msgs := []fantasy.Message{ + fantasy.NewSystemMessage("system prompt"), + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "look at this"}, + fantasy.FilePart{Filename: "diagram.png", MediaType: "image/png", Data: []byte{1, 2, 3}}, + }, + }, + } + + out := stripMediaParts(msgs) + + if len(out) != 2 { + t.Fatalf("got %d messages, want 2", len(out)) + } + // Untouched message shares content. + if len(out[0].Content) != 1 { + t.Errorf("system message should be unchanged") + } + // File part replaced by placeholder text. + if len(out[1].Content) != 2 { + t.Fatalf("user message has %d parts, want 2", len(out[1].Content)) + } + tp, ok := out[1].Content[1].(fantasy.TextPart) + if !ok { + t.Fatalf("second part is %T, want TextPart", out[1].Content[1]) + } + if !strings.Contains(tp.Text, "diagram.png") || !strings.Contains(tp.Text, "removed after context overflow") { + t.Errorf("placeholder text = %q", tp.Text) + } + // Original input must not be mutated. + if _, ok := msgs[1].Content[1].(fantasy.FilePart); !ok { + t.Error("input messages were mutated") + } +} + +func TestStripMediaParts_PlaceholderNameFallbacks(t *testing.T) { + msgs := []fantasy.Message{ + {Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{ + fantasy.FilePart{MediaType: "image/jpeg"}, // no filename + fantasy.FilePart{}, // no filename, no media type + }}, + } + + out := stripMediaParts(msgs) + + t0 := out[0].Content[0].(fantasy.TextPart) + if !strings.Contains(t0.Text, "image/jpeg") { + t.Errorf("media-type fallback missing: %q", t0.Text) + } + t1 := out[0].Content[1].(fantasy.TextPart) + if !strings.Contains(t1.Text, "attachment") { + t.Errorf("generic fallback missing: %q", t1.Text) + } +} + +func TestStripMediaParts_NoFilesPassthrough(t *testing.T) { + msgs := []fantasy.Message{ + fantasy.NewUserMessage("plain text"), + fantasy.NewSystemMessage("system"), + } + out := stripMediaParts(msgs) + for i := range out { + if len(out[i].Content) != len(msgs[i].Content) { + t.Errorf("message %d content length changed", i) + } + } +} + +// --------------------------------------------------------------------------- +// prepareOverflowRetry +// --------------------------------------------------------------------------- + +// newOverflowTestKit builds a minimal Kit with an in-memory session, an event +// bus, and empty hook registries — enough to exercise the reactive +// compaction path without a provider (compaction itself is short-circuited +// by a BeforeCompact hook supplying a custom summary). +func newOverflowTestKit(t *testing.T) *Kit { + t.Helper() + tree, err := InitTreeSession(&Options{NoSession: true}) + if err != nil { + t.Fatalf("InitTreeSession: %v", err) + } + return &Kit{ + session: NewTreeManagerAdapter(tree), + events: newEventBus(), + beforeCompact: newHookRegistry[BeforeCompactHook, BeforeCompactResult](), + contextPrepare: newHookRegistry[ContextPrepareHook, ContextPrepareResult](), + } +} + +// appendTestMessage appends a fixture message to the Kit's session, failing +// the test immediately when the append errors so broken setup never produces +// misleading assertions downstream. +func appendTestMessage(t *testing.T, k *Kit, msg fantasy.Message) { + t.Helper() + if _, err := k.session.AppendMessage(msg); err != nil { + t.Fatalf("append test message: %v", err) + } +} + +func TestPrepareOverflowRetry_CompactsAndStripsMedia(t *testing.T) { + k := newOverflowTestKit(t) + + // Supply a custom summary so compaction needs no LLM call. + k.OnBeforeCompact(HookPriorityNormal, func(h BeforeCompactHook) *BeforeCompactResult { + if !h.IsAutomatic { + t.Error("reactive compaction should run as automatic") + } + return &BeforeCompactResult{Summary: "compacted summary of earlier work"} + }) + + // Seed a conversation with a media attachment in the latest user turn. + appendTestMessage(t, k, fantasy.NewUserMessage("first question")) + appendTestMessage(t, k, fantasy.Message{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{fantasy.TextPart{Text: "first answer"}}, + }) + appendTestMessage(t, k, fantasy.Message{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "analyze this image"}, + fantasy.FilePart{Filename: "big.png", MediaType: "image/png", Data: make([]byte, 64)}, + }, + }) + + messages, err := k.prepareOverflowRetry(context.Background()) + if err != nil { + t.Fatalf("prepareOverflowRetry should succeed: %v", err) + } + if len(messages) == 0 { + t.Fatal("replay context is empty") + } + + sawSummary := false + for _, msg := range messages { + for _, part := range msg.Content { + if _, isFile := part.(fantasy.FilePart); isFile { + t.Error("replay context still contains a media part") + } + if tp, isText := part.(fantasy.TextPart); isText && strings.Contains(tp.Text, "compacted summary of earlier work") { + sawSummary = true + } + } + } + if !sawSummary { + t.Error("replay context does not include the compaction summary") + } +} + +func TestPrepareOverflowRetry_CompactionFailureReturnsError(t *testing.T) { + k := newOverflowTestKit(t) + + // Fewer than 2 messages — compactImpl refuses, so recovery must report + // the failure; runTurn then wraps it with the original provider error. + appendTestMessage(t, k, fantasy.NewUserMessage("only message")) + + if _, err := k.prepareOverflowRetry(context.Background()); err == nil { + t.Fatal("prepareOverflowRetry should fail when compaction is impossible") + } else if !strings.Contains(err.Error(), "compaction failed") { + t.Errorf("error should explain the compaction failure: %v", err) + } +} + +func TestPrepareOverflowRetry_CancelledCompactionReturnsError(t *testing.T) { + k := newOverflowTestKit(t) + k.OnBeforeCompact(HookPriorityNormal, func(BeforeCompactHook) *BeforeCompactResult { + return &BeforeCompactResult{Cancel: true, Reason: "no compaction in tests"} + }) + + appendTestMessage(t, k, fantasy.NewUserMessage("q")) + appendTestMessage(t, k, fantasy.Message{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{fantasy.TextPart{Text: "a"}}, + }) + + if _, err := k.prepareOverflowRetry(context.Background()); err == nil { + t.Fatal("cancelled compaction should abort the retry") + } else if !strings.Contains(err.Error(), "no compaction in tests") { + t.Errorf("error should carry the cancellation reason: %v", err) + } +} + +func TestPrepareOverflowRetry_RunsContextPrepareHooks(t *testing.T) { + k := newOverflowTestKit(t) + k.OnBeforeCompact(HookPriorityNormal, func(BeforeCompactHook) *BeforeCompactResult { + return &BeforeCompactResult{Summary: "s"} + }) + hookRan := false + k.contextPrepare.register(HookPriorityNormal, func(h ContextPrepareHook) *ContextPrepareResult { + hookRan = true + return nil + }) + + appendTestMessage(t, k, fantasy.NewUserMessage("q")) + appendTestMessage(t, k, fantasy.Message{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{fantasy.TextPart{Text: "a"}}, + }) + + if _, err := k.prepareOverflowRetry(context.Background()); err != nil { + t.Fatalf("prepareOverflowRetry should succeed: %v", err) + } + if !hookRan { + t.Error("ContextPrepare hooks must run on the replayed context") + } +} + +func TestPrepareOverflowRetry_StripsMediaFromHookResult(t *testing.T) { + // A ContextPrepare hook may replace the messages and reintroduce media + // attachments; the recovery must strip those too or the replay can + // overflow again. + k := newOverflowTestKit(t) + k.OnBeforeCompact(HookPriorityNormal, func(BeforeCompactHook) *BeforeCompactResult { + return &BeforeCompactResult{Summary: "s"} + }) + k.contextPrepare.register(HookPriorityNormal, func(h ContextPrepareHook) *ContextPrepareResult { + return &ContextPrepareResult{Messages: []fantasy.Message{{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "hook-injected"}, + fantasy.FilePart{Filename: "sneaky.png", MediaType: "image/png", Data: []byte{1}}, + }, + }}} + }) + + appendTestMessage(t, k, fantasy.NewUserMessage("q")) + appendTestMessage(t, k, fantasy.Message{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{fantasy.TextPart{Text: "a"}}, + }) + + messages, err := k.prepareOverflowRetry(context.Background()) + if err != nil { + t.Fatalf("prepareOverflowRetry should succeed: %v", err) + } + for _, msg := range messages { + for _, part := range msg.Content { + if _, isFile := part.(fantasy.FilePart); isFile { + t.Fatal("media reintroduced by a ContextPrepare hook must be stripped") + } + } + } +} diff --git a/skills/kit-sdk/SKILL.md b/skills/kit-sdk/SKILL.md index 7455fafe..b3d4ca1e 100644 --- a/skills/kit-sdk/SKILL.md +++ b/skills/kit-sdk/SKILL.md @@ -1034,6 +1034,14 @@ host, _ := kit.New(ctx, &kit.Options{ }) ``` +**Reactive compaction (always on):** independent of `AutoCompact`, when a +provider call fails with a context-overflow error the turn loop compacts the +conversation and replays the turn once. Media attachments in the replayed +request are replaced with text placeholders. If the replay still overflows, +the turn fails with `kit.ErrContextOverflow` ("conversation too large to +compact"). `AutoCompact: true` additionally compacts *proactively* before +turns that near the limit. + --- ## In-Process Subagents diff --git a/www/pages/cli/flags.md b/www/pages/cli/flags.md index 45998986..a93dbb96 100644 --- a/www/pages/cli/flags.md +++ b/www/pages/cli/flags.md @@ -37,7 +37,7 @@ These flags control Kit's behavior. When a prompt is passed as a positional argu | `--max-steps` | — | `0` | Maximum agent steps (0 for unlimited) | | `--stream` | — | `true` | Enable streaming output | | `--compact` | — | `false` | Enable compact output mode | -| `--auto-compact` | — | `false` | Auto-compact conversation near context limit | +| `--auto-compact` | — | `false` | Compact proactively when near the context limit (reactive compact-and-retry on provider overflow errors is [always on](/sessions#reactive-compaction-on-overflow)) | ## Extensions diff --git a/www/pages/sdk/options.md b/www/pages/sdk/options.md index 20a5db2d..2a058894 100644 --- a/www/pages/sdk/options.md +++ b/www/pages/sdk/options.md @@ -224,7 +224,7 @@ context files at runtime (e.g. per user or per session), use the | Field | Type | Default | Description | |-------|------|---------|-------------| -| `AutoCompact` | `bool` | `false` | Auto-compact when near context limit | +| `AutoCompact` | `bool` | `false` | Compact proactively before turns that near the context limit. Independent of this setting, Kit always compacts **reactively** and replays the turn once when a provider call fails with a context-overflow error. | | `CompactionOptions` | `*CompactionOptions` | — | Configuration for compaction; zero-value budget fields adapt to the model's limits. See [CompactionOptions](#compactionoptions) below. | | `MCPAuthHandler` | `MCPAuthHandler` | — | OAuth handler for remote MCP servers. `nil` disables OAuth (servers returning 401 fail with the authorization-required error). See [MCP OAuth](#mcp-oauth-authorization) below. | | `MCPTokenStoreFactory` | `func` | — | Custom OAuth token storage for MCP servers (default: JSON file in `$XDG_CONFIG_HOME/.kit/mcp_tokens.json`). | @@ -254,6 +254,24 @@ When a session compacts more than once, the previous summary is fed back into the summarization prompt and updated incrementally (anchored summaries), so detail is preserved across compaction generations. +#### Reactive compaction on context overflow + +Token estimates inevitably drift from real tokenizer counts, and a single +huge mid-turn tool result can overflow the context even when the turn started +well under the limit. When the provider rejects a request with a +context-overflow error, Kit automatically: + +1. Compacts the conversation (persisting any completed steps first, so the + replay resumes rather than restarts). +2. Replays the turn once with the rebuilt context, replacing media + attachments with text placeholders (media cannot be shrunk by + summarization). +3. If the replay still overflows, fails the turn with `kit.ErrContextOverflow` + ("conversation too large to compact"). + +This safety net is always on — it does not require `AutoCompact`, which only +controls the *proactive* check before each turn. + ## MCP OAuth Authorization When a remote MCP server (SSE or Streamable HTTP) requires OAuth, Kit runs diff --git a/www/pages/sdk/overview.md b/www/pages/sdk/overview.md index ccf1bb4a..ef7ebf3b 100644 --- a/www/pages/sdk/overview.md +++ b/www/pages/sdk/overview.md @@ -557,6 +557,11 @@ model's context and output limits when left at their zero values; pass a `*kit.CompactionOptions` as the second argument to `Compact` to override them — see [SDK options → CompactionOptions](/sdk/options#compactionoptions). +Estimates can still undercount. When a provider call fails with a +context-overflow error, the turn loop automatically compacts and replays the +turn once before surfacing `kit.ErrContextOverflow` — see +[Reactive compaction](/sdk/options#reactive-compaction-on-context-overflow). + ## Provider error classification Provider failures are wrapped with exported sentinels so you can branch on the @@ -568,7 +573,10 @@ also classify any provider error yourself with `kit.ClassifyProviderError`: _, err := host.PromptResult(ctx, prompt) switch { case errors.Is(err, kit.ErrContextOverflow): - host.Compact(ctx, nil, "") // compact and retry + // Kit already compacted and replayed the turn once before surfacing + // this — reaching here means the conversation cannot fit even after + // compaction (e.g. start a new session or trim input). + handleUnrecoverableOverflow() case errors.Is(err, kit.ErrRateLimit): backoffAndRetry() case errors.Is(err, kit.ErrAuth): @@ -582,7 +590,7 @@ case errors.Is(err, kit.ErrInvalidRequest): | Sentinel | Meaning | |----------|---------| -| `kit.ErrContextOverflow` | Request exceeded the model's context window | +| `kit.ErrContextOverflow` | Request exceeded the model's context window — surfaced only after Kit's automatic compact-and-replay recovery also failed | | `kit.ErrRateLimit` | Provider throttled the request | | `kit.ErrAuth` | Credential / authorization failure | | `kit.ErrProviderUnavailable` | Transient upstream failure (5xx, network, timeout) | diff --git a/www/pages/sessions.md b/www/pages/sessions.md index ee9fd9ae..2616ef72 100644 --- a/www/pages/sessions.md +++ b/www/pages/sessions.md @@ -33,6 +33,10 @@ When conversations grow long, Kit can compact them to free up context window spa Use `/compact [focus]` to manually compact, or enable `--auto-compact` to compact automatically near the context limit. +### Reactive compaction on overflow + +Independent of `--auto-compact`, Kit always recovers from provider context-overflow errors reactively: it compacts the conversation and replays the failed turn once, replacing media attachments with text placeholders in the replayed request. Token estimates inevitably drift from real tokenizer counts, and a single huge mid-turn tool result can overflow the context even when the turn started under the limit — this safety net makes those cases non-fatal. Only when the replay also overflows does the turn fail, with a clear "conversation too large to compact" error. + ## Auto-cleanup Kit automatically cleans up empty sessions on shutdown and when using `/resume`. A session is considered empty if it has no messages beyond the initial system prompt. This prevents cluttering your sessions directory with unused files.