diff --git a/Package.swift b/Package.swift index d0b7d3b..0e796c2 100644 --- a/Package.swift +++ b/Package.swift @@ -20,7 +20,7 @@ let package = Package( .library(name: "SwiftAgentKitMac", targets: ["SwiftAgentKitMac"]), ], dependencies: [ - .package(url: "https://github.com/ayman3000/LLMProviderKit.git", from: "0.1.0-alpha.39"), + .package(url: "https://github.com/ayman3000/LLMProviderKit.git", from: "0.1.0-alpha.41"), .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "603.0.0"), .package(url: "https://github.com/modelcontextprotocol/swift-sdk.git", from: "0.12.1"), .package(url: "https://github.com/swiftlang/swift-docc-plugin.git", from: "1.4.0"), @@ -55,6 +55,10 @@ let package = Package( // that pin one request per streamed tool step. .product(name: "LLMProviderKitOllama", package: "LLMProviderKit"), .product(name: "LLMProviderKitOpenAI", package: "LLMProviderKit"), + // Anthropic wire test: the regression that the trailing note + // and the stop-reasoning nudge keep the request's roles + // alternating through AnthropicProvider.prepareRequest. + .product(name: "LLMProviderKitAnthropic", package: "LLMProviderKit"), ] ), diff --git a/Sources/SwiftAgentKit/Context/ArtifactTools.swift b/Sources/SwiftAgentKit/Context/ArtifactTools.swift index 50909a2..d9e295f 100644 --- a/Sources/SwiftAgentKit/Context/ArtifactTools.swift +++ b/Sources/SwiftAgentKit/Context/ArtifactTools.swift @@ -14,7 +14,7 @@ public struct ArtifactReadTool: AgentTool { public var isReadOnly: Bool { true } public let description = """ Read the full stored output of a previous tool call by its artifact id \ - (shown in brackets in the tool ledger, e.g. artifact-abc123). Use `offset` \ + (shown in brackets in the receipt that replaced the call, e.g. artifact-abc123). Use `offset` \ and `limit` to page through large outputs. """ public let parameters = ToolParameters( @@ -71,7 +71,7 @@ public struct ArtifactSearchTool: AgentTool { Pass ALL the terms you want to check in ONE call via `queries` (e.g. \ ["error:", "TEST FAILED", "warning"]) instead of one call per term. Returns \ matching lines with 2 lines of context each. Use the artifact id from the \ - tool ledger. + receipt that replaced the call. """ public let parameters = ToolParameters( properties: [ diff --git a/Sources/SwiftAgentKit/Context/ContextManager.swift b/Sources/SwiftAgentKit/Context/ContextManager.swift index 7dc00bd..b3e436e 100644 --- a/Sources/SwiftAgentKit/Context/ContextManager.swift +++ b/Sources/SwiftAgentKit/Context/ContextManager.swift @@ -3,9 +3,9 @@ // SwiftAgentKit // // ContextSift-style context management: keep the full main conversation, but -// move *completed* tool exchanges out of active model context — replaced by a -// compact receipt ledger, with full outputs preserved in an `ArtifactStore` and -// retrievable on demand via `artifact_read` / `artifact_search`. +// move *completed* tool exchanges out of active model context — each replaced, +// in its own place, by compact receipts, with full outputs preserved in an +// `ArtifactStore` and retrievable on demand via `artifact_read` / `artifact_search`. // // Opt-in: set `AgentConfig.contextManager`. When nil, the agent uses its normal // (trim-based) context handling and behaves exactly as before. @@ -25,7 +25,9 @@ public final class ContextManager: @unchecked Sendable { /// shown to the model as a bounded preview + reference — even while active. public var maxActiveResultChars: Int - /// How many recent completed-tool receipts to include in the ledger. + /// No longer used (0.4.0-alpha.120): every evicted step keeps its own + /// receipts in its own place, so there is no ledger to cap. Kept so + /// callers compile; compaction bounds the growth. public var ledgerEntries: Int /// Length of the receipt summary drawn from a tool result. @@ -112,6 +114,103 @@ public final class ContextManager: @unchecked Sendable { /// byte-stable between eviction events so provider prompt caches hit. public var evictionTargetFraction: Double = 0.5 + /// A batch happens only when at least this share of the message budget + /// can be evicted at once. Between batches the model-facing prefix is + /// byte-identical, so provider prompt caches hit — even when what cannot + /// be evicted (the floor) already fills the budget, where evicting one + /// step per call used to change the prefix on every call. + public var minEvictionBatchFraction: Double = 0.25 + + /// Headroom above the floor a batch evicts down to, as a share of the + /// message budget: the target is `max(budget × evictionTargetFraction, + /// floor + budget × evictionSlackFraction)`. + public var evictionSlackFraction: Double = 0.10 + + /// The messages' budget never drops below this share of + /// `inlineBudgetChars`, however long the system prompt is. + public var minMessageBudgetFraction: Double = 0.25 + + /// The agent's overflow safety net: when the sifted request, tool + /// definitions counted, passes this fraction of (context window − output + /// reserve), calls leave out the oldest whole steps (stored history is + /// kept) and the agent emits `.historyTrimmed(reason: .overflow)`. + /// + /// An app with its own compaction trigger must set this ABOVE that + /// trigger, in the net's terms (the trigger's share of the window plus + /// the tool definitions), or the net fires first and compaction never + /// does: the net's trimmed request is what `lastPromptTokens` then + /// records. 1.0 makes the net a pure overflow guard (the request would + /// not fit beside the output reserve). Default 0.8, the old fit's bound. + public var overflowFraction: Double = 0.8 + + /// Low watermark for the net's cut, as a share of its bound: a breach + /// leaves out the fewest oldest steps that bring the request to this, + /// and the cut then holds (byte-identical prefix) until the request + /// passes the bound again. + public var overflowTargetFraction: Double = 0.7 + + /// What the messages may use: the inline budget less the system prompt, + /// so system + messages keep the ceiling `inlineBudgetChars` always + /// meant. The system prompt itself is not sifted — it is the same on + /// every call and nothing here can shrink it. + public func messageBudget(systemChars: Int) -> Int { + let floor = Int(Double(inlineBudgetChars) * min(1, max(0, minMessageBudgetFraction))) + return max(floor, inlineBudgetChars - systemChars) + } + + /// A fresh manager over the same store with every setting copied — for a + /// sub-agent. Evictions, receipts and spill ids are per conversation and + /// start empty. + public func childManager() -> ContextManager { + let child = ContextManager(store: store, maxActiveResultChars: maxActiveResultChars, + ledgerEntries: ledgerEntries, summaryLength: summaryLength, + inlineBudgetChars: inlineBudgetChars, + keepLatestReadsInline: keepLatestReadsInline, + readToolNames: readToolNames, readIdentityParams: readIdentityParams) + child.eagerPersistMinChars = eagerPersistMinChars + child.evictionTargetFraction = evictionTargetFraction + child.minEvictionBatchFraction = minEvictionBatchFraction + child.evictionSlackFraction = evictionSlackFraction + child.minMessageBudgetFraction = minMessageBudgetFraction + child.overflowFraction = overflowFraction + child.overflowTargetFraction = overflowTargetFraction + return child + } + + /// How many of `candidates` (sizes of the evictable steps, oldest first) + /// one pass evicts. Pure. + /// - `remaining`: message characters still sent inline (earlier + /// evictions already taken off). + /// - Nothing goes while `remaining` fits the budget, or while less than a + /// minimum batch is evictable. Otherwise a batch frees at least + /// `max(remaining − target, minimum batch)`, where target is + /// `max(budget × targetFraction, floor + budget × slackFraction)` and + /// the floor is what cannot be evicted. + public static func evictionCount(candidates: [Int], remaining: Int, budget: Int, + targetFraction: Double, minBatchFraction: Double, + slackFraction: Double) -> Int { + guard remaining > budget, !candidates.isEmpty else { return 0 } + func share(_ fraction: Double) -> Int { Int((Double(budget) * min(1, max(0, fraction))).rounded(.up)) } + let evictable = candidates.reduce(0, +) + let minBatch = share(minBatchFraction) + guard evictable >= max(1, minBatch) else { return 0 } + let floor = remaining - evictable + let target = max(Int(Double(budget) * min(1, max(0, targetFraction))), floor + share(slackFraction)) + let needed = max(remaining - target, minBatch) + var freed = 0 + var count = 0 + for size in candidates where freed < needed { + freed += size + count += 1 + } + return count + } + + /// What a message weighs for sifting: its text plus its tool results. + static func size(of message: AgentMessage) -> Int { + message.content.count + (message.toolResults?.reduce(0) { $0 + $1.result.count } ?? 0) + } + /// Head-message ids of exchanges evicted by previous calls. Sticky — /// never un-evicted — so the prefix cannot flap. private var stickyEvicted: Set = [] @@ -145,10 +244,7 @@ public final class ContextManager: @unchecked Sendable { indices.append(j) j += 1 } - let chars = indices.reduce(0) { sum, idx in - sum + rest[idx].content.count - + (rest[idx].toolResults?.reduce(0) { $0 + $1.result.count } ?? 0) - } + let chars = indices.reduce(0) { $0 + Self.size(of: rest[$1]) } spans.append(ExchangeSpan(head: i, indices: indices, chars: chars)) i = j } @@ -177,9 +273,9 @@ public final class ContextManager: @unchecked Sendable { // MARK: - Build - /// Build the model-facing messages: identity/system + receipt ledger, then - /// main messages (completed assistant turns keep only their text), then the - /// active tool exchange (bounded). + /// Build the model-facing messages: the system prompt, then the main + /// messages (each evicted step replaced in place by its receipts), then + /// the active tool exchange (bounded). public func modelMessages( _ messages: [AgentMessage], systemTemplate: (String) -> String @@ -188,19 +284,23 @@ public final class ContextManager: @unchecked Sendable { .filter { $0.role == .system } .map { systemTemplate($0.content) } .filter { !$0.isEmpty } + let systemText = systemBlocks.joined(separator: "\n\n") let rest = messages.filter { $0.role != .system } - // Budget gate: while the whole conversation is small, keep everything - // inline (full tool calls + results, no ledger) so the model has its - // complete recent history. Only externalize once context grows large. - let totalChars = messages.reduce(0) { sum, m in - sum + m.content.count + (m.toolResults?.reduce(0) { $0 + $1.result.count } ?? 0) - } - if totalChars <= inlineBudgetChars { + // What sifting measures is the messages, not the system prompt: that + // is the same on every call and nothing here can shrink it. It comes + // off the budget instead (`messageBudget`). + let budget = messageBudget(systemChars: systemText.count) + let restChars = rest.reduce(0) { $0 + Self.size(of: $1) } + + // Budget gate: while the messages are small, keep everything inline + // (full tool calls + results, no receipts) so the model has its + // complete recent history. Only externalize once they grow large. + if restChars <= budget { var inline: [LLMMessage] = [] - if !systemBlocks.isEmpty { inline.append(.system(systemBlocks.joined(separator: "\n\n"))) } - for message in rest where message.role != .system { + if !systemText.isEmpty { inline.append(.system(systemText)) } + for message in rest { inline.append(contentsOf: message.toLLMMessages()) } return inline @@ -210,96 +310,75 @@ public final class ContextManager: @unchecked Sendable { // Map each tool-call id → its call, so a receipt can name the invocation // (e.g. the shell command), not just the tool. Built here (before the - // eviction loop) so we can look up a read's file path while deciding what - // to keep. + // eviction pass) so we can look up a read's file path while deciding + // what to keep. var callsByID: [String: AgentToolCall] = [:] for message in rest where message.role == .assistant { for call in message.toolCalls ?? [] { callsByID[call.id] = call } } - // The `rest` index of the most-recent read of each distinct file path — - // these exchanges are kept inline so the model doesn't re-read the file. + // The `rest` index of the most-recent read of each distinct target — + // these exchanges are kept inline so the model doesn't re-read it. let latestReadIndexByPath = keepLatestReadsInline ? latestReadIndices(in: rest, upTo: activeStart, callsByID: callsByID) : [:] let protectedIndices = Set(latestReadIndexByPath.values) - // Over budget: externalize whole tool exchanges OLDEST-FIRST, keeping - // the most RECENT tool results inline. An exchange is an - // assistant-with-toolCalls turn plus its following tool-result messages; - // evicting whole exchanges keeps tool_call/result pairing valid. - // - // CACHE-STABILITY HYSTERESIS: continuous eviction (evict just enough, - // every turn) changes the model-facing prefix on every call, so - // provider prompt caches never hit. Instead, evictions are STICKY - // (remembered by message id, never undone) and a budget breach evicts - // down to `evictionTargetFraction` of the budget — then the evicted - // set, the ledger, and the whole prefix stay byte-stable until roughly - // half a budget of new content accumulates. - var externalized = Set() // indices in `rest` to move to the ledger - var remaining = totalChars + // Over budget: externalize whole tool exchanges OLDEST-FIRST, in + // BATCHES. An exchange is an assistant-with-toolCalls turn plus its + // following tool-result messages; evicting whole exchanges keeps + // tool_call/result pairing valid. Evictions are sticky (remembered by + // message id, never undone), and a new batch happens only when at + // least `minEvictionBatchFraction` of the budget can go at once + // (`evictionCount`): between batches the prefix stays byte-identical. + var externalized = Set() // indices in `rest` that are evicted + var remaining = restChars let spans = exchangeSpans(in: rest, upTo: activeStart) let sticky = currentStickyEvicted() for span in spans where sticky.contains(rest[span.head].id) { span.indices.forEach { externalized.insert($0) } remaining -= span.chars } - if remaining > inlineBudgetChars { - let fraction = min(1, max(0, evictionTargetFraction)) - let target = Int(Double(inlineBudgetChars) * fraction) - var newlyEvicted: [UUID] = [] - for span in spans where !externalized.contains(span.head) { - guard remaining > target else { break } - // Keep the whole exchange inline if it holds a latest-per-path - // read (preserving tool_call/result pairing); evict the rest. - // Sticky evictions above are exempt — never un-evict. - if span.indices.contains(where: { protectedIndices.contains($0) }) { continue } - span.indices.forEach { externalized.insert($0) } - remaining -= span.chars - newlyEvicted.append(rest[span.head].id) - } - if !newlyEvicted.isEmpty { rememberEvicted(newlyEvicted) } + // A step holding a latest-per-target read is part of the floor. + let candidates = spans.filter { span in + !externalized.contains(span.head) && !span.indices.contains { protectedIndices.contains($0) } + } + let count = Self.evictionCount( + candidates: candidates.map(\.chars), remaining: remaining, budget: budget, + targetFraction: evictionTargetFraction, minBatchFraction: minEvictionBatchFraction, + slackFraction: evictionSlackFraction) + if count > 0 { + let batch = candidates.prefix(count) + for span in batch { span.indices.forEach { externalized.insert($0) } } + rememberEvicted(batch.map { rest[$0.head].id }) } - // Receipts for the externalized (older) tool results only. - var receipts: [ToolReceipt] = [] - for index in externalized.sorted() where rest[index].role == .tool { - for result in rest[index].toolResults ?? [] { - receipts.append(await receipt(for: result, call: callsByID[result.toolCallId])) + // Receipts, per evicted step, for that step's own place. + var receiptsByHead: [Int: [ToolReceipt]] = [:] + for span in spans where externalized.contains(span.head) { + var receipts: [ToolReceipt] = [] + for index in span.indices where rest[index].role == .tool { + for result in rest[index].toolResults ?? [] { + receipts.append(await receipt(for: result, call: callsByID[result.toolCallId])) + } } + receiptsByHead[span.head] = receipts } - var out: [LLMMessage] = [] + // A completed result is bounded (preview + artifact reference) only + // as part of a batch, like an eviction, or when it already was while + // active: crossing into sifting alone must not change what the normal + // path sent whole. A batch starts at its first evicted step, so only + // results after it may change. + let batchStart = count > 0 ? candidates.first?.head : nil - // One combined system message (identity + ledger) — provider-safe - // (some providers keep only a single system block). - var systemParts = systemBlocks - if !receipts.isEmpty { - var lines = receipts.suffix(ledgerEntries).map { "- " + $0.ledgerLine() }.joined(separator: "\n") - // Receipts past the ledger cap vanish from the prompt entirely; on a - // long run the model has no way to know that older outputs exist, - // let alone how to reach them. One count line keeps the store - // discoverable (artifact_list is registered iff the store is Listable). - let omitted = receipts.count - min(receipts.count, ledgerEntries) - if omitted > 0 { - let hint = store is any ListableArtifactStore - ? " — use artifact_list to enumerate all stored outputs" : "" - lines += "\n(+\(omitted) older tool call\(omitted == 1 ? "" : "s") not shown\(hint))" - } - systemParts.append( - "Older tool ledger — these tool calls already completed and their full output is " - + "NOT in this message; retrieve it with artifact_read / artifact_search using the " - + "artifact id in brackets when you need the details:\n" + lines - ) - } - if !systemParts.isEmpty { - out.append(.system(systemParts.joined(separator: "\n\n"))) - } + var out: [LLMMessage] = [] + // The system prompt alone. Receipts used to join it, so every batch + // changed the start of the request and the whole conversation missed + // the prompt cache; in place, a batch changes nothing before its + // first evicted step. + if !systemText.isEmpty { out.append(.system(systemText)) } - // Main messages. Externalized exchanges collapse to a ledger line - // (assistant keeps text only, tool results dropped); everything else — - // recent completed exchanges and the active exchange — stays inline - // (tool results bounded by `activeDisplay`). for (index, message) in rest.enumerated() { let isExternalized = externalized.contains(index) switch message.role { @@ -307,18 +386,29 @@ public final class ContextManager: @unchecked Sendable { out.append(contentsOf: message.toLLMMessages()) case .assistant: if isExternalized { - if !message.content.isEmpty { out.append(.assistant(message.content)) } // drop tool calls + // The evicted step in its own place: its text, then one + // receipt per call (the calls themselves are dropped and + // their results are in the store). + if let text = Self.receiptMessage(text: message.content, receipts: receiptsByHead[index] ?? []) { + out.append(.assistant(text)) + } } else { out.append(contentsOf: message.toLLMMessages()) // keep tool calls (recent/active) } case .tool: if !isExternalized, let results = message.toolResults { + let mayBound = index >= activeStart || (batchStart.map { index > $0 } ?? false) for result in results { - let display = await activeDisplay(for: result) - out.append(.tool(display, toolCallId: result.toolCallId)) + let display = mayBound || isBounded(result.toolCallId) + ? await activeDisplay(for: result) + : Self.fullDisplay(for: result) + // Images stay with their result, the active one included: + // a chat past the budget used to stop seeing screenshots. + out.append(LLMMessage(role: .tool, content: display, images: result.images, + toolCallId: result.toolCallId)) } } - // externalized tool results are dropped (represented in the ledger) + // evicted tool results are represented by their step's receipts case .system: break } @@ -327,6 +417,23 @@ public final class ContextManager: @unchecked Sendable { return out } + /// Opens an evicted step's receipts (the bracketed-evidence style the + /// app's own history replay uses). + public static let receiptHeader = "[Tool calls in this step — their output was moved out of context; artifact_read / artifact_search with the artifact id in brackets fetch it:" + + /// The assistant message an evicted step becomes: its text, then one + /// receipt line per call, in brackets. nil when there is nothing to say. + static func receiptMessage(text: String, receipts: [ToolReceipt]) -> String? { + guard !receipts.isEmpty else { return text.isEmpty ? nil : text } + let block = receiptHeader + "\n" + receipts.map { "- " + $0.ledgerLine() }.joined(separator: "\n") + "\n]" + return text.isEmpty ? block : text + "\n\n" + block + } + + /// A tool result exactly as the normal (un-sifted) path sends it. + private static func fullDisplay(for result: AgentToolResult) -> String { + "[Tool: \(result.toolName ?? "tool")] \(result.isError ? "ERROR" : "OK")\n\(result.result)" + } + // MARK: - Private /// Index in `rest` where the active tool exchange begins, or nil if there is @@ -443,6 +550,7 @@ public final class ContextManager: @unchecked Sendable { if Self.retrievalToolNames.contains(name) || result.result.count <= maxActiveResultChars { return "[Tool: \(name)] \(status)\n\(result.result)" } + rememberBounded(result.toolCallId) // `modelMessages` runs every turn, so a large result that stays the active // exchange would spill a fresh artifact each turn. Reuse the artifact id // for this tool-call id instead of duplicating the content in the store. @@ -485,6 +593,43 @@ public final class ContextManager: @unchecked Sendable { return out } + /// Tool-call ids whose result has been sent bounded: it stays bounded, + /// so the prefix does not flip back to the whole output. + private var boundedCallIDs: Set = [] + + private func isBounded(_ callID: String) -> Bool { + lock.lock(); defer { lock.unlock() } + return boundedCallIDs.contains(callID) + } + + /// The sift state that later calls depend on: the sticky evictions and + /// the results sent bounded. (Receipt and artifact caches are keyed by + /// call id and idempotent, so they are not part of it.) + struct SiftState: Equatable, Sendable { + var stickyEvicted: Set + var boundedCallIDs: Set + } + + /// Read or replace the sift state — the overflow net's trial sifts + /// restore it, so only the request actually sent commits anything. + var siftState: SiftState { + get { + lock.lock(); defer { lock.unlock() } + return SiftState(stickyEvicted: stickyEvicted, boundedCallIDs: boundedCallIDs) + } + set { + lock.lock(); defer { lock.unlock() } + stickyEvicted = newValue.stickyEvicted + boundedCallIDs = newValue.boundedCallIDs + } + } + + private func rememberBounded(_ callID: String) { + guard !callID.isEmpty else { return } + lock.lock(); defer { lock.unlock() } + boundedCallIDs.insert(callID) + } + private func cachedActiveArtifact(_ callID: String) -> String? { guard !callID.isEmpty else { return nil } lock.lock(); defer { lock.unlock() } diff --git a/Sources/SwiftAgentKit/Core/Agent.swift b/Sources/SwiftAgentKit/Core/Agent.swift index d03bb4b..98761d1 100644 --- a/Sources/SwiftAgentKit/Core/Agent.swift +++ b/Sources/SwiftAgentKit/Core/Agent.swift @@ -395,6 +395,14 @@ public actor Agent { /// Set at the start of a run; the next request built records its digest. private var firstRequestPending = false + /// The overflow safety net's cut: the id of the last stored message calls + /// leave out (with every older evictable step). Sticky until the request + /// passes the bound again; cleared by compaction, or when that message is + /// no longer stored. + private var overflowCutAfter: UUID? + /// The id of the current run's task message: the overflow net never + /// leaves it out, whatever user-role nudges the run appends after it. + private var runTaskID: UUID? /// Actor-isolated run/cancel state — actor serialization is the guard now. private var isRunActive = false @@ -985,11 +993,9 @@ public actor Agent { } // Add user message to conversation - if images.isEmpty { - conversation.append(.user(query)) - } else { - conversation.append(.user(query, images: images)) - } + let task: AgentMessage = images.isEmpty ? .user(query) : .user(query, images: images) + runTaskID = task.id + conversation.append(task) // Get registered tools and strengthen system prompt (must happen before skill injection) let registeredToolsEarly = await tools.allTools() @@ -1090,18 +1096,20 @@ public actor Agent { await compactHistory(reason: .nearLimit) } - // Get messages for LLM call (trimmed to context window) - var messagesForLLM = conversation.messagesForLLMCall() - // Budget checkpoint: transient system note for THIS call only - // (not appended to the conversation), nudging the model to - // reassess instead of grinding one subproblem to the turn cap. - if pendingNudges.remove(totalTurns) != nil { - messagesForLLM.append(.system( - Self.progressNudge(turn: totalTurns, maxTurns: config.maxTurns))) - } + // The history this call starts from (see historyForCall) + var messagesForLLM = historyForCall() + // Budget checkpoint: a note for THIS call only (never stored), + // nudging the model to reassess instead of grinding one + // subproblem to the turn cap. It goes at the very end, as a + // user message: in the system message it changed the prompt + // prefix (and on Anthropic, without a ContextManager, it + // replaced the whole system prompt). + let progressNote: String? = pendingNudges.remove(totalTurns) != nil + ? Self.progressNudge(turn: totalTurns, maxTurns: config.maxTurns) : nil let removedCount = conversation.allMessages().count - messagesForLLM.count if removedCount > 0 { - emit(.historyTrimmed(removedCount: removedCount, remainingCount: messagesForLLM.count)) + emit(.historyTrimmed(removedCount: removedCount, remainingCount: messagesForLLM.count, + reason: .fitToWindow)) } emit(.llmCallStarted(turn: totalTurns)) @@ -1138,7 +1146,7 @@ public actor Agent { plan: plan, startTime: startTime ) - _ = conversation.trim() + trimAfterStep() continue } onText?(intercepted.text) @@ -1160,7 +1168,8 @@ public actor Agent { // Build LLM request (with state-templated system prompt) let llmToolDefs = makeLLMToolDefinitions(from: visibleTools(registeredTools)) - var request = await makeLLMRequest(messagesForLLM: messagesForLLM, tools: llmToolDefs) + var request = await makeLLMRequest(messagesForLLM: messagesForLLM, tools: llmToolDefs, + trailingNote: progressNote) // Call the provider (streamed when onText is set). Transient // provider errors — network blips, proxy 5xx, Ollama cloud @@ -1182,8 +1191,9 @@ public actor Agent { if !overflowCompacted, !isCancelled, ContextCompaction.isContextOverflow(error), await compactHistory(reason: .overflow) != nil { overflowCompacted = true - messagesForLLM = conversation.messagesForLLMCall() - request = await makeLLMRequest(messagesForLLM: messagesForLLM, tools: llmToolDefs) + messagesForLLM = historyForCall() + request = await makeLLMRequest(messagesForLLM: messagesForLLM, tools: llmToolDefs, + trailingNote: progressNote) continue } llmAttempt += 1 @@ -1401,7 +1411,7 @@ public actor Agent { ) // Trim conversation - _ = conversation.trim() + trimAfterStep() } // Max turns reached @@ -1419,7 +1429,7 @@ public actor Agent { } else { // Single-shot or multi-turn chat (no tools) - let messagesForLLM = conversation.messagesForLLMCall() + let messagesForLLM = historyForCall() let request = await makeLLMRequest(messagesForLLM: messagesForLLM) emit(.llmCallStarted(turn: 1)) @@ -1461,7 +1471,7 @@ public actor Agent { if reasoningTimedOut, agentResponse.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { conversation.append(.assistant("")) conversation.append(.user("You have been reasoning for a long time without answering, so that step was stopped. Give your answer now.")) - let again = await makeLLMRequest(messagesForLLM: conversation.messagesForLLMCall()) + let again = await makeLLMRequest(messagesForLLM: historyForCall()) agentResponse = try await executeTurn(request: again, onText: onText, onReasoning: onReasoning) } } catch { @@ -1780,21 +1790,219 @@ public actor Agent { estimate: conversation.estimateTokens) guard !middle.isEmpty, let checkpoint = await compactor.summarize(middle: middle, reason: reason) else { return nil } conversation.replaceNonSystemMessages(ContextCompaction.assemble(checkpoint: checkpoint, tail: tail)) + overflowCutAfter = nil let after = conversation.estimateTotalTokens(conversation.allMessages()) emit(.contextCompacted(tokensBefore: before, tokensAfter: after, reason: reason)) return (before, after) } + /// The stored history a call starts from. With a ContextManager, all of + /// it: ContextSift bounds what is sent, and trimming the stored history + /// first slid the window on every call once it passed 80% of the window. + /// Without one, trimmed to fit, as before. + private func historyForCall() -> [AgentMessage] { + config.contextManager == nil ? conversation.messagesForLLMCall() : conversation.allMessages() + } + + /// After a step: the message-count cap, and — only without a + /// ContextManager — the token trim. Every removal is reported, by reason. + private func trimAfterStep() { + let capped = conversation.trim(byTokens: false) + if capped.removed > 0 { + emit(.historyTrimmed(removedCount: capped.removed, remainingCount: capped.remaining, + reason: .messageCap)) + } + guard config.contextManager == nil else { return } + let fitted = conversation.trim(byTokens: true) + if fitted.removed > 0 { + emit(.historyTrimmed(removedCount: fitted.removed, remainingCount: fitted.remaining, + reason: .tokenBudget)) + } + } + + /// The overflow safety net's bound: `ContextManager.overflowFraction` of + /// the window less the output reserve (0 without a ContextManager). + var overflowBoundTokens: Int { + guard let manager = config.contextManager else { return 0 } + return max(0, Int(Double(conversation.contextWindow - conversation.outputReserve) * manager.overflowFraction)) + } + + /// The steps the overflow net may leave out, oldest first: each a whole + /// unit (a tool-call turn with its results, or one message). Never the + /// system prompt, the run's task (the message with id `task`, recorded + /// when the run starts; without one, the latest user message), or the + /// step the model is waiting on: the last tool-call turn with everything + /// after it, when that is only its results and user-role nudges (loop, + /// repair, verifier, plan, reasoning, progress); else the last message. + static func overflowUnits(_ stored: [AgentMessage], task: UUID? = nil) -> [[Int]] { + let pinned = task.map { id in stored.firstIndex { $0.id == id } } + ?? stored.lastIndex(where: { $0.role == .user }) + let nonSystem = stored.indices.filter { stored[$0].role != .system } + guard let last = nonSystem.last else { return [] } + var end = last + if let head = stored.lastIndex(where: { $0.role == .assistant && $0.toolCalls?.isEmpty == false }), + stored[(head + 1)...].allSatisfy({ $0.role == .tool || $0.role == .user }) { + end = head + } + var units: [[Int]] = [] + var i = 0 + while i < end { + let message = stored[i] + guard message.role != .system, i != pinned else { i += 1; continue } + var unit = [i] + i += 1 + if message.role == .assistant, message.toolCalls?.isEmpty == false { + while i < end, stored[i].role == .tool { unit.append(i); i += 1 } + } + units.append(unit) + } + return units + } + + /// One trial sift for the overflow net's cut search: `view` sifted as if + /// it were sent, with the manager's sticky state put back afterwards, so + /// a request that is never sent commits nothing. Returns the sift and the + /// state it would have left (committed only for the cut chosen). + static func trialSift(_ view: [AgentMessage], manager: ContextManager, + systemTemplate: (String) -> String) async -> (messages: [LLMMessage], state: ContextManager.SiftState) { + let saved = manager.siftState + let messages = await manager.modelMessages(view, systemTemplate: systemTemplate) + let after = manager.siftState + manager.siftState = saved + return (messages, after) + } + + /// The overflow cut (how many of the oldest units a call leaves out). + /// Held while the request at `current` fits `bound`; on a breach, the + /// fewest more units that bring it to `target`, or all of them. Pure. + static func overflowCut(current: Int, units: Int, bound: Int, target: Int, + cost: (Int) async -> Int) async -> Int { + let current = min(current, units) + if await cost(current) <= bound { return current } + guard current < units else { return current } + for cut in (current + 1)...units where await cost(cut) <= target { + return cut + } + return units + } + + /// The estimator's token count for a request: characters ÷ 3.5 plus 4 per message. + static func estimatedTokens(_ messages: [LLMMessage]) -> Int { + let chars = messages.reduce(0) { $0 + $1.content.count } + return Int((Double(chars) / 3.5).rounded()) + messages.count * 4 + } + + /// The old fit's estimate (`Conversation.estimateTokens`: the + /// conversation's tokenCounter, or characters ÷ charsPerToken with a fixed + /// budget per tool-result image) of what one call sends: the sifted + /// messages AND the tool definitions, which the old fit left to its 20% + /// headroom. + func estimatedRequestTokens(_ messages: [LLMMessage], tools: [LLMToolDefinition]) -> Int { + let asStored = messages.map { message -> AgentMessage in + let calls = (message.toolCalls ?? []).map { $0.name + $0.arguments }.joined() + switch message.role { + case .system: + return .system(message.content) + case .user: + return .user(message.content, images: message.images) + case .assistant: + return .assistant(message.content + calls) + case .tool: + return .tool(results: [.success(toolCallId: message.toolCallId ?? "", toolName: nil, + result: message.content, images: message.images)]) + } + } + return conversation.estimateTotalTokens(asStored) + toolDefinitionTokens(tools) + } + + /// The tool definitions' share of a request, at the conversation's + /// characters per token. + func toolDefinitionTokens(_ tools: [LLMToolDefinition]) -> Int { + let chars = tools.reduce(0) { $0 + Self.toolDefinitionChars($1) } + return Int((Double(chars) / conversation.charsPerToken).rounded(.up)) + } + + /// A tool definition's size on the wire: name, description and the + /// sorted-key JSON of its schema. + static func toolDefinitionChars(_ tool: LLMToolDefinition) -> Int { + let schema = (try? JSONSerialization.data(withJSONObject: tool.parameters, options: [.sortedKeys]))?.count ?? 0 + return tool.name.count + tool.description.count + schema + } + + /// The overflow safety net (see `makeLLMRequest`): `sifted` is the sift + /// of `messagesForLLM` (the whole stored history plus any per-call tail). + private func overflowSafeMessages(_ messagesForLLM: [AgentMessage], sifted: [LLMMessage], + tools: [LLMToolDefinition], manager: ContextManager) async -> [LLMMessage] { + let stored = conversation.allMessages() + let transient = messagesForLLM.count > stored.count ? Array(messagesForLLM.dropFirst(stored.count)) : [] + let units = Self.overflowUnits(stored, task: runTaskID) + var current = 0 + if let after = overflowCutAfter { + if let index = stored.firstIndex(where: { $0.id == after }) { + current = units.prefix { $0.last! <= index }.count + } else { + overflowCutAfter = nil // compacted or capped away + } + } + let bound = overflowBoundTokens + let target = Int(Double(bound) * min(1, max(0, manager.overflowTargetFraction))) + let full = estimatedRequestTokens(sifted, tools: tools) + guard current > 0 || full > bound else { return sifted } + + // Each candidate cut is a trial sift: it starts from, and leaves, the + // state the full sift committed; only the cut chosen commits its own. + var built: [Int: (messages: [LLMMessage], tokens: Int, state: ContextManager.SiftState)] = + [0: (sifted, full, manager.siftState)] + func build(_ cut: Int) async -> (messages: [LLMMessage], tokens: Int, state: ContextManager.SiftState) { + if let done = built[cut] { return done } + let dropped = Set(units.prefix(cut).flatMap { $0 }) + let kept = stored.indices.filter { !dropped.contains($0) }.map { stored[$0] } + transient + let trial = await Self.trialSift(kept, manager: manager) { [state] content in state.template(content) } + let result = (trial.messages, estimatedRequestTokens(trial.messages, tools: tools), trial.state) + built[cut] = result + return result + } + let cut = await Self.overflowCut(current: current, units: units.count, bound: bound, target: target) { cut in + await build(cut).tokens + } + let chosen = await build(cut) + manager.siftState = chosen.state + if cut != current { + overflowCutAfter = cut > 0 ? stored[units[cut - 1].last!].id : nil + let removed = units.prefix(cut).reduce(0) { $0 + $1.count } + emit(.historyTrimmed(removedCount: removed, remainingCount: stored.count - removed, reason: .overflow)) + } + if chosen.tokens > bound { + logger.warning("Overflow net: the request (~\(chosen.tokens) tokens) still passes its bound (\(bound)) with every evictable step left out") + } + return chosen.messages + } + private func makeLLMRequest( messagesForLLM: [AgentMessage], - tools: [LLMToolDefinition] = [] + tools: [LLMToolDefinition] = [], + trailingNote: String? = nil ) async -> LLMRequest { - let llmMessages: [LLMMessage] + var llmMessages: [LLMMessage] if let contextManager = config.contextManager { // ContextSift-style: externalize completed tool exchanges. llmMessages = await contextManager.modelMessages(messagesForLLM) { [state] content in state.template(content) } + // Safety net: the sifted request, tool definitions counted, + // passes `overflowFraction` of the window less the output + // reserve — compaction off or set later, or a model that would + // truncate it silently (local Ollama drops the system prompt and + // tools first). Calls then leave out the oldest whole steps, + // measured by what is SENT (sifted), the fewest that bring the + // request to the low watermark; that cut then holds, so the + // prefix stays byte-identical until the bound is passed again. + // Nothing stored is deleted; a set or moved cut is reported + // (`.overflow`). Messages the caller added past the stored + // history for this call only are kept after it (the progress + // note is not one: it is appended after this, below). + llmMessages = await overflowSafeMessages(messagesForLLM, sifted: llmMessages, tools: tools, + manager: contextManager) } else { llmMessages = messagesForLLM.flatMap { msg -> [LLMMessage] in if msg.role == .system { @@ -1804,11 +2012,13 @@ public actor Agent { } } + // A note for this call only goes last, after ContextSift has run, so it + // can't disturb the active-step detection or the prompt prefix. + if let trailingNote { llmMessages.append(.user(trailingNote)) } + // Record the size of what we actually send (post context-management), so // an app can show real context usage rather than raw-history size. - let promptChars = llmMessages.reduce(0) { $0 + $1.content.count } - let estimate = Int((Double(promptChars) / 3.5).rounded()) + llmMessages.count * 4 - lastPromptTokens = estimate + lastPromptTokens = Self.estimatedTokens(llmMessages) if firstRequestPending { firstRequestPending = false firstRequestSystemDigest = PromptDigest.hex( diff --git a/Sources/SwiftAgentKit/Core/AgentEvent.swift b/Sources/SwiftAgentKit/Core/AgentEvent.swift index bea3f17..e063478 100644 --- a/Sources/SwiftAgentKit/Core/AgentEvent.swift +++ b/Sources/SwiftAgentKit/Core/AgentEvent.swift @@ -107,8 +107,10 @@ public enum AgentEvent: Sendable { // MARK: - Memory - /// History was trimmed to fit the context window. - case historyTrimmed(removedCount: Int, remainingCount: Int) + /// History was trimmed: `removedCount` messages left out (or deleted, + /// by `reason`), `remainingCount` kept. `.overflow` is the signal to + /// compact: the request no longer fits without leaving steps out. + case historyTrimmed(removedCount: Int, remainingCount: Int, reason: HistoryTrimReason) /// The older part of the history was replaced by a summary. case contextCompacted(tokensBefore: Int, tokensAfter: Int, reason: CompactionReason) @@ -223,4 +225,21 @@ public final class BlockObserver: AgentObserver { public func onEvent(_ event: AgentEvent) { block(event) } -} \ No newline at end of file +} +/// Why history was trimmed (`AgentEvent.historyTrimmed`). +public enum HistoryTrimReason: String, Sendable { + /// Without a ContextManager: this call sends the stored history trimmed + /// to fit the window. Nothing stored is deleted. + case fitToWindow + /// The message-count cap (`maxMessages`) deleted the oldest stored messages. + case messageCap + /// Without a ContextManager: the token budget deleted the oldest stored + /// messages after a step. + case tokenBudget + /// With a ContextManager: the sifted request, tool definitions counted, + /// passed `ContextManager.overflowFraction` of the window, so calls now + /// leave out the oldest whole steps (stored history is kept). Emitted + /// when that cut is set or moves, not on every call it holds. An app + /// that compacts should compact on it. + case overflow +} diff --git a/Sources/SwiftAgentKit/Memory/Conversation.swift b/Sources/SwiftAgentKit/Memory/Conversation.swift index 1fd1417..4024424 100644 --- a/Sources/SwiftAgentKit/Memory/Conversation.swift +++ b/Sources/SwiftAgentKit/Memory/Conversation.swift @@ -99,15 +99,34 @@ public class Conversation: @unchecked Sendable { /// Get messages trimmed to fit the context window. public func messagesForLLMCall() -> [AgentMessage] { + messagesForLLMCall(reservingTokens: 0) + } + + /// Messages trimmed to fit `fitBudgetTokens` less `reservedTokens`: what + /// the call sends besides the messages (its tool definitions). + public func messagesForLLMCall(reservingTokens reservedTokens: Int) -> [AgentMessage] { lock.lock() defer { lock.unlock() } - return ensureContextWindowFits(messages: messages) + return ensureContextWindowFits(messages: messages, reserving: reservedTokens) + } + + /// The fit's bound: 80% of the window less the output reserve. The 20% + /// left over absorbs the estimator's error. + public var fitBudgetTokens: Int { + max(0, Int(Double(contextWindow - outputReserve) * 0.8)) } // MARK: - Trim /// Trim history to stay within both message-count and token-budget limits. public func trim() -> (removed: Int, remaining: Int) { + trim(byTokens: true) + } + + /// Trim history: the message-count cap, and — when `byTokens` — the + /// token budget. An agent with a ContextManager passes false: ContextSift + /// bounds what is sent, and the token trim deleted stored history for good. + public func trim(byTokens: Bool) -> (removed: Int, remaining: Int) { lock.lock() defer { lock.unlock() } @@ -122,7 +141,7 @@ public class Conversation: @unchecked Sendable { } // 2. Token-budget trim - trimmed = trimByTokens(trimmed) + if byTokens { trimmed = trimByTokens(trimmed) } let removed = messages.count - trimmed.count messages = trimmed @@ -252,9 +271,9 @@ public class Conversation: @unchecked Sendable { return trimmed + [lastUser] } - /// Trim to fit within 80% of the context window. - private func ensureContextWindowFits(messages: [AgentMessage]) -> [AgentMessage] { - let budget = max(0, Int(Double(contextWindow - outputReserve) * 0.8)) + /// Trim to fit within 80% of the context window, less `reserving` tokens. + private func ensureContextWindowFits(messages: [AgentMessage], reserving: Int = 0) -> [AgentMessage] { + let budget = max(0, fitBudgetTokens - reserving) var trimmed = messages while estimateTotalTokens(trimmed) > budget && trimmed.contains(where: { $0.role != .system }) { diff --git a/Sources/SwiftAgentKit/SubAgents/SubAgentSpawner.swift b/Sources/SwiftAgentKit/SubAgents/SubAgentSpawner.swift index 0987b4f..3b64ea1 100644 --- a/Sources/SwiftAgentKit/SubAgents/SubAgentSpawner.swift +++ b/Sources/SwiftAgentKit/SubAgents/SubAgentSpawner.swift @@ -138,15 +138,9 @@ public final class SubAgentSpawner: @unchecked Sendable { config.systemPrompt = prompt if let parentCM = config.contextManager { - config.contextManager = ContextManager( - store: parentCM.store, - maxActiveResultChars: parentCM.maxActiveResultChars, - ledgerEntries: parentCM.ledgerEntries, - summaryLength: parentCM.summaryLength, - inlineBudgetChars: parentCM.inlineBudgetChars, - keepLatestReadsInline: parentCM.keepLatestReadsInline, - readToolNames: parentCM.readToolNames - ) + // Every setting, readIdentityParams included: without it a child's + // sim_ui/mac_ui reads were never protected. + config.contextManager = parentCM.childManager() } let child = Agent(config: config) diff --git a/Tests/SwiftAgentKitReplayTests/ContextChurnGuardTests.swift b/Tests/SwiftAgentKitReplayTests/ContextChurnGuardTests.swift index 5f0ac96..f120dac 100644 --- a/Tests/SwiftAgentKitReplayTests/ContextChurnGuardTests.swift +++ b/Tests/SwiftAgentKitReplayTests/ContextChurnGuardTests.swift @@ -53,17 +53,18 @@ struct ContextChurnGuardTests { contextManager: cm, loopDetection: nil) _ = try await run.run("drive the app") - // The LAST request the loop built must contain the newest snapshots inline - // and must NOT carry the oldest one's full result in non-system messages - // (SNAP1 externalized to ledger — a summary appears in the system message, - // but the full inline content must be gone from tool/user/assistant messages). + // The LAST request the loop built must carry the newest snapshots as tool + // results and must NOT carry the oldest one's result as one: SNAP1 is + // evicted, and its receipt (which may quote its first characters) + // stands in its step's place as an assistant message. let last = try #require(run.capturedRequests.last) - let nonSystemText = last.messages - .filter { $0.role != .system } + let toolText = last.messages + .filter { $0.role == .tool } .map { $0.content } .joined(separator: "\n") - #expect(nonSystemText.contains("SNAP3")) // active exchange kept inline - #expect(nonSystemText.contains("SNAP2")) // newest completed read protected inline - #expect(!nonSystemText.contains("SNAP1")) // oldest externalized to ledger + #expect(toolText.contains("SNAP3")) // active exchange kept inline + #expect(toolText.contains("SNAP2")) // newest completed read protected inline + #expect(!toolText.contains("SNAP1")) // oldest evicted + #expect(last.messages.contains { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }) } } diff --git a/Tests/SwiftAgentKitReplayTests/ReceiptWireShapeTests.swift b/Tests/SwiftAgentKitReplayTests/ReceiptWireShapeTests.swift new file mode 100644 index 0000000..bc38b1e --- /dev/null +++ b/Tests/SwiftAgentKitReplayTests/ReceiptWireShapeTests.swift @@ -0,0 +1,80 @@ +import Testing +import Foundation +import LLMProviderKit +import LLMProviderKitAnthropic +import LLMProviderKitGemini +import SwiftAgentKit + +/// Receipts in place make an assistant message followed by another +/// assistant message the normal shape of a sifted request. The two strictest +/// wires must still pair every tool result with its own call, and Gemini +/// must never see a model turn right before a function-call turn. +struct ReceiptWireShapeTests { + static func siftedMessages() async -> [LLMMessage] { + let manager = ContextManager(summaryLength: 40, inlineBudgetChars: 0) + var messages: [AgentMessage] = [.system("sys"), .user("do three things")] + for i in 1...3 { + messages.append(.assistant(content: "", toolCalls: [AgentToolCall(id: "c\(i)", name: "run_shell")])) + messages.append(.tool(results: [.success(toolCallId: "c\(i)", toolName: "run_shell", + result: "output \(i) " + String(repeating: "x", count: 200))])) + } + // A fourth step still running: its call and result stay inline. + messages.append(.assistant(content: "", toolCalls: [AgentToolCall(id: "c4", name: "run_shell")])) + messages.append(.tool(results: [.success(toolCallId: "c4", toolName: "run_shell", result: "output 4")])) + return await manager.modelMessages(messages) { $0 } + } + + @Test func theSiftedShapeHasConsecutiveAssistantMessages() async { + #expect(await Self.siftedMessages().map(\.role) + == [.system, .user, .assistant, .assistant, .assistant, .assistant, .tool]) + } + + @Test func anthropicPairsEveryToolResultWithItsCall() async throws { + let request = LLMRequest(model: "claude-sonnet-4-6", messages: await Self.siftedMessages()) + let provider = AnthropicProvider(configuration: AnthropicProvider.anthropic(apiKey: "k", model: "claude-sonnet-4-6")) + let body = try #require(try provider.prepareRequest(request, stream: false).httpBody) + let json = try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any]) + let messages = try #require(json["messages"] as? [[String: Any]]) + var results = 0 + for (i, message) in messages.enumerated() { + guard let blocks = message["content"] as? [[String: Any]] else { continue } + for block in blocks where block["type"] as? String == "tool_result" { + results += 1 + let previous = i > 0 ? (messages[i - 1]["content"] as? [[String: Any]] ?? []) : [] + let ids = previous.filter { $0["type"] as? String == "tool_use" }.compactMap { $0["id"] as? String } + #expect(ids.contains(block["tool_use_id"] as? String ?? "")) + } + } + #expect(results == 1) + #expect(String(decoding: body, as: UTF8.self).contains("Tool calls in this step")) + } + + @Test func geminiPairsEveryFunctionResponseWithItsCall() async throws { + let request = LLMRequest(model: "gemini-2.0-flash", messages: await Self.siftedMessages()) + let provider = GeminiProvider(configuration: GeminiProvider.gemini(apiKey: "k", model: "gemini-2.0-flash")) + let body = try #require(try provider.prepareRequest(request, stream: false).httpBody) + let json = try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any]) + let contents = try #require(json["contents"] as? [[String: Any]]) + var responses = 0 + for (i, content) in contents.enumerated() { + for part in content["parts"] as? [[String: Any]] ?? [] where part["functionResponse"] != nil { + responses += 1 + let previous = i > 0 ? (contents[i - 1]["parts"] as? [[String: Any]] ?? []) : [] + #expect(previous.contains { $0["functionCall"] != nil }) + } + } + #expect(responses == 1) + } + + /// Gemini rejects a function-call turn that follows a model turn; since + /// LLMProviderKit 0.1.0-alpha.40 a run of one role is one content. + @Test func geminiNeverSendsTwoContentsOfOneRoleInARow() async throws { + let request = LLMRequest(model: "gemini-2.0-flash", messages: await Self.siftedMessages()) + let provider = GeminiProvider(configuration: GeminiProvider.gemini(apiKey: "k", model: "gemini-2.0-flash")) + let body = try #require(try provider.prepareRequest(request, stream: false).httpBody) + let json = try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any]) + let roles = try #require(json["contents"] as? [[String: Any]]).compactMap { $0["role"] as? String } + // user (task), model (three receipts + the active call), user (its result) + #expect(roles == ["user", "model", "user"]) + } +} diff --git a/Tests/SwiftAgentKitTests/AnthropicRoleAlternationTests.swift b/Tests/SwiftAgentKitTests/AnthropicRoleAlternationTests.swift new file mode 100644 index 0000000..9a366e0 --- /dev/null +++ b/Tests/SwiftAgentKitTests/AnthropicRoleAlternationTests.swift @@ -0,0 +1,117 @@ +import Testing +import Foundation +import LLMProviderKit +import LLMProviderKitAnthropic +@testable import SwiftAgentKit + +/// Regression for the Anthropic 400 flagged in the task-14 review: a +/// tool-result turn followed, on the very next call, by a STORED +/// "stop reasoning, act now" nudge AND a transient progress note used to put +/// three consecutive `user`-mapped entries on the wire (tool_result, the +/// stored nudge, the trailing note) — all three built as `.user` messages by +/// `Agent`, none of them merged. Anthropic rejects consecutive same-role +/// messages with HTTP 400. +/// +/// LLMProviderKit 0.1.0-alpha.41 fixes this in the adapter itself +/// (`AnthropicProvider.mergingConsecutiveRoles` merges any run of one role, +/// not just tool-result blocks). This test builds the actual wire body via +/// `AnthropicProvider.prepareRequest` — not `ScriptedProvider` at the +/// `LLMMessage` level — so a future regression in either SwiftAgentKit's +/// message ordering or the adapter's merge would fail here. +struct AnthropicRoleAlternationTests { + private struct EchoTool: AgentTool { + let name = "echo" + let description = "Echo." + let parameters = ToolParameters(properties: [ + "n": ToolParameterProperty(type: "integer", description: "which"), + ], required: ["n"]) + func execute(parameters: [String: Any]) async throws -> AgentToolResult { + .success(toolCallId: "", toolName: name, result: "echoed") + } + } + + /// Call 1: a tool call (produces a stored tool-result turn). Call 2: + /// reasons past `maxReasoningSeconds` without acting — stored as the + /// "stop reasoning, act now" nudge. Call 3: answers plainly, and is also + /// the turn `nudgeTurns(maxTurns: 4, fractions: [0.75])` picks (turn 3), + /// so its request carries both the stored nudge from call 2 AND the + /// transient progress note — the exact collision the review flagged. + private final class ToolThenStalledReasoningThenDoneProvider: LLMProvider, @unchecked Sendable { + static let name = "tool-then-stalled-reasoning" + let configuration = LLMProviderConfiguration( + name: name, baseURL: URL(string: "inprocess://t")!, defaultModel: "mock") + private let lock = NSLock() + private var calls = 0 + private var requests: [LLMRequest] = [] + var captured: [LLMRequest] { lock.withLock { requests } } + + func complete(_ request: LLMRequest) async throws -> LLMResponse { + LLMResponse(text: "done", finishReason: .stop, request: request, providerName: Self.name) + } + + func stream(_ request: LLMRequest) -> AsyncThrowingStream { + let n: Int = lock.withLock { calls += 1; requests.append(request); return calls } + return AsyncThrowingStream { continuation in + let task = Task { + switch n { + case 1: + continuation.yield(.toolCall(LLMToolCall(id: "e1", name: "echo", arguments: #"{"n":1}"#))) + continuation.yield(.finish(reason: .toolCalls, usage: nil)) + case 2: + for _ in 0..<200 { // stopped early by maxReasoningSeconds + if Task.isCancelled { break } + continuation.yield(.reasoning("hmm ")) + try? await Task.sleep(nanoseconds: 50_000_000) + } + continuation.yield(.finish(reason: .stop, usage: nil)) + default: + continuation.yield(.text("done")) + continuation.yield(.finish(reason: .stop, usage: nil)) + } + continuation.finish() + } + continuation.onTermination = { _ in task.cancel() } + } + } + } + + /// The `messages` array of the wire body `AnthropicProvider.prepareRequest` + /// would send for `request`. + private static func anthropicMessages(_ request: LLMRequest) throws -> [[String: Any]] { + let provider = AnthropicProvider(configuration: AnthropicProvider.anthropic(apiKey: "k", model: "claude-sonnet-4-6")) + let body = try #require(try provider.prepareRequest(request, stream: false).httpBody) + let object = try JSONSerialization.jsonObject(with: body) as? [String: Any] + return try #require(object?["messages"] as? [[String: Any]]) + } + + @Test func rolesAlternateWithTheNoteLastOnTheCollisionTurn() async throws { + let provider = ToolThenStalledReasoningThenDoneProvider() + let agent = Agent(config: AgentConfig( + provider: provider, model: "mock", systemPrompt: "BASE", maxTurns: 4, + tools: [EchoTool()], contextManager: nil, loopDetection: nil, + progressNudgeFractions: [0.75], maxReasoningSeconds: 0.15)) + var text = "" + for try await chunk in agent.runStreaming("go") { text += chunk } + #expect(text == "done") + + let requests = provider.captured + #expect(requests.count == 3) + + // Turn 3's request: tool_result (call 1), the stored stop-reasoning + // nudge (call 2's "continue"), and the transient progress note — all + // three built by Agent as `.user`-mapped content. + let messages = try Self.anthropicMessages(requests[2]) + let roles = messages.compactMap { $0["role"] as? String } + #expect(roles.count == messages.count, "every message dict carries a role") + for (a, b) in zip(roles, roles.dropFirst()) { + #expect(a != b, "consecutive same-role entries were not merged: \(roles)") + } + + // The note is the last text block of the last (merged) user entry. + let lastContent = try #require(messages.last?["content"] as? [[String: Any]]) + let lastBlock = try #require(lastContent.last) + #expect(lastBlock["type"] as? String == "text") + let noteText = try #require(lastBlock["text"] as? String) + #expect(noteText == Agent.progressNudge(turn: 3, maxTurns: 4)) + } +} diff --git a/Tests/SwiftAgentKitTests/CacheStableRequestTests.swift b/Tests/SwiftAgentKitTests/CacheStableRequestTests.swift index a45ea0f..5547502 100644 --- a/Tests/SwiftAgentKitTests/CacheStableRequestTests.swift +++ b/Tests/SwiftAgentKitTests/CacheStableRequestTests.swift @@ -157,6 +157,12 @@ extension CacheStableRequestTests { return try #require(try provider.prepareRequest(request, stream: false).httpBody) } + /// The same request on the streaming path (`stream: true`). + static func streamWireBody(_ request: LLMRequest) throws -> Data { + let provider = OpenAIProvider(configuration: OpenAIProvider.openAI(apiKey: "k", model: "mock")) + return try #require(try provider.prepareRequest(request, stream: true).httpBody) + } + /// The body's bytes up to (not including) the `]` that closes its messages /// array — "messages" is the first key of a sorted OpenAI body. static func messagesPrefix(_ body: Data) -> Data? { @@ -256,3 +262,95 @@ extension CacheStableRequestTests { #expect(String(decoding: bodies[3], as: UTF8.self).contains(#"\"limit\":1,"#)) } } + +private struct StepTool: AgentTool { + let name = "run_step" + let description = "Run one step." + let parameters = ToolParameters(properties: [ + "n": ToolParameterProperty(type: "integer", description: "step"), + ], required: ["n"]) + func execute(parameters: [String: Any]) async throws -> AgentToolResult { + .success(toolCallId: "", toolName: name, result: String(repeating: "o", count: 600)) + } +} + +extension CacheStableRequestTests { + static func stepCalls(_ range: ClosedRange) -> [[LLMToolCall]] { + range.map { [LLMToolCall(id: "s\($0)", name: "run_step", arguments: #"{"n":\#($0)}"#)] } + } + + /// For each consecutive pair of requests: nil when the later wire body + /// starts with the earlier one's messages (append-only); otherwise the + /// index of the first message that differs. + static func breakIndices(_ requests: [LLMRequest]) throws -> [Int?] { + try zip(requests, requests.dropFirst()).map { earlier, later in + let prefix = try #require(messagesPrefix(try wireBody(earlier))) + if try wireBody(later).starts(with: prefix) { return nil } + let pairs = Array(zip(earlier.messages, later.messages)) + return pairs.firstIndex { $0.0 != $0.1 } ?? pairs.count + } + } + + /// With a ContextSift batch between two calls, exactly that pair breaks, + /// and only from the first evicted step on: the system prompt and the + /// task are the same bytes, and the tool definitions never change. + @Test func anEvictionBatchBreaksThePrefixOnlyFromItsFirstEvictedStep() async throws { + let provider = ScriptedProvider(turns: Self.stepCalls(1...6) + [[]]) + // Progress notes off: each is a tail-only break of its own (see + // aProgressNoteChangesOnlyTheTail), and this test counts batch breaks. + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", + maxTurns: 8, tools: [StepTool()], + contextManager: ContextManager(inlineBudgetChars: 2_500), + loopDetection: nil, progressNudgeFractions: [])) + _ = try await agent.run("do six steps") + + let requests = provider.captured + #expect(requests.count == 7) + let breaks = try Self.breakIndices(requests) + // Budget ≈ 2,500 − the system prompt (about 470 characters): the fifth + // call is the first over it, and one batch evicts steps 1–3. + #expect(breaks.compactMap { $0 } == [2], "one batch, starting at the first step") + let batch = try #require(breaks.firstIndex { $0 != nil }) + let later = requests[batch + 1] + #expect(later.messages[2].content.hasPrefix(ContextManager.receiptHeader)) + // Byte level: both bodies start with the same system prompt and task. + let shared = try #require(Self.messagesPrefix(try Self.wireBody( + LLMRequest(model: "mock", messages: Array(later.messages.prefix(2)), tools: later.tools)))) + #expect(try Self.wireBody(requests[batch]).starts(with: shared)) + #expect(try Self.wireBody(later).starts(with: shared)) + let tools = try requests.map { Self.toolsSection(try Self.wireBody($0)) } + #expect(Set(tools.compactMap { $0 }).count == 1) + + // The streaming body (`stream: true`) holds the same promise: append-only + // between batches, and the batch call keeps the system prompt and task. + let streamed = try requests.map(Self.streamWireBody) + for (pair, index) in breaks.enumerated() { + let prefix = try #require(Self.messagesPrefix(streamed[pair])) + #expect(streamed[pair + 1].starts(with: prefix) == (index == nil), "streamed pair \(pair)") + } + #expect(streamed[batch].starts(with: shared) && streamed[batch + 1].starts(with: shared)) + } + + /// A progress note is for its call only: the next call drops it, so that + /// pair breaks exactly at the note — the earlier request's last message — + /// and nothing before it changes. (Default nudges: turns 4 and 6 of 8.) + @Test func aProgressNoteChangesOnlyTheTail() async throws { + let provider = ScriptedProvider(turns: Self.stepCalls(1...6) + [[]]) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", + maxTurns: 8, tools: [StepTool()], + contextManager: ContextManager(inlineBudgetChars: 100_000), + loopDetection: nil)) + _ = try await agent.run("do six steps") + let requests = provider.captured + #expect(requests.count == 7) + let breaks = try Self.breakIndices(requests) + #expect(breaks.compactMap { $0 }.count == 2) + for (pair, index) in breaks.enumerated() { + guard let index else { continue } + let noted = requests[pair].messages + #expect(index == noted.count - 1) + #expect(noted.last?.role == .user) + #expect(noted.last?.content.hasPrefix("[Progress check]") == true) + } + } +} diff --git a/Tests/SwiftAgentKitTests/ContextSiftBatchTests.swift b/Tests/SwiftAgentKitTests/ContextSiftBatchTests.swift new file mode 100644 index 0000000..625aa58 --- /dev/null +++ b/Tests/SwiftAgentKitTests/ContextSiftBatchTests.swift @@ -0,0 +1,204 @@ +import Testing +import Foundation +import LLMProviderKit +@testable import SwiftAgentKit + +/// ContextSift that keeps the prompt prefix: evictions only in real batches, +/// the system prompt off the sifted size (and off the budget), receipts in +/// place, images kept, a child manager with every setting. +struct ContextSiftBatchTests { + /// One completed (or, when last, active) tool step of `size` characters. + static func step(_ i: Int, size: Int) -> [AgentMessage] { + [.assistant(content: "", toolCalls: [AgentToolCall(id: "st\(i)", name: "run_shell")]), + .tool(results: [.success(toolCallId: "st\(i)", toolName: "run_shell", + result: String(repeating: "o", count: size))])] + } + + /// How many calls did not extend the previous call's messages unchanged — + /// each one is a prompt-cache break. + static func breaks(_ outputs: [[LLMMessage]]) -> Int { + zip(outputs, outputs.dropFirst()).filter { earlier, later in + later.count < earlier.count || zip(earlier, later).contains { $0 != $1 } + }.count + } + + static func count(_ candidates: [Int], remaining: Int, budget: Int) -> Int { + ContextManager.evictionCount(candidates: candidates, remaining: remaining, budget: budget, + targetFraction: 0.5, minBatchFraction: 0.25, slackFraction: 0.1) + } + + @Test func nothingIsEvictedUnderTheBudget() { + #expect(Self.count([500, 500], remaining: 900, budget: 1_000) == 0) + } + + @Test func lessThanAMinimumBatchWaits() { + // Over budget, but only 200 of evictable output: wait for a real batch. + #expect(Self.count([100, 100], remaining: 1_500, budget: 1_000) == 0) + } + + @Test func aBatchFreesDownToTheTarget() { + // floor 100, target max(500, 100 + 100) = 500: free at least 800 → three steps. + #expect(Self.count([300, 300, 300, 300], remaining: 1_300, budget: 1_000) == 3) + } + + @Test func aFullFloorStillEvictsAWholeBatch() { + // floor 1,200 is over the budget: target 1,300, so at least the minimum + // batch (250) goes in one pass — three steps — never one per call. + #expect(Self.count([100, 100, 100], remaining: 1_500, budget: 1_000) == 3) + } + + @Test func aZeroBudgetEvictsEverything() { + #expect(Self.count([10, 20], remaining: 40, budget: 0) == 2) + } + + @Test func theSystemPromptComesOffTheBudget() { + let manager = ContextManager(inlineBudgetChars: 10_000) + #expect(manager.messageBudget(systemChars: 0) == 10_000) + #expect(manager.messageBudget(systemChars: 4_000) == 6_000) + // Never below a quarter of the budget, however long the system prompt is. + #expect(manager.messageBudget(systemChars: 9_000) == 2_500) + #expect(manager.messageBudget(systemChars: 50_000) == 2_500) + } + + /// The system prompt is not part of the sifted size: a system prompt that + /// alone nearly fills the budget leaves the messages a quarter of it, + /// instead of making every completed step evictable. + @Test func aLongSystemPromptLeavesTheMessagesAQuarterOfTheBudget() async { + let manager = ContextManager(inlineBudgetChars: 4_000) + let messages: [AgentMessage] = [.system(String(repeating: "s", count: 3_900)), .user("go")] + + Self.step(0, size: 300) + Self.step(1, size: 300) + [.assistant("done"), .user("next")] + let out = await manager.modelMessages(messages) { $0 } + #expect(out.filter { $0.role == .tool }.count == 2) // nothing evicted: 610 ≤ 1,000 + } + + /// A protected read fills the budget by itself. Before batches, every call + /// then evicted the step that had just finished, and the prefix changed on + /// every call; now it changes once per minimum batch of new output. + @Test func aFloorAtTheBudgetDoesNotChangeThePrefixOnEveryCall() async { + let manager = ContextManager(maxActiveResultChars: 10_000, inlineBudgetChars: 4_000) + var messages: [AgentMessage] = [ + .user("task"), + .assistant(content: "", toolCalls: [AgentToolCall(id: "r1", name: "read_file", + parameters: ["path": AnyCodable("/big")])]), + .tool(results: [.success(toolCallId: "r1", toolName: "read_file", + result: String(repeating: "r", count: 4_200))]), + ] + var outputs: [[LLMMessage]] = [] + for i in 0..<20 { + messages += Self.step(i, size: 200) + outputs.append(await manager.modelMessages(messages) { $0 }) + } + // Minimum batch 1,000 = five 200-character steps: batches at calls 6, 11 and 16. + #expect(Self.breaks(outputs) == 3) + } +} + +extension ContextSiftBatchTests { + @Test func anEvictedStepBecomesOneAssistantMessageInItsPlace() async { + let manager = ContextManager(summaryLength: 40, inlineBudgetChars: 0) + let messages: [AgentMessage] = [ + .system("sys"), + .user("task"), + .assistant(content: "Checking both.", toolCalls: [ + AgentToolCall(id: "a", name: "run_shell", parameters: ["command": AnyCodable("ls")]), + AgentToolCall(id: "b", name: "fetch_url", parameters: ["url": AnyCodable("https://x")]), + ]), + .tool(results: [.success(toolCallId: "a", toolName: "run_shell", result: "a.txt"), + .success(toolCallId: "b", toolName: "fetch_url", result: "hello")]), + .assistant("Done."), + .user("next"), + ] + let out = await manager.modelMessages(messages) { $0 } + #expect(out.map(\.role) == [.system, .user, .assistant, .assistant, .user]) + #expect(out[0].content == "sys") // the system prompt alone + #expect(out[2].content == "Checking both.\n\n" + ContextManager.receiptHeader + + "\n- a run_shell(ls) → OK: a.txt\n- b fetch_url(https://x) → OK: hello\n]") + #expect(out[2].toolCalls == nil) + } + + /// Calls through several batches: each break is a batch, and it starts at + /// the first newly evicted step — the system prompt, the task and every + /// earlier receipt are the same as before. + @Test func aLaterBatchKeepsEverythingBeforeItsFirstEvictedStep() async { + let manager = ContextManager(inlineBudgetChars: 2_000) + var messages: [AgentMessage] = [.system("sys"), .user("long task")] + var outputs: [[LLMMessage]] = [] + for i in 0..<20 { + messages += Self.step(i, size: 300) + outputs.append(await manager.modelMessages(messages) { $0 }) + } + var breaks = 0 + for (earlier, later) in zip(outputs, outputs.dropFirst()) { + guard let first = zip(earlier, later).enumerated().first(where: { $0.element.0 != $0.element.1 })?.offset + else { continue } + breaks += 1 + #expect(first >= 2, "the system prompt and the task never change") + #expect(later[first].role == .assistant) + #expect(later[first].content.hasPrefix(ContextManager.receiptHeader)) + #expect(earlier[first].toolCalls?.isEmpty == false) // it was that step's call + } + // Budget 1,997 (2,000 − "sys"), minimum batch 500: four steps per batch, + // at calls 7, 11, 15 and 19. + #expect(breaks == 4) + } + + @Test func siftedToolResultsKeepTheirImages() async { + let manager = ContextManager(inlineBudgetChars: 0) + let shot = LLMImage(data: Data([1, 2, 3]), mimeType: "image/png") + let messages: [AgentMessage] = [ + .user("look"), + .assistant(content: "", toolCalls: [AgentToolCall(id: "s1", name: "mac_screenshot")]), + .tool(results: [.success(toolCallId: "s1", toolName: "mac_screenshot", result: "captured", images: [shot])]), + ] + let out = await manager.modelMessages(messages) { $0 } + #expect(out.first { $0.role == .tool }?.images.map(\.base64) == [shot.base64]) + } + + /// Crossing into sifting alone changes nothing: a large completed result + /// the normal path sent whole is still sent whole until a batch, and is + /// bounded (or evicted) only as part of one. + @Test func enteringSiftingDoesNotTruncateACompletedResult() async { + let manager = ContextManager(maxActiveResultChars: 1_000, inlineBudgetChars: 6_000) + manager.minEvictionBatchFraction = 1.0 // a batch needs 6,000 evictable; there is 5,000 + var messages: [AgentMessage] = [.user("task")] + Self.step(0, size: 5_000) + [.assistant("ok")] + let before = await manager.modelMessages(messages) { $0 } // 5,006 ≤ 6,000: the normal path + messages += Self.step(1, size: 1_200) // 6,206: sifting, but no batch + let sifted = await manager.modelMessages(messages) { $0 } + #expect(Array(sifted.prefix(before.count)) == before) // step 0 still whole + #expect(!sifted.contains { $0.content.hasPrefix(ContextManager.receiptHeader) }) + + // With a batch, the large completed step goes, as a receipt. + let batching = ContextManager(maxActiveResultChars: 1_000, inlineBudgetChars: 6_000) + let batched = await batching.modelMessages(messages) { $0 } + #expect(batched.contains { $0.role == .assistant && $0.content.hasPrefix(ContextManager.receiptHeader) }) + } +} + +extension ContextSiftBatchTests { + @Test func aChildManagerCopiesEverySetting() { + let parent = ContextManager(maxActiveResultChars: 1_234, ledgerEntries: 7, summaryLength: 99, + inlineBudgetChars: 5_555, keepLatestReadsInline: false, + readToolNames: ["read_file", "sim_ui"], + readIdentityParams: ["sim_ui": "bundle_id"]) + parent.eagerPersistMinChars = 42 + parent.evictionTargetFraction = 0.4 + parent.minEvictionBatchFraction = 0.3 + parent.evictionSlackFraction = 0.2 + parent.minMessageBudgetFraction = 0.35 + let child = parent.childManager() + #expect(child !== parent) + #expect(child.store === parent.store) + #expect(child.maxActiveResultChars == 1_234) + #expect(child.ledgerEntries == 7) + #expect(child.summaryLength == 99) + #expect(child.inlineBudgetChars == 5_555) + #expect(child.keepLatestReadsInline == false) + #expect(child.readToolNames == ["read_file", "sim_ui"]) + #expect(child.readIdentityParams == ["sim_ui": "bundle_id"]) + #expect(child.eagerPersistMinChars == 42) + #expect(child.evictionTargetFraction == 0.4) + #expect(child.minEvictionBatchFraction == 0.3) + #expect(child.evictionSlackFraction == 0.2) + #expect(child.minMessageBudgetFraction == 0.35) + } +} diff --git a/Tests/SwiftAgentKitTests/HistoryTrimTests.swift b/Tests/SwiftAgentKitTests/HistoryTrimTests.swift new file mode 100644 index 0000000..6bcb7e5 --- /dev/null +++ b/Tests/SwiftAgentKitTests/HistoryTrimTests.swift @@ -0,0 +1,393 @@ +import Testing +import Foundation +import LLMProviderKit +@testable import SwiftAgentKit + +/// With a ContextManager the agent sends from the whole stored history: +/// ContextSift bounds what is sent. The fit-to-80%-of-window trim used to +/// slide the window on every call once the STORED history passed it, and the +/// token trim after each step deleted history for good. What is left is an +/// overflow-only safety net, and every trim is reported. +struct HistoryTrimTests { + private struct OutputTool: AgentTool { + let name = "make_output" + let description = "Make some output." + let parameters = ToolParameters(properties: [ + "n": ToolParameterProperty(type: "integer", description: "which"), + ], required: ["n"]) + let size: Int + func execute(parameters: [String: Any]) async throws -> AgentToolResult { + .success(toolCallId: "", toolName: name, result: String(repeating: "x", count: size)) + } + } + + final class Trims: @unchecked Sendable { + private let lock = NSLock() + private var removed: [Int] = [] + func add(_ n: Int) { lock.withLock { removed.append(n) } } + var all: [Int] { lock.withLock { removed } } + } + + /// Four tool steps, then "done". + static func fourSteps() -> [[LLMToolCall]] { + (1...4).map { [LLMToolCall(id: "m\($0)", name: "make_output", arguments: #"{"n":\#($0)}"#)] } + [[]] + } + + private static func agent(_ provider: ScriptedProvider, size: Int, contextManager: ContextManager?, + contextWindow: Int = 4_096, maxMessages: Int = 50) -> (Agent, Trims) { + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", maxTurns: 6, + contextWindow: contextWindow, maxMessages: maxMessages, + tools: [OutputTool(size: size)], contextManager: contextManager, + loopDetection: nil)) + let trims = Trims() + agent.onEvent { event in + if case .historyTrimmed(let removed, _, _) = event { trims.add(removed) } + } + return (agent, trims) + } + + @Test func withAContextManagerTheStoredHistoryIsNotTrimmedFirst() async throws { + let provider = ScriptedProvider(turns: Self.fourSteps()) + // Window 6,144: the old trim line (80% of window − reserve) is 3,276 + // tokens. Four 4,000-character outputs put the stored history well + // past it; the sifted request (three short receipts, one output, the + // tool definitions) stays well under it, so the safety net never acts. + let (agent, trims) = Self.agent(provider, size: 4_000, + contextManager: ContextManager(summaryLength: 40, inlineBudgetChars: 2_000), + contextWindow: 6_144) + _ = try await agent.run("do the task") + // The stored history is past the old trim line… + let stored = agent.conversation.estimateTotalTokens(agent.conversation.allMessages()) + #expect(stored > agent.conversation.fitBudgetTokens) + // …yet the last request carries every step: three as receipts, the active one inline. + let last = try #require(provider.captured.last) + #expect(last.messages.filter { $0.role == .assistant }.count == 4) + #expect(last.messages.filter { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }.count == 3) + #expect(trims.all.isEmpty) + // Nothing stored was deleted: system, task, four calls and results, the answer. + #expect(agent.conversation.allMessages().count == 11) + } + + @Test func aSiftedRequestTooBigForTheWindowFallsBackAndSaysSo() async throws { + let provider = ScriptedProvider(turns: Self.fourSteps()) + // A budget that never sifts: the request outgrows the window instead. + let (agent, trims) = Self.agent(provider, size: 3_000, contextManager: ContextManager(inlineBudgetChars: 100_000)) + _ = try await agent.run("do the task") + #expect(!trims.all.isEmpty) + let last = try #require(provider.captured.last) + #expect(last.messages.filter { $0.role == .assistant }.count < 4) + #expect(last.messages.contains { $0.role == .user && $0.content == "do the task" }) + // A note added for one call only (the progress nudge, turns 3 and 5 + // of 6) survives the fallback as that call's last message: it is not + // stored, so not trimmed. + let noted = provider.captured[4].messages.last + #expect(noted?.role == .user && noted?.content.contains("[Progress check]") == true) + // The fallback is per call: the stored history is complete. + #expect(agent.conversation.allMessages().count == 11) + } + + /// A tool with a long description, so the tool block is a large share of + /// a small window: the messages alone always fit, the request does not. + private struct WideTool: AgentTool { + let name = "make_output" + let description = "Make some output. " + String(repeating: "Detail. ", count: 300) // 2,418 characters + let parameters = ToolParameters(properties: [ + "n": ToolParameterProperty(type: "integer", description: "which"), + ], required: ["n"]) + func execute(parameters: [String: Any]) async throws -> AgentToolResult { + .success(toolCallId: "", toolName: name, result: String(repeating: "x", count: 600)) + } + } + + /// The safety net counts what the call sends besides the messages. The + /// messages here stay far under the bound; with the tool definitions the + /// third and fourth calls pass it, and fall back with room left for them. + @Test func theSafetyNetCountsTheToolDefinitions() async throws { + let provider = ScriptedProvider(turns: (1...3).map { + [LLMToolCall(id: "w\($0)", name: "make_output", arguments: #"{"n":\#($0)}"#)] + } + [[]]) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", maxTurns: 6, + contextWindow: 4_096, tools: [WideTool()], + contextManager: ContextManager(inlineBudgetChars: 100_000), + loopDetection: nil, progressNudgeFractions: [])) + let trims = Trims() + agent.onEvent { event in + if case .historyTrimmed(let removed, _, _) = event { trims.add(removed) } + } + _ = try await agent.run("do the task") + let last = try #require(provider.captured.last) + let messageTokens = agent.conversation.estimateTotalTokens(agent.conversation.allMessages()) + #expect(messageTokens < agent.conversation.fitBudgetTokens) // the messages alone fit + #expect(!trims.all.isEmpty) + #expect(last.messages.filter { $0.role == .assistant }.count < 3) + #expect(last.messages.contains { $0.role == .user && $0.content == "do the task" }) + // Nothing stored was deleted: system, task, three calls and results, the answer. + #expect(agent.conversation.allMessages().count == 9) + } + + @Test func aCapTrimIsReported() async throws { + let provider = ScriptedProvider(turns: [ + [LLMToolCall(id: "m1", name: "make_output", arguments: #"{"n":1}"#)], + [LLMToolCall(id: "m2", name: "make_output", arguments: #"{"n":2}"#)], + [], + ]) + let (agent, trims) = Self.agent(provider, size: 10, contextManager: nil, maxMessages: 4) + _ = try await agent.run("do the task") + #expect(trims.all.contains { $0 > 0 }) + } + + @Test func aConversationCanSkipTheTokenTrim() { + let conversation = Conversation(contextWindow: 3_000, maxMessages: 0) + conversation.setSystemMessage(.system("sys")) + conversation.append(.user("task")) + for i in 0..<6 { conversation.append(.assistant(String(repeating: "\(i)", count: 2_000))) } + #expect(conversation.trim(byTokens: false).removed == 0) + #expect(conversation.allMessages().count == 8) + #expect(conversation.trim().removed > 0) + } + + // MARK: - Fix round 1: the overflow cut (sifted cost, sticky) + + /// Records, per call, whether the safety net moved its cut on that call. + final class OverflowLog: @unchecked Sendable { + private let lock = NSLock() + private var turn = 0 + private var moved: Set = [] + private var reasons: [HistoryTrimReason] = [] + func started(_ t: Int) { lock.withLock { turn = t } } + func trimmed(_ reason: HistoryTrimReason) { + lock.withLock { + reasons.append(reason) + if reason == .overflow { moved.insert(turn) } + } + } + var movedTurns: Set { lock.withLock { moved } } + var allReasons: [HistoryTrimReason] { lock.withLock { reasons } } + } + + /// Eleven 700-character steps under a 4,096 window, no sifting: the + /// request passes the bound part-way through, the net cuts, and the cut + /// then holds for several calls before the next breach moves it. + private static func longRun() async throws -> (Agent, ScriptedProvider, OverflowLog) { + let steps = 11 + let provider = ScriptedProvider(turns: (1...steps).map { + [LLMToolCall(id: "m\($0)", name: "make_output", arguments: #"{"n":\#($0)}"#)] + } + [[]]) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", + maxTurns: steps + 2, contextWindow: 4_096, + tools: [OutputTool(size: 700)], + contextManager: ContextManager(inlineBudgetChars: 100_000), + loopDetection: nil, progressNudgeFractions: [])) + let log = OverflowLog() + agent.onEvent { event in + switch event { + case .llmCallStarted(let turn): log.started(turn) + case .historyTrimmed(_, _, let reason): log.trimmed(reason) + default: break + } + } + _ = try await agent.run("do the task") + return (agent, provider, log) + } + + private static func hasPrefix(_ later: LLMRequest, _ earlier: LLMRequest) -> Bool { + later.messages.count >= earlier.messages.count + && Array(later.messages.prefix(earlier.messages.count)) == earlier.messages + } + + @Test func twoConsecutiveFallbackCallsSendTheSamePrefix() async throws { + let (agent, provider, log) = try await Self.longRun() + let first = try #require(log.movedTurns.min()) + let captured = provider.captured + // The call after the cut does not move it, and sends the cut call's + // request unchanged, with only the new step after it. + #expect(!log.movedTurns.contains(first + 1)) + #expect(Self.hasPrefix(captured[first], captured[first - 1])) + // It is still a fallback call: the oldest step is left out. + #expect(!captured[first].messages.contains { $0.toolCallId == "m1" }) + // The request the cut built fits the bound, and leaves headroom. + let bound = await agent.overflowBoundTokens + #expect(await agent.estimatedRequestTokens(captured[first - 1].messages, tools: captured[first - 1].tools) <= bound) + // Nothing stored was deleted. + #expect(agent.conversation.allMessages().count == 25) // system, task, 11 calls and results, the answer + } + + @Test func theCutMovesOnlyWhenTheBoundIsBreachedAgain() async throws { + let (agent, provider, log) = try await Self.longRun() + let captured = provider.captured + let moved = log.movedTurns + let first = try #require(moved.min()) + // It moves again later in the run, but not on every call. + #expect(moved.count >= 2) + #expect(moved.count < captured.count - first + 1) + let bound = await agent.overflowBoundTokens + for turn in first..= 1 && index <= 11 { + #expect(request.messages.last?.role == .tool) + #expect(request.messages.last?.toolCallId == "m\(index)") + #expect(request.messages.contains { $0.role == .user && $0.content == "do the task" }) + } + } + + /// The pure decision: hold the cut while it fits the bound; on a breach, + /// the fewest more steps that bring the request to the target. + @Test func theCutDecision() async { + let cost: @Sendable (Int) async -> Int = { 1_000 - 100 * $0 } + // Fits at the current cut: held, even with the full history over. + #expect(await Agent.overflowCut(current: 2, units: 8, bound: 800, target: 600, cost: cost) == 2) + // Breached: the fewest steps that reach the target (600 at 4). + #expect(await Agent.overflowCut(current: 0, units: 8, bound: 800, target: 600, cost: cost) == 4) + #expect(await Agent.overflowCut(current: 1, units: 8, bound: 800, target: 600, cost: cost) == 4) + // Nothing reaches it: every evictable step, never more. + #expect(await Agent.overflowCut(current: 0, units: 3, bound: 800, target: 600, cost: cost) == 3) + } + + @Test func theBoundIsConfigurable() { + let manager = ContextManager() + #expect(manager.overflowFraction == 0.8) + manager.overflowFraction = 1.0 + manager.overflowTargetFraction = 0.6 + let child = manager.childManager() + #expect(child.overflowFraction == 1.0) + #expect(child.overflowTargetFraction == 0.6) + } + + @Test func aHigherOverflowFractionLeavesTheRunToCompaction() async throws { + // The same run as aSiftedRequestTooBigForTheWindowFallsBackAndSaysSo, + // with the net's bound raised past what the run reaches. + let manager = ContextManager(inlineBudgetChars: 100_000) + manager.overflowFraction = 3.0 + let provider = ScriptedProvider(turns: Self.fourSteps()) + let (agent, trims) = Self.agent(provider, size: 3_000, contextManager: manager) + _ = try await agent.run("do the task") + #expect(trims.all.isEmpty) + #expect(provider.captured.last?.messages.filter { $0.role == .assistant }.count == 4) + } + + /// An active result too big for the bound on its own: the net leaves out + /// every older step but never the call the model is waiting on, nor its + /// result. + @Test func anOversizedActiveStepIsKeptWhole() async throws { + let provider = ScriptedProvider(turns: Self.fourSteps()) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", maxTurns: 6, + contextWindow: 4_096, tools: [OutputTool(size: 7_000)], + contextManager: ContextManager(inlineBudgetChars: 100_000), + loopDetection: nil, progressNudgeFractions: [])) + _ = try await agent.run("do the task") + for (index, request) in provider.captured.enumerated() where index >= 1 && index <= 4 { + let last = try #require(request.messages.last) + #expect(last.role == .tool && last.toolCallId == "m\(index)") + #expect(request.messages.contains { $0.toolCalls?.contains { $0.id == "m\(index)" } == true }) + #expect(request.messages.contains { $0.role == .user && $0.content == "do the task" }) + // Older steps are all left out. + #expect(request.messages.filter { $0.role == .tool }.count == 1) + } + } + + // MARK: - Fix round 2: the run's task and the current step are pinned + + private static func call(_ id: String) -> AgentMessage { + .assistant(content: "", toolCalls: [AgentToolCall(id: id, name: "t")]) + } + + private static func result(_ id: String) -> AgentMessage { + .tool(results: [.success(toolCallId: id, toolName: "t", result: "out")]) + } + + /// The reviewer's probe: a mid-run user nudge used to become the pinned + /// "task", so the units were [[1], [2, 3]] and unit 0 — the first any + /// cut drops — was the real task. + @Test func aMidRunNudgeDoesNotUnpinTheTask() { + let task = AgentMessage.user("THE TASK") + let stored: [AgentMessage] = [.system("S"), task, Self.call("c1"), Self.result("c1"), + .user("You've called t with the same arguments 3 times"), + Self.call("c2"), Self.result("c2")] + let units = Agent.overflowUnits(stored, task: task.id) + #expect(!units.flatMap { $0 }.contains(1)) + #expect(units == [[2, 3], [4]]) + } + + /// A loop nudge after the tool results is part of the step the model is + /// waiting on: neither that exchange nor the nudge is droppable. + @Test func aTrailingNudgeKeepsTheCurrentStepPinned() { + let task = AgentMessage.user("THE TASK") + let stored: [AgentMessage] = [.system("S"), task, Self.call("c1"), Self.result("c1"), + Self.call("c2"), Self.result("c2"), + .user("You've called t with the same arguments 3 times")] + #expect(Agent.overflowUnits(stored, task: task.id) == [[2, 3]]) + } + + /// End to end: every step repeats the same call, so the loop detector + /// appends a nudge after the results on most calls. The net cuts, and + /// still every request carries the task and the step being answered. + @Test func loopNudgesNeverLetTheNetDropTheTask() async throws { + let steps = 11 + let provider = ScriptedProvider(turns: (1...steps).map { + [LLMToolCall(id: "m\($0)", name: "make_output", arguments: #"{"n":1}"#)] + } + [[]]) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", + maxTurns: steps + 2, contextWindow: 4_096, + tools: [OutputTool(size: 700)], + contextManager: ContextManager(inlineBudgetChars: 100_000), + loopDetection: LoopDetectionConfig(stopThreshold: 100), + progressNudgeFractions: [])) + let log = OverflowLog() + agent.onEvent { event in + if case .historyTrimmed(_, _, let reason) = event { log.trimmed(reason) } + } + _ = try await agent.run("do the task") + #expect(log.allReasons.contains(.overflow)) + let captured = provider.captured + #expect(captured.count == steps + 1) + for (index, request) in captured.enumerated() where index >= 1 { + #expect(request.messages.contains { $0.role == .user && $0.content == "do the task" }) + #expect(request.messages.contains { $0.toolCalls?.contains { $0.id == "m\(index)" } == true }) + #expect(request.messages.contains { $0.role == .tool && $0.toolCallId == "m\(index)" }) + } + // The nudges really were trailing the results. + #expect(captured.contains { $0.messages.last?.role == .user && $0.messages.last?.content.contains("same arguments") == true }) + } + + // MARK: - Fix round 2: trial sifts commit nothing + + /// A trial sift of a shorter view, under a configuration where it evicts + /// (slack ≥ minimum batch, so the proof that trials are inert does not + /// hold), leaves the manager's sticky state as it was. + @Test func aTrialSiftLeavesTheManagerUnchanged() async { + let manager = ContextManager(summaryLength: 40, inlineBudgetChars: 2_000) + manager.evictionSlackFraction = 0.5 + manager.minEvictionBatchFraction = 0.1 + var view: [AgentMessage] = [.system("S"), .user("go")] + for i in 0..<6 { view += ContextSiftBatchTests.step(i, size: 800) } + let before = manager.siftState + let trial = await Agent.trialSift(view, manager: manager) { $0 } + // The trial itself evicted (so a committed sift would have changed state)… + #expect(trial.state != before) + #expect(trial.messages.contains { $0.content.contains(ContextManager.receiptHeader) }) + // …and the manager is as it was. + #expect(manager.siftState == before) + // Committing the trial's state gives what a real sift leaves. + let committed = ContextManager(summaryLength: 40, inlineBudgetChars: 2_000) + committed.evictionSlackFraction = 0.5 + committed.minEvictionBatchFraction = 0.1 + let real = await committed.modelMessages(view) { $0 } + #expect(real.map(\.role) == trial.messages.map(\.role)) // artifact ids differ per store + #expect(committed.siftState == trial.state) + } +} diff --git a/Tests/SwiftAgentKitTests/ProgressNoteTests.swift b/Tests/SwiftAgentKitTests/ProgressNoteTests.swift new file mode 100644 index 0000000..5d27d8e --- /dev/null +++ b/Tests/SwiftAgentKitTests/ProgressNoteTests.swift @@ -0,0 +1,47 @@ +import Testing +import Foundation +import LLMProviderKit +@testable import SwiftAgentKit + +/// The turn-budget note is for one call only and goes at the very end, as a +/// user message. In the system message it changed the prompt prefix (and on +/// Anthropic, without a ContextManager, it replaced the whole system prompt: +/// the last system message wins there). +struct ProgressNoteTests { + private struct EchoTool: AgentTool { + let name = "echo" + let description = "Echo." + let parameters = ToolParameters(properties: [ + "n": ToolParameterProperty(type: "integer", description: "which"), + ], required: ["n"]) + func execute(parameters: [String: Any]) async throws -> AgentToolResult { + .success(toolCallId: "", toolName: name, result: "echoed") + } + } + + @Test(arguments: [false, true]) + func theNoteIsTheLastMessageOfItsCallOnly(withContextManager: Bool) async throws { + let provider = ScriptedProvider(turns: (1...3).map { + [LLMToolCall(id: "e\($0)", name: "echo", arguments: #"{"n":\#($0)}"#)] + } + [[]]) + let agent = Agent(config: AgentConfig(provider: provider, model: "mock", systemPrompt: "BASE", maxTurns: 4, + tools: [EchoTool()], + contextManager: withContextManager ? ContextManager() : nil, + loopDetection: nil, progressNudgeFractions: [0.5])) + _ = try await agent.run("go") + let requests = provider.captured + #expect(requests.count == 4) + // Turn 2 of 4 carries the note, as its very last message, from the user. + let noted = requests[1].messages + #expect(noted.last?.role == .user) + #expect(noted.last?.content.hasPrefix("[Progress check] You have used 2 of 4 turns.") == true) + for request in requests { + let systems = request.messages.filter { $0.role == .system } + #expect(systems.count == 1) + #expect(systems.allSatisfy { !$0.content.contains("[Progress check]") }) + } + for index in [0, 2, 3] { + #expect(!requests[index].messages.contains { $0.content.contains("[Progress check]") }) + } + } +} diff --git a/Tests/SwiftAgentKitTests/StreamedReasoningTests.swift b/Tests/SwiftAgentKitTests/StreamedReasoningTests.swift index f04ce31..1553391 100644 --- a/Tests/SwiftAgentKitTests/StreamedReasoningTests.swift +++ b/Tests/SwiftAgentKitTests/StreamedReasoningTests.swift @@ -59,12 +59,21 @@ private final class EndlessThinker: LLMProvider, @unchecked Sendable { let configuration = LLMProviderConfiguration(name: name, baseURL: URL(string: "inprocess://t")!, defaultModel: "m") private let lock = NSLock() private(set) var calls = 0 - private(set) var lastUserMessage = "" + /// Every `.user`-role message of the most recently sent request, in + /// order. With progress nudges on, a transient progress note can land + /// after the stored "stop reasoning" nudge as the call's last user + /// message, so a test must find the right one among these rather than + /// assume it is the last. + private(set) var lastUserMessages: [String] = [] func complete(_ request: LLMRequest) async throws -> LLMResponse { LLMResponse(text: "done", finishReason: .stop, request: request, providerName: Self.name) } func stream(_ request: LLMRequest) -> AsyncThrowingStream { - let n: Int = lock.withLock { calls += 1; lastUserMessage = request.messages.last { $0.role == .user }?.content ?? ""; return calls } + let n: Int = lock.withLock { + calls += 1 + lastUserMessages = request.messages.filter { $0.role == .user }.map(\.content) + return calls + } return AsyncThrowingStream { continuation in let task = Task { if n == 1 { @@ -87,6 +96,16 @@ private final class EndlessThinker: LLMProvider, @unchecked Sendable { /// Single "Thinking…" pauses of 11, 17 and 23 minutes, some after the work /// was done (xontel review, I-11): a call that only reasons past the limit is /// stopped and the model is told to act. +/// +/// Progress nudges stay ON (default fractions) here on purpose: with +/// `withTools: true` and `maxTurns: 4`, turn 2's stop-reasoning nudge is +/// stored, and turn 2 is also a nudge turn, so the SAME call carries both the +/// stored nudge and the transient progress note as its last two user +/// messages — the collision `AnthropicRoleAlternationTests` exercises at the +/// wire level. Disabling nudges here would hide that overlap instead of +/// exercising it, so the assertion below looks for the stop-reasoning +/// message among all of the call's user messages rather than assuming it is +/// the last one. @Test(arguments: [false, true]) func aCallThatOnlyThinksPastTheLimitIsStoppedAndToldToAct(withTools: Bool) async throws { let p = EndlessThinker() @@ -98,5 +117,6 @@ func aCallThatOnlyThinksPastTheLimitIsStoppedAndToldToAct(withTools: Bool) async #expect(Date().timeIntervalSince(started) < 5) // not the full 10 s #expect(p.calls == 2) #expect(text == "answer") - #expect(p.lastUserMessage.lowercased().contains("reasoning"), "\(p.lastUserMessage)") + #expect(p.lastUserMessages.contains { $0.lowercased().contains("reasoning") }, + "expected the stop-reasoning nudge among the turn's user messages: \(p.lastUserMessages)") } diff --git a/Tests/SwiftAgentKitTests/SubAgentTests.swift b/Tests/SwiftAgentKitTests/SubAgentTests.swift index ca4e43a..116253b 100644 --- a/Tests/SwiftAgentKitTests/SubAgentTests.swift +++ b/Tests/SwiftAgentKitTests/SubAgentTests.swift @@ -741,3 +741,18 @@ private struct ViewImageStub: AgentTool { let child = await SubAgentSpawner(parent: agent).makeChild() #expect(await child.callbacks?.beforeTool != nil) } + +/// A sub-agent's sim_ui/mac_ui reads are protected like the parent's: the +/// child manager used to drop readIdentityParams (and fall back to `path`, +/// which those tools don't have). +@Test func testChildContextManagerKeepsTheParentsReadProtection() async throws { + let parentCM = ContextManager(readToolNames: ["read_file", "sim_ui"], readIdentityParams: ["sim_ui": "bundle_id"]) + parentCM.minEvictionBatchFraction = 0.3 + let agent = Agent(config: AgentConfig(provider: PlainAnswerProvider(text: "x"), contextManager: parentCM)) + let child = await SubAgentSpawner(parent: agent).makeChild() + let childCM = try #require(child.config.contextManager) + #expect(childCM !== parentCM) + #expect(childCM.readToolNames == ["read_file", "sim_ui"]) + #expect(childCM.readIdentityParams == ["sim_ui": "bundle_id"]) + #expect(childCM.minEvictionBatchFraction == 0.3) +} diff --git a/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift b/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift index 867eba8..2df1a21 100644 --- a/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift +++ b/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift @@ -1764,12 +1764,13 @@ private func firstArtifactID(in text: String) -> String? { // The raw tool output must NOT be resent as an inline tool message (a short // conclusion snippet in the ledger is fine — and desired). #expect(!out.contains { $0.role == .tool && $0.content.contains("DEEP_NEEDLE_END") }) - // …and a ledger + artifact reference must appear in the (single) system block. - let system = out.first { $0.role == .system } - #expect(system != nil) - #expect(system?.content.contains("tool ledger") == true) - #expect(system?.content.contains("read_file") == true) - let artifactID = system.flatMap { firstArtifactID(in: $0.content) } + // …and a receipt with the artifact reference stands in the step's own + // place; the system message is the system prompt alone. + #expect(out.map(\.role) == [.system, .user, .assistant, .assistant, .user]) + #expect(out.first { $0.role == .system }?.content == "You are helpful.") + let receipt = out.first { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) } + #expect(receipt?.content.contains("read_file") == true) + let artifactID = receipt.flatMap { firstArtifactID(in: $0.content) } #expect(artifactID != nil) // Main messages survive. @@ -1784,10 +1785,10 @@ private func firstArtifactID(in text: String) -> String? { } } -@Test func testLedgerNotesOmittedReceiptsBeyondCap() async { - // More externalized exchanges than ledger slots → the ledger must say how - // many older calls fell off and point at artifact_list (Listable store). - let manager = ContextManager(ledgerEntries: 3, summaryLength: 40, inlineBudgetChars: 0) +@Test func testEveryEvictedStepKeepsItsOwnReceipt() async { + // No cap any more: each evicted step keeps its receipt in its own place, + // so nothing falls off and no receipt ever moves. + let manager = ContextManager(summaryLength: 40, inlineBudgetChars: 0) var messages: [AgentMessage] = [.system("sys"), .user("do five things")] for i in 1...5 { messages.append(.assistant(content: "", toolCalls: [AgentToolCall(id: "c\(i)", name: "run_shell")])) @@ -1798,15 +1799,17 @@ private func firstArtifactID(in text: String) -> String? { messages.append(.user("next")) let out = await manager.modelMessages(messages) { $0 } - let system = out.first { $0.role == .system }?.content ?? "" - // 5 receipts, 3 shown → 2 omitted, with the discovery hint. - #expect(system.contains("(+2 older tool calls not shown — use artifact_list")) - // Ledger shows only the newest 3. - #expect(!system.contains("output 1")) - #expect(system.contains("output 5")) + #expect(out.first { $0.role == .system }?.content == "sys") + let receipts = out.filter { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) } + #expect(receipts.count == 5) + for (i, receipt) in receipts.enumerated() { + #expect(receipt.content.contains("output \(i + 1) ")) + #expect(receipt.content.hasSuffix("\n]")) + } + #expect(out.map(\.content).filter { $0.hasPrefix("done") } == ["done 1", "done 2", "done 3", "done 4", "done 5"]) } -@Test func testLedgerHasNoOmittedLineWhenAllReceiptsFit() async { +@Test func testReceiptsNeverJoinTheSystemMessage() async { let manager = ContextManager(summaryLength: 40, inlineBudgetChars: 0) let messages: [AgentMessage] = [ .system("sys"), @@ -1818,9 +1821,9 @@ private func firstArtifactID(in text: String) -> String? { .user("next"), ] let out = await manager.modelMessages(messages) { $0 } - let system = out.first { $0.role == .system }?.content ?? "" - #expect(system.contains("tool ledger")) - #expect(!system.contains("not shown")) + #expect(out.filter { $0.role == .system }.map(\.content) == ["sys"]) + #expect(out.contains { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }) + #expect(!out.contains { $0.content.contains("tool ledger") }) } @Test func testContextManagerDoesNotRetruncateArtifactReads() async { @@ -1866,7 +1869,7 @@ private func firstArtifactID(in text: String) -> String? { // …the tool-call turn is preserved… #expect(out.contains { $0.role == .assistant && ($0.toolCalls?.isEmpty == false) }) // …and no "tool ledger" externalization text appears. - #expect(out.allSatisfy { !$0.content.contains("tool ledger") }) + #expect(out.allSatisfy { !$0.content.contains(ContextManager.receiptHeader) }) } @Test func testReceiptSummaryCapturesConclusionAtTail() async { @@ -1886,11 +1889,11 @@ private func firstArtifactID(in text: String) -> String? { ] let out = await manager.modelMessages(messages) { $0 } - let system = out.first { $0.role == .system }?.content ?? "" + let receipt = out.first { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }?.content ?? "" - #expect(system.contains("BULLET_CONCLUSION")) // the tail (real error) survived - #expect(system.contains("Traceback")) // the head too - #expect(system.contains("ERROR")) // marked as a failure + #expect(receipt.contains("BULLET_CONCLUSION")) // the tail (real error) survived + #expect(receipt.contains("Traceback")) // the head too + #expect(receipt.contains("ERROR")) // marked as a failure } @Test func testReceiptIncludesCallArgHint() async { @@ -1909,10 +1912,10 @@ private func firstArtifactID(in text: String) -> String? { ] let out = await manager.modelMessages(messages) { $0 } - let system = out.first { $0.role == .system }?.content ?? "" + let receipt = out.first { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }?.content ?? "" - #expect(system.contains("run_shell(python3 naseem_pdf_build.py)")) // command named - #expect(system.contains("ERROR")) + #expect(receipt.contains("run_shell(python3 naseem_pdf_build.py)")) // command named + #expect(receipt.contains("ERROR")) } @Test func testContextManagerEvictsOldestKeepsRecent() async { @@ -1937,7 +1940,7 @@ private func firstArtifactID(in text: String) -> String? { // The old exchange is externalized — not kept as an inline tool message // (a conclusion snippet in the ledger is fine). #expect(!out.contains { $0.role == .tool && $0.content.contains("OLDTAIL") }) - #expect(out.contains { $0.role == .system && $0.content.contains("ledger") }) + #expect(out.contains { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }) } @Test func testContextManagerKeepsActiveExchange() async { @@ -1983,7 +1986,7 @@ private func readCall(_ id: String, path: String) -> AgentToolCall { #expect(out.contains { $0.role == .tool && $0.content.contains("READ_KEPT") }) // read kept inline #expect(!out.contains { $0.role == .tool && $0.content.contains("SHELL_MARKER") }) // shell externalized - #expect(out.contains { $0.role == .system && $0.content.contains("ledger") }) + #expect(out.contains { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }) } @Test func testProtectsStructuredReadKeyedByNonPathParam() async { @@ -2092,7 +2095,7 @@ private func readCall(_ id: String, path: String) -> AgentToolCall { let out = await manager.modelMessages(messages) { $0 } #expect(!out.contains { $0.role == .tool && $0.content.contains("BIGREAD_") }) // size guard → externalized - #expect(out.contains { $0.role == .system && $0.content.contains("ledger") }) + #expect(out.contains { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) }) } @Test func testKeepLatestReadsDisabledRestoresOldBehavior() async { @@ -2800,10 +2803,9 @@ private func completedShellExchange(id i: Int, size: Int) -> [AgentMessage] { let out = await manager.modelMessages(messages) { $0 } - // 6 exchanges ≈ 2400 chars; target 500 → at least 5 evicted (ledger'd). - let system = out.first { $0.role == .system }?.content ?? "" - let ledgerLines = system.split(separator: "\n").filter { $0.hasPrefix("- ") }.count - #expect(ledgerLines >= 5) + // 6 exchanges ≈ 2400 chars; target 500 → at least 5 evicted, each a receipt in place. + let receipts = out.filter { $0.role == .assistant && $0.content.contains(ContextManager.receiptHeader) } + #expect(receipts.count >= 5) // Inline tool results that remain: at most one exchange. #expect(out.filter { $0.role == .tool }.count <= 1) }