diff --git a/.changeset/ltr-use-char-cap.md b/.changeset/ltr-use-char-cap.md new file mode 100644 index 000000000..3ab656837 --- /dev/null +++ b/.changeset/ltr-use-char-cap.md @@ -0,0 +1,5 @@ +--- +'@truefoundry/trueforge-core': patch +--- + +Large tool response guidance now includes content size in the blocked preamble for MCP and sandbox dumps, shares the same "Content too big (size; reason)" framing, uses RFC 2119 wording (MUST/MAY/SHOULD), and states the per-response character limit for dumped-file reads and oversized sandbox stdout. Elevated tools (deferred-tools `get_tool_info` only) skip offload up to min(50000 characters, 80% of remaining context). Individual offload uses a strict `>` threshold so the stated character cap is usable. Size figures are plain digits without thousands separators. diff --git a/packages/trueforge-core/src/agent-session/builtinsFromSpec.ts b/packages/trueforge-core/src/agent-session/builtinsFromSpec.ts index cb0139088..06f278010 100644 --- a/packages/trueforge-core/src/agent-session/builtinsFromSpec.ts +++ b/packages/trueforge-core/src/agent-session/builtinsFromSpec.ts @@ -57,6 +57,7 @@ export function builtinsFromSpec(input: { capabilities.push( largeToolResponse({ dynamicSubAgentsPresent: !isChild && dynamicSubAgentsEnabled, + modelContextLength: definition.modelProperties?.contextLength, logger, }), ); diff --git a/packages/trueforge-core/src/core/capabilities/builtins/DynamicSubAgents.ts b/packages/trueforge-core/src/core/capabilities/builtins/DynamicSubAgents.ts index d8c3c0589..7212d27d4 100644 --- a/packages/trueforge-core/src/core/capabilities/builtins/DynamicSubAgents.ts +++ b/packages/trueforge-core/src/core/capabilities/builtins/DynamicSubAgents.ts @@ -24,7 +24,7 @@ export type ModelSetConfig = Record; export function createDynamicSubAgentLargeToolResponseGuidance(): string { return dedent` For large unstructured output like web search, documentation lookup, or broad search results, - the Agent should prefer delegating to a sub-agent and ask it to return only the relevant summary, with citations + it SHOULD prefer delegating to a sub-agent and MUST ask it to return only the relevant summary, with citations when they improve verification.`; } diff --git a/packages/trueforge-core/src/core/capabilities/builtins/LargeToolResponse.ts b/packages/trueforge-core/src/core/capabilities/builtins/LargeToolResponse.ts index 9ef631cda..7281454ab 100644 --- a/packages/trueforge-core/src/core/capabilities/builtins/LargeToolResponse.ts +++ b/packages/trueforge-core/src/core/capabilities/builtins/LargeToolResponse.ts @@ -3,17 +3,23 @@ import type { Logger } from 'winston'; import { describeContentSize, estimateTokensForString } from '../../llm/usage'; import type { ToolCallResult } from '../../mcp/executeToolCalls'; import { getOperationMetrics } from '../../observability/operationMetrics'; +import { DEFERRED_TOOLS_SERVER_ID, GET_TOOL_INFO_NAME } from '../../runtime/DeferredTool'; +import type { CurrentContextUsage } from '../../runtime/contextUsage'; import type { HarnessSandbox, SandboxInfo } from '../../sandbox/Sandbox'; import { createSandboxLargeToolResponseGuidance, SANDBOX_SCHEMA_INFER_TAG } from '../../sandbox/Sandbox'; import { extractErrorLogFields } from '../../util/errorLogFields'; import type { AgentCapability } from '../AgentCapability'; import type { AgentThreadExecutionContext } from '../AgentContextProcessor'; import type { ToolResponseProcessor, ToolResponseProcessorResult } from '../ToolResponseProcessor'; +import { DEFAULT_CONTEXT_COMPACTION_RATIO } from './ContextCompaction'; import { createDynamicSubAgentLargeToolResponseGuidance } from './DynamicSubAgents'; export const DEFAULT_INDIVIDUAL_TOOL_TOKEN_THRESHOLD = 6000; export const DEFAULT_TOTAL_TOOL_TOKEN_THRESHOLD = 10000; export const DEFAULT_PREVIEW_NUMBER_OF_CHARACTERS = 100; +/** Upper bound for elevated tools; effective limit is min(this, 80% of remaining context characters). */ +const ELEVATED_TOOL_CHARACTER_LIMIT_CAP = 50_000; +const ELEVATED_TOOLS: ReadonlySet = new Set([GET_TOOL_INFO_NAME]); const FAILURE_MESSAGE_TRUNCATION_LENGTH = 500; type ResultCategory = 'failure' | 'sandbox' | 'mcp'; @@ -45,11 +51,40 @@ function createContentPreview(content: string, previewChars: number): string { return `${content.slice(0, previewChars)}\n...\n${content.slice(-previewChars)}`; } +function remainingContextCharacters(params: { + modelContextLength: number | undefined; + currentContextUsage: CurrentContextUsage; +}): number | undefined { + if (params.modelContextLength === undefined) { + return undefined; + } + const used = params.currentContextUsage.prompt_tokens + params.currentContextUsage.completion_tokens; + return Math.max(0, (params.modelContextLength - used) * 4); +} + +function elevatedCharacterLimit(remainingChars: number | undefined): number { + if (remainingChars === undefined) { + return ELEVATED_TOOL_CHARACTER_LIMIT_CAP; + } + // Match compaction's 80% trigger so elevated tools leave headroom for summarization. + return Math.min(ELEVATED_TOOL_CHARACTER_LIMIT_CAP, Math.floor(remainingChars * DEFAULT_CONTEXT_COMPACTION_RATIO)); +} + +function isWithinElevatedCharacterLimit(entry: ToolResultWithInfo, limit: number): boolean { + const info = entry.toolCallResult.info; + // Name alone is not enough: a preloaded remote MCP tool can also be called get_tool_info. + if (info?.toolSet.name !== DEFERRED_TOOLS_SERVER_ID) { + return false; + } + return ELEVATED_TOOLS.has(info.originalToolName) && entry.originalContent.length <= limit; +} + export class LargeToolResponseProcessor implements ToolResponseProcessor { private dynamicSubAgentsPresent: boolean; private individualTokenThreshold: number; private totalTokenThreshold: number; private previewNumberOfCharacters: number; + private modelContextLength: number | undefined; private readonly logger: Logger; constructor(param: { @@ -57,6 +92,7 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { individualTokenThreshold: number; totalTokenThreshold: number; previewNumberOfCharacters: number; + modelContextLength: number | undefined; logger: Logger; }) { if (param.individualTokenThreshold > param.totalTokenThreshold) { @@ -68,21 +104,34 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { this.individualTokenThreshold = param.individualTokenThreshold; this.totalTokenThreshold = param.totalTokenThreshold; this.previewNumberOfCharacters = param.previewNumberOfCharacters; + this.modelContextLength = param.modelContextLength; this.logger = param.logger.child({ module: 'LargeToolResponse' }); } // Thresholds are configured in tokens, estimated at 4 characters per token; state them in characters for the model. - // An entry at/above the individual threshold was blocked for its own size; below it, only the combined limit can block it. + private individualResponseCharacterLimit(): string { + return String(this.individualTokenThreshold * 4); + } + + // Strict `>` so a response at the stated character cap (threshold * 4) is allowed, matching guidance. private blockReasonDescription(entry: ToolResultWithInfo): string { - const perResponse = (this.individualTokenThreshold * 4).toLocaleString('en-US'); - const combined = (this.totalTokenThreshold * 4).toLocaleString('en-US'); - return entry.originalTokens >= this.individualTokenThreshold + const perResponse = this.individualResponseCharacterLimit(); + const combined = String(this.totalTokenThreshold * 4); + return entry.originalTokens > this.individualTokenThreshold ? `this exceeds the ${perResponse}-character tool call response limit` : `the parallel tool call responses together exceed the ${combined}-character combined tool call response limit`; } + private formatBlockedContentPreamble(entry: ToolResultWithInfo): string { + return `Content too big (${describeContentSize(entry.originalContent)}; ${this.blockReasonDescription(entry)})`; + } + + private formatPreview(preview: string): string { + return `Preview (first and last ${String(this.previewNumberOfCharacters)} chars):\n${preview}`; + } + private createLargeToolResponseGuidance(sandbox: HarnessSandbox | undefined, entry: ToolResultWithInfo): string { - const steps = ['Check if some parameter can be passed in order to reduce the output size.']; + const steps = ['It MUST check whether a parameter can reduce the output size.']; if (sandbox) { steps.push(createSandboxLargeToolResponseGuidance()); @@ -92,39 +141,54 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { steps.push(createDynamicSubAgentLargeToolResponseGuidance()); } - return `Content too big (${this.blockReasonDescription(entry)}). To handle a large tool call the Agent can do the following:\n${steps + return `${this.formatBlockedContentPreamble(entry)}. To handle a large tool call it MUST do the following:\n${steps .map((step, index) => `${String(index + 1)}. ${step}`) .join('\n')}`; } - private createFileDumpMessage(params: { filePath: string; preview: string; size: string; reason: string }): string { - const { filePath, preview, size, reason } = params; + private createFileDumpMessage(params: { filePath: string; preview: string; entry: ToolResultWithInfo }): string { + const { filePath, preview, entry } = params; + const charCap = this.individualResponseCharacterLimit(); const steps: string[] = [ - dedent`Use sandbox to read parts of the file or extract data. - Use ${SANDBOX_SCHEMA_INFER_TAG} to understand the schema. - The file's total size is given above. Plan reads against it, for example extract fields with a script or read sized ranges, instead of probing with small incremental reads.`, + dedent`It MUST use the sandbox to read or extract from the file. + It MUST use ${SANDBOX_SCHEMA_INFER_TAG} to understand the schema. + Each sandbox tool response MUST stay under the ${charCap}-character tool call response limit. + It MUST prefer one read or extract that returns only what it needs; it MAY use up to ${charCap} character limit when useful.`, ]; if (this.dynamicSubAgentsPresent) { steps.push( - `${createDynamicSubAgentLargeToolResponseGuidance()} The Agent must pass file path to sub-agent also.`, + `${createDynamicSubAgentLargeToolResponseGuidance()} It MUST pass the file path to the sub-agent also.`, ); } - const guidance = `The Agent can do the following to handle it:\n${steps + const guidance = `To handle a large tool call it MUST do the following:\n${steps .map((step, index) => `${String(index + 1)}. ${step}`) .join('\n')}`; - return `Content too large (${size}; ${reason}). Result saved to: ${filePath}.\n\n${guidance}\n\nPreview (first and last ${String(this.previewNumberOfCharacters)} chars):\n${preview}`; + return `${this.formatBlockedContentPreamble(entry)}. Result saved to: ${filePath}.\n\n${guidance}\n\n${this.formatPreview(preview)}`; } - private buildResultsWithInfo(results: ToolCallResult[], sandbox: HarnessSandbox | undefined): ToolResultWithInfo[] { - return results.map(toolCallResult => ({ - toolCallResult, - originalContent: toolCallResult.message.content, - originalTokens: estimateTokensForString(toolCallResult.message.content), - category: categorize(toolCallResult, sandbox), - })); + private buildResultsWithInfo(params: { + results: ToolCallResult[]; + sandbox: HarnessSandbox | undefined; + elevatedLimit: number; + }): ToolResultWithInfo[] { + const { results, sandbox, elevatedLimit } = params; + const out: ToolResultWithInfo[] = []; + for (const toolCallResult of results) { + const entry: ToolResultWithInfo = { + toolCallResult, + originalContent: toolCallResult.message.content, + originalTokens: estimateTokensForString(toolCallResult.message.content), + category: categorize(toolCallResult, sandbox), + }; + if (isWithinElevatedCharacterLimit(entry, elevatedLimit)) { + continue; + } + out.push(entry); + } + return out; } private blockResult( @@ -142,10 +206,10 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { newContent = entry.originalContent.slice(0, FAILURE_MESSAGE_TRUNCATION_LENGTH); break; case 'sandbox': - newContent = `Content too big (${this.blockReasonDescription(entry)}). The Agent should write to a file if required and use grep and sed.\n\nPreview:\n${preview}`; + newContent = `${this.formatBlockedContentPreamble(entry)}. It MUST write the full output to a file, then return only what it needs. Each tool call response MUST stay under the ${this.individualResponseCharacterLimit()} character tool call response limit. It MAY use grep and sed on the file as needed.\n\n${this.formatPreview(preview)}`; break; case 'mcp': - newContent = `${this.createLargeToolResponseGuidance(sandbox, entry)}\n\nPreview (first and last ${String(this.previewNumberOfCharacters)} chars):\n${preview}`; + newContent = `${this.createLargeToolResponseGuidance(sandbox, entry)}\n\n${this.formatPreview(preview)}`; if (sandbox) { pendingSandboxDumps.push(entry); } @@ -178,8 +242,7 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { entry.toolCallResult.message.content = this.createFileDumpMessage({ filePath: stored.filePath, preview, - size: describeContentSize(entry.originalContent), - reason: this.blockReasonDescription(entry), + entry, }); entry.toolCallResult.isStructuredContent = false; return stored.sandboxCreated; @@ -207,10 +270,16 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { execution: Readonly, ): Promise { const sandbox = execution.sandbox; - const resultsWithInfo = this.buildResultsWithInfo(results, sandbox); + const elevatedLimit = elevatedCharacterLimit( + remainingContextCharacters({ + modelContextLength: this.modelContextLength, + currentContextUsage: execution.currentContextUsage, + }), + ); + const resultsWithInfo = this.buildResultsWithInfo({ results, sandbox, elevatedLimit }); let runningTotal = resultsWithInfo.reduce((sum, e) => sum + e.originalTokens, 0); - if (runningTotal < this.individualTokenThreshold) { + if (runningTotal <= this.individualTokenThreshold) { return {}; } @@ -218,7 +287,7 @@ export class LargeToolResponseProcessor implements ToolResponseProcessor { const unblocked: ToolResultWithInfo[] = []; for (const entry of resultsWithInfo) { - if (entry.originalTokens >= this.individualTokenThreshold) { + if (entry.originalTokens > this.individualTokenThreshold) { runningTotal -= this.blockResult(entry, pendingSandboxDumps, sandbox, 'large_tool_response_individual'); } else { unblocked.push(entry); @@ -246,6 +315,7 @@ export interface LargeToolResponseSettings { export function largeToolResponse(options: { dynamicSubAgentsPresent: boolean; + modelContextLength?: number | undefined; settings?: LargeToolResponseSettings | undefined; logger: Logger; }): AgentCapability { @@ -256,6 +326,7 @@ export function largeToolResponse(options: { individualTokenThreshold: options.settings?.individualTokenThreshold ?? DEFAULT_INDIVIDUAL_TOOL_TOKEN_THRESHOLD, totalTokenThreshold: options.settings?.totalTokenThreshold ?? DEFAULT_TOTAL_TOOL_TOKEN_THRESHOLD, previewNumberOfCharacters: options.settings?.previewNumberOfCharacters ?? DEFAULT_PREVIEW_NUMBER_OF_CHARACTERS, + modelContextLength: options.modelContextLength, logger: options.logger, }), ], diff --git a/packages/trueforge-core/src/core/llm/usage.ts b/packages/trueforge-core/src/core/llm/usage.ts index 86e6072c0..3806f7712 100644 --- a/packages/trueforge-core/src/core/llm/usage.ts +++ b/packages/trueforge-core/src/core/llm/usage.ts @@ -7,7 +7,7 @@ export function estimateTokensForString(s: string): number { /** Size label for prompts replacing full content, so the model can size reads without guessing. */ export function describeContentSize(content: string): string { - return `${content.length.toLocaleString('en-US')} characters`; + return `${String(content.length)} characters`; } function sumOptional(a: number | undefined, b: number | undefined): number | undefined { diff --git a/packages/trueforge-core/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts b/packages/trueforge-core/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts index c670d3d32..710024881 100644 --- a/packages/trueforge-core/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts +++ b/packages/trueforge-core/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts @@ -48,8 +48,10 @@ describe('LargeToolResponse sandbox object identity (ISSUE-035)', () => { context: [], }); - expect(sandboxResult.message.content).toContain('write to a file'); - expect(mcpResult.message.content).not.toContain('write to a file'); - expect(mcpResult.message.content).toContain('Content too big'); + expect(sandboxResult.message.content).toContain('It MUST write the full output to a file'); + expect(sandboxResult.message.content).toContain('character tool call response limit'); + expect(sandboxResult.message.content).toMatch(/Content too big \([^;]+;/); + expect(mcpResult.message.content).not.toContain('It MUST write the full output to a file'); + expect(mcpResult.message.content).toMatch(/Content too big \([^;]+;/); }); }); diff --git a/packages/trueforge-core/tests/core/usage.test.ts b/packages/trueforge-core/tests/core/usage.test.ts index f1d542219..6a8c538c1 100644 --- a/packages/trueforge-core/tests/core/usage.test.ts +++ b/packages/trueforge-core/tests/core/usage.test.ts @@ -35,7 +35,7 @@ describe('CompletionUsage', () => { describe('describeContentSize', () => { it('formats characters with thousands separators', () => { - expect(describeContentSize('x'.repeat(50_000))).toBe('50,000 characters'); + expect(describeContentSize('x'.repeat(50_000))).toBe('50000 characters'); expect(describeContentSize('')).toBe('0 characters'); }); });