From 1979e227919baea52aa794f42d5a7058ffd36fcc Mon Sep 17 00:00:00 2001 From: Chirag Jain Date: Mon, 19 Oct 2026 08:33:22 +0530 Subject: [PATCH 1/5] feat: fine-grained tool streaming via TFY beta header TrueFoundry uses anthropic_beta instead of mutating tool defs; native Anthropic keeps call-level toolStreaming. Co-authored-by: Cursor --- .../tfy-fine-grained-tool-streaming-header.md | 5 +++++ .../src/core/llm/VercelAILLM.ts | 14 +++++++++++- .../tests/core/llm/VercelAILLM.model.test.ts | 22 +++++++++++++++++++ .../tests/core/llm/VercelAILLM.output.test.ts | 12 +++++++--- 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 .changeset/tfy-fine-grained-tool-streaming-header.md diff --git a/.changeset/tfy-fine-grained-tool-streaming-header.md b/.changeset/tfy-fine-grained-tool-streaming-header.md new file mode 100644 index 000000000..bbfce5f54 --- /dev/null +++ b/.changeset/tfy-fine-grained-tool-streaming-header.md @@ -0,0 +1,5 @@ +--- +'@truefoundry/trueforge-core': patch +--- + +Enable fine-grained tool streaming via TrueFoundry `anthropic_beta` header and Anthropic `toolStreaming` diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts index ec126ba88..6bf434ce8 100644 --- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts +++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts @@ -127,11 +127,19 @@ function isFunctionToolCall(toolCall: T): toolCall i * own packages: those drop `json_schema`, and Fireworks also clamps efforts its models do accept. * TODO: move Z AI to @ai-sdk/zai once https://github.com/vercel/ai/pull/17340 ships. */ +/** Legacy Anthropic beta that enables fine-grained tool input streaming when tool defs omit the field. */ +const TRUEFOUNDRY_FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14'; + function compatibleModel(config: VercelAIProviderConfig): LanguageModel { const { provider, model, apiKey, headers, baseUrl } = config; if (baseUrl === undefined) { throw new Error(`Provider "${provider.type}" requires a baseUrl`); } + // TFY gateway → Anthropic: prefer the beta header over mutating tool defs (safer rollout). + const outboundHeaders = + provider.type === 'truefoundry' + ? { ...headers, anthropic_beta: TRUEFOUNDRY_FINE_GRAINED_TOOL_STREAMING_BETA } + : headers; const client = createOpenAICompatible({ name: provider.type, baseURL: baseUrl, @@ -141,7 +149,7 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel { supportsStructuredOutputs: true, // These endpoints omit token counts from streamed responses unless asked. includeUsage: true, - ...(Object.keys(headers).length > 0 ? { headers } : {}), + ...(Object.keys(outboundHeaders).length > 0 ? { headers: outboundHeaders } : {}), }); return client(model.id); } @@ -390,6 +398,9 @@ function openaiProviderOptions({ * * Prompt caching defaults on: Anthropic bills a write once, then cheaper reads on stable prefixes * (system / tools / early messages). Callers can override via `cache_control` in the request body. + * + * `toolStreaming` is pinned on so every function tool gets `eager_input_streaming` on the Anthropic + * wire (call-level default). */ function anthropicProviderOptions(rawBody: unknown): JSONObject | undefined { const cacheControl = readBodyField({ rawBody, key: 'cache_control' }) ?? { type: 'ephemeral' }; @@ -398,6 +409,7 @@ function anthropicProviderOptions(rawBody: unknown): JSONObject | undefined { const effort = readBodyField({ rawBody, key: 'effort' }); const opts: JSONObject = { cacheControl, + toolStreaming: true, ...(disableParallelToolUse !== undefined ? { disableParallelToolUse } : {}), ...(thinking !== undefined ? { thinking } : {}), ...(effort !== undefined ? { effort } : {}), diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts index 5b7766a0e..378de665c 100644 --- a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts +++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts @@ -75,6 +75,28 @@ describe('buildLanguageModel', () => { if (typeof model !== 'object' || model === null) throw new Error('Expected model to be an object'); expect(Reflect.get(model, 'modelId')).toBe('claude-sonnet-5'); }); + + it('sets anthropic_beta fine-grained tool streaming header for truefoundry only', () => { + function resolveHeaders(model: unknown): unknown { + if (typeof model !== 'object' || model === null) { + throw new Error('Expected language model object'); + } + const config = Reflect.get(model, 'config'); + if (typeof config !== 'object' || config === null) { + throw new Error('Expected model config'); + } + const headers = Reflect.get(config, 'headers'); + return typeof headers === 'function' ? headers() : headers; + } + expect( + resolveHeaders(buildLanguageModel(makeConfig({ provider: 'truefoundry', baseUrl: 'http://localhost/v1' }))), + ).toMatchObject({ + anthropic_beta: 'fine-grained-tool-streaming-2025-05-14', + }); + expect( + resolveHeaders(buildLanguageModel(makeConfig({ provider: 'fireworks', baseUrl: 'http://localhost/v1' }))), + ).not.toHaveProperty('anthropic_beta'); + }); }); // ─────────── resolveMaxOutputTokens ─────────── diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.output.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.output.test.ts index 6c9a784f3..955d2aca1 100644 --- a/packages/trueforge-core/tests/core/llm/VercelAILLM.output.test.ts +++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.output.test.ts @@ -249,7 +249,9 @@ describe('buildProviderOptions', () => { structuredOutputSpec: textSpec, rawBody: {}, }); - expect(opts).toEqual({ anthropic: { cacheControl: { type: 'ephemeral' } } }); + expect(opts).toEqual({ + anthropic: { cacheControl: { type: 'ephemeral' }, toolStreaming: true }, + }); } }); @@ -263,6 +265,7 @@ describe('buildProviderOptions', () => { }); expect(opts['anthropic']).toEqual({ cacheControl: { type: 'ephemeral' }, + toolStreaming: true, thinking: { type: 'adaptive', display: 'summarized' }, effort: 'max', }); @@ -275,7 +278,9 @@ describe('buildProviderOptions', () => { structuredOutputSpec: schemaSpecStrict, rawBody: {}, }); - expect(opts).toEqual({ anthropic: { cacheControl: { type: 'ephemeral' } } }); + expect(opts).toEqual({ + anthropic: { cacheControl: { type: 'ephemeral' }, toolStreaming: true }, + }); }); it('defaults cacheControl to ephemeral when cache_control is omitted', () => { @@ -285,7 +290,7 @@ describe('buildProviderOptions', () => { structuredOutputSpec: textSpec, rawBody: {}, }); - expect(opts['anthropic']).toEqual({ cacheControl: { type: 'ephemeral' } }); + expect(opts['anthropic']).toEqual({ cacheControl: { type: 'ephemeral' }, toolStreaming: true }); }); it('forwards cache_control and disable_parallel_tool_use from rawBody', () => { @@ -298,6 +303,7 @@ describe('buildProviderOptions', () => { expect(opts['anthropic']).toMatchObject({ cacheControl: { type: 'ephemeral', ttl: '1h' }, disableParallelToolUse: true, + toolStreaming: true, }); }); }); From 4ee2e7d0681c92c8fa9cf375c9b6581107973b77 Mon Sep 17 00:00:00 2001 From: Chirag Jain Date: Mon, 19 Oct 2026 09:17:03 +0530 Subject: [PATCH 2/5] refactor: inline TFY fine-grained tool streaming beta string Co-authored-by: Cursor --- packages/trueforge-core/src/core/llm/VercelAILLM.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts index 6bf434ce8..3a07d7eaf 100644 --- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts +++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts @@ -127,9 +127,6 @@ function isFunctionToolCall(toolCall: T): toolCall i * own packages: those drop `json_schema`, and Fireworks also clamps efforts its models do accept. * TODO: move Z AI to @ai-sdk/zai once https://github.com/vercel/ai/pull/17340 ships. */ -/** Legacy Anthropic beta that enables fine-grained tool input streaming when tool defs omit the field. */ -const TRUEFOUNDRY_FINE_GRAINED_TOOL_STREAMING_BETA = 'fine-grained-tool-streaming-2025-05-14'; - function compatibleModel(config: VercelAIProviderConfig): LanguageModel { const { provider, model, apiKey, headers, baseUrl } = config; if (baseUrl === undefined) { @@ -138,7 +135,7 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel { // TFY gateway → Anthropic: prefer the beta header over mutating tool defs (safer rollout). const outboundHeaders = provider.type === 'truefoundry' - ? { ...headers, anthropic_beta: TRUEFOUNDRY_FINE_GRAINED_TOOL_STREAMING_BETA } + ? { ...headers, anthropic_beta: 'fine-grained-tool-streaming-2025-05-14' } : headers; const client = createOpenAICompatible({ name: provider.type, From 321d5df3c924e3ac10f1c4c719b3816afded4cd0 Mon Sep 17 00:00:00 2001 From: Chirag Jain Date: Mon, 19 Oct 2026 09:28:46 +0530 Subject: [PATCH 3/5] fix: use anthropic-beta header name for TFY tool streaming Match Anthropic's wire header spelling instead of anthropic_beta. Co-authored-by: Cursor --- .changeset/tfy-fine-grained-tool-streaming-header.md | 2 +- packages/trueforge-core/src/core/llm/VercelAILLM.ts | 2 +- .../trueforge-core/tests/core/llm/VercelAILLM.model.test.ts | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.changeset/tfy-fine-grained-tool-streaming-header.md b/.changeset/tfy-fine-grained-tool-streaming-header.md index bbfce5f54..6d8560e88 100644 --- a/.changeset/tfy-fine-grained-tool-streaming-header.md +++ b/.changeset/tfy-fine-grained-tool-streaming-header.md @@ -2,4 +2,4 @@ '@truefoundry/trueforge-core': patch --- -Enable fine-grained tool streaming via TrueFoundry `anthropic_beta` header and Anthropic `toolStreaming` +Enable fine-grained tool streaming via TrueFoundry `anthropic-beta` header and Anthropic `toolStreaming` diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts index 3a07d7eaf..791b97bf8 100644 --- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts +++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts @@ -135,7 +135,7 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel { // TFY gateway → Anthropic: prefer the beta header over mutating tool defs (safer rollout). const outboundHeaders = provider.type === 'truefoundry' - ? { ...headers, anthropic_beta: 'fine-grained-tool-streaming-2025-05-14' } + ? { ...headers, 'anthropic-beta': 'fine-grained-tool-streaming-2025-05-14' } : headers; const client = createOpenAICompatible({ name: provider.type, diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts index 378de665c..af1185ca7 100644 --- a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts +++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts @@ -76,7 +76,7 @@ describe('buildLanguageModel', () => { expect(Reflect.get(model, 'modelId')).toBe('claude-sonnet-5'); }); - it('sets anthropic_beta fine-grained tool streaming header for truefoundry only', () => { + it('sets anthropic-beta fine-grained tool streaming header for truefoundry only', () => { function resolveHeaders(model: unknown): unknown { if (typeof model !== 'object' || model === null) { throw new Error('Expected language model object'); @@ -91,11 +91,11 @@ describe('buildLanguageModel', () => { expect( resolveHeaders(buildLanguageModel(makeConfig({ provider: 'truefoundry', baseUrl: 'http://localhost/v1' }))), ).toMatchObject({ - anthropic_beta: 'fine-grained-tool-streaming-2025-05-14', + 'anthropic-beta': 'fine-grained-tool-streaming-2025-05-14', }); expect( resolveHeaders(buildLanguageModel(makeConfig({ provider: 'fireworks', baseUrl: 'http://localhost/v1' }))), - ).not.toHaveProperty('anthropic_beta'); + ).not.toHaveProperty('anthropic-beta'); }); }); From 4514a557f1cacf33c5861ce7a13bff32396ed34b Mon Sep 17 00:00:00 2001 From: Chirag Jain Date: Mon, 19 Oct 2026 09:34:44 +0530 Subject: [PATCH 4/5] chore: drop TFY beta header unit test; note deprecation Co-authored-by: Cursor --- .../src/core/llm/VercelAILLM.ts | 3 ++- .../tests/core/llm/VercelAILLM.model.test.ts | 22 ------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts index 791b97bf8..71cfdc8d1 100644 --- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts +++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts @@ -132,7 +132,8 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel { if (baseUrl === undefined) { throw new Error(`Provider "${provider.type}" requires a baseUrl`); } - // TFY gateway → Anthropic: prefer the beta header over mutating tool defs (safer rollout). + // More straightforward than mutating tool definitions for fine-grained tool streaming. + // TODO: fine-grained-tool-streaming-2025-05-14 is considered deprecated. const outboundHeaders = provider.type === 'truefoundry' ? { ...headers, 'anthropic-beta': 'fine-grained-tool-streaming-2025-05-14' } diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts index af1185ca7..5b7766a0e 100644 --- a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts +++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts @@ -75,28 +75,6 @@ describe('buildLanguageModel', () => { if (typeof model !== 'object' || model === null) throw new Error('Expected model to be an object'); expect(Reflect.get(model, 'modelId')).toBe('claude-sonnet-5'); }); - - it('sets anthropic-beta fine-grained tool streaming header for truefoundry only', () => { - function resolveHeaders(model: unknown): unknown { - if (typeof model !== 'object' || model === null) { - throw new Error('Expected language model object'); - } - const config = Reflect.get(model, 'config'); - if (typeof config !== 'object' || config === null) { - throw new Error('Expected model config'); - } - const headers = Reflect.get(config, 'headers'); - return typeof headers === 'function' ? headers() : headers; - } - expect( - resolveHeaders(buildLanguageModel(makeConfig({ provider: 'truefoundry', baseUrl: 'http://localhost/v1' }))), - ).toMatchObject({ - 'anthropic-beta': 'fine-grained-tool-streaming-2025-05-14', - }); - expect( - resolveHeaders(buildLanguageModel(makeConfig({ provider: 'fireworks', baseUrl: 'http://localhost/v1' }))), - ).not.toHaveProperty('anthropic-beta'); - }); }); // ─────────── resolveMaxOutputTokens ─────────── From e3e7090344d47c75980896700d920b2fb94fa975 Mon Sep 17 00:00:00 2001 From: Chirag Jain Date: Mon, 19 Oct 2026 09:38:23 +0530 Subject: [PATCH 5/5] feat: report invalid tool-arg JSON as a distinct parse error Fine-grained streaming can leave incomplete arguments; surface parse_error without echoing the raw payload. Co-authored-by: Cursor --- .../tfy-fine-grained-tool-streaming-header.md | 2 +- .../src/core/mcp/executeToolCalls.ts | 50 ++++++++++++++++++- .../tests/core/mcp/executeToolCalls.test.ts | 38 ++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 packages/trueforge-core/tests/core/mcp/executeToolCalls.test.ts diff --git a/.changeset/tfy-fine-grained-tool-streaming-header.md b/.changeset/tfy-fine-grained-tool-streaming-header.md index 6d8560e88..9af6dac41 100644 --- a/.changeset/tfy-fine-grained-tool-streaming-header.md +++ b/.changeset/tfy-fine-grained-tool-streaming-header.md @@ -2,4 +2,4 @@ '@truefoundry/trueforge-core': patch --- -Enable fine-grained tool streaming via TrueFoundry `anthropic-beta` header and Anthropic `toolStreaming` +Enable fine-grained tool streaming via TrueFoundry `anthropic-beta` header and Anthropic `toolStreaming`, and report invalid tool-arg JSON as a distinct parse error diff --git a/packages/trueforge-core/src/core/mcp/executeToolCalls.ts b/packages/trueforge-core/src/core/mcp/executeToolCalls.ts index f8a8e77ab..80625f11c 100644 --- a/packages/trueforge-core/src/core/mcp/executeToolCalls.ts +++ b/packages/trueforge-core/src/core/mcp/executeToolCalls.ts @@ -37,6 +37,36 @@ export interface ExecuteToolCallsResult { events: RegisteredPassthroughEvent[]; } +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Parse tool-call `function.arguments`. Fine-grained streaming can leave incomplete JSON; + * failures are reported as tool errors so the model can retry rather than treating them as + * generic execution failures. + */ +function parseToolCallArguments( + rawArguments: string, +): { ok: true; args: Record } | { ok: false; parseError: string } { + let parsed: unknown; + try { + parsed = JSON.parse(rawArguments || '{}'); + } catch (error) { + return { + ok: false, + parseError: error instanceof Error ? error.message : 'JSON parse failed', + }; + } + if (!isJsonObject(parsed)) { + return { + ok: false, + parseError: 'Tool call arguments must be a JSON object', + }; + } + return { ok: true, args: parsed }; +} + export async function executeToolCalls({ assistantMessage, toolMapping, @@ -82,12 +112,28 @@ export async function executeToolCalls({ }; } + const parsedArgs = parseToolCallArguments(toolCall.function.arguments); + if (!parsedArgs.ok) { + return { + toolCall, + toolInfo, + response: toolResultResponse({ + text: JSON.stringify({ + error: 'Failed to parse tool call arguments as JSON', + parse_error: parsedArgs.parseError, + }), + isError: true, + }), + failure: true, + completedAt: new Date().toISOString(), + }; + } + try { - const args: Record = JSON.parse(toolCall.function.arguments || '{}') as Record; const response = await toolInfo.toolSet.callTool( { name: toolInfo.originalToolName, - arguments: args, + arguments: parsedArgs.args, }, approvalDecisions.get(toolCall.id), ); diff --git a/packages/trueforge-core/tests/core/mcp/executeToolCalls.test.ts b/packages/trueforge-core/tests/core/mcp/executeToolCalls.test.ts new file mode 100644 index 000000000..251387a6d --- /dev/null +++ b/packages/trueforge-core/tests/core/mcp/executeToolCalls.test.ts @@ -0,0 +1,38 @@ +import type { MappedMCPTool } from '../../../src/core/mcp/convertMCPServers'; +import { executeToolCalls } from '../../../src/core/mcp/executeToolCalls'; +import { makeMockIMCPServer } from '../harnessMocks'; + +describe('executeToolCalls', () => { + it('reports invalid tool-arg JSON as a parse failure without calling the tool', async () => { + const toolSet = makeMockIMCPServer({ name: 'srv', preload: true }); + const toolMapping = new Map([['search', { toolSet, originalToolName: 'search' }]]); + + const { toolCallResults } = await executeToolCalls({ + assistantMessage: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { name: 'search', arguments: '{"q":' }, + tool_info: { + type: 'mcp', + mcp_server_id: 'srv', + mcp_server_name: 'srv', + original_tool_name: 'search', + }, + }, + ], + }, + toolMapping, + threadId: 'thread-1', + approvalDecisions: new Map(), + }); + + expect(toolSet.callTool).not.toHaveBeenCalled(); + expect(toolCallResults).toHaveLength(1); + expect(toolCallResults[0]?.failure).toBe(true); + expect(toolCallResults[0]?.message.content).toContain('Failed to parse tool call arguments as JSON'); + }); +});