From d821e841982f275b705749775f500c18e5d497bf Mon Sep 17 00:00:00 2001 From: Leon Weber Date: Fri, 23 Oct 2026 16:59:49 +0400 Subject: [PATCH 1/3] fix(ai-gemini): run built-in tools beside function tools on Gemini 3 Set toolConfig.includeServerSideToolInvocations on the Gemini API when a request mixes built-in tools with function declarations, and send the step's signed toolCall, toolResponse, executableCode and codeExecutionResult parts back in place on the next request. --- .changeset/gemini-server-tool-invocations.md | 5 + docs/adapters/gemini-advanced.md | 31 ++++ docs/config.json | 3 +- packages/ai-gemini/src/adapters/text.ts | 127 +++++++++++-- packages/ai-gemini/src/message-types.ts | 12 +- .../ai-gemini/tests/gemini-adapter.test.ts | 167 +++++++++++++++++- testing/e2e/global-setup.ts | 129 ++++++++++++-- testing/e2e/src/routeTree.gen.ts | 108 ++++++----- .../routes/api.gemini-server-tools-wire.ts | 47 +++++ .../tests/gemini-server-tools-wire.spec.ts | 28 +++ 10 files changed, 578 insertions(+), 79 deletions(-) create mode 100644 .changeset/gemini-server-tool-invocations.md create mode 100644 testing/e2e/src/routes/api.gemini-server-tools-wire.ts create mode 100644 testing/e2e/tests/gemini-server-tools-wire.spec.ts diff --git a/.changeset/gemini-server-tool-invocations.md b/.changeset/gemini-server-tool-invocations.md new file mode 100644 index 0000000000..785254e4b9 --- /dev/null +++ b/.changeset/gemini-server-tool-invocations.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-gemini': patch +--- + +Let Gemini 3 use a built-in tool (`googleSearchTool`, `urlContextTool`, `googleMapsTool`, `fileSearchTool`, `codeExecutionTool`) beside function tools in one `chat()` run. On the Gemini API, the adapter now sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs for this mix. A request without it failed with a 400. The adapter also keeps the signed `toolCall`, `toolResponse`, `executableCode` and `codeExecutionResult` parts of a step on its provider-executed tool call (`metadata.gemini.serverParts`) and sends them back in place on the next request. Without them, the next agent-loop step failed with "Function call is missing a thought_signature". Vertex AI does not get the flag, because `@google/genai` rejects it there. diff --git a/docs/adapters/gemini-advanced.md b/docs/adapters/gemini-advanced.md index 03e79e7ad3..49cc4d9c6e 100644 --- a/docs/adapters/gemini-advanced.md +++ b/docs/adapters/gemini-advanced.md @@ -415,6 +415,37 @@ Gemini has tools of its own, next to the function tools that you define. Import For the concept, a comparison table, and type gating, see [Provider Tools](../tools/provider-tools.md). +### Provider tools beside your own tools + +On Gemini 3, put provider tools and your function tools in one `tools` array. The model can search the web and call your tool in the same step. + +```typescript +import { chat, toolDefinition } from "@tanstack/ai"; +import { geminiText } from "@tanstack/ai-gemini"; +import { googleSearchTool } from "@tanstack/ai-gemini/tools"; +import { z } from "zod"; + +const getOrder = toolDefinition({ + name: "get_order", + description: "Get an order by its id", + inputSchema: z.object({ id: z.string() }), +}).server(async ({ id }) => ({ id, status: "shipped" })); + +const stream = chat({ + adapter: geminiText("gemini-3.8-flash"), + messages: [ + { role: "user", content: "Where is order 42, and does the carrier report delays today?" }, + ], + threadId: "thread-1", + tools: [googleSearchTool(), getOrder], +}); +``` + +The adapter does two things for this mix: + +- **The flag.** On the Gemini API, it sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs to mix the two kinds of tools. The SDK rejects this flag on Vertex AI, so the adapter does not set it there. A `toolConfig` in `modelOptions` wins. +- **The signed parts.** Gemini returns each run of a provider tool as signed parts. The adapter keeps them on a provider-executed tool call, in `metadata.gemini.serverParts`, and sends them back on the next request. If you store messages, keep that metadata. + ### `codeExecutionTool` Gemini runs Python code in a sandbox and returns the results inline. It takes no arguments. Add it to the `tools` array. diff --git a/docs/config.json b/docs/config.json index 359b32483f..85494e9dfc 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1703,7 +1703,8 @@ { "label": "Google Gemini: Advanced", "to": "adapters/gemini-advanced", - "addedAt": "2026-10-21" + "addedAt": "2026-10-21", + "updatedAt": "2026-10-23" }, { "label": "Google Vertex AI", diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index 575a822edd..74dd5d3581 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -15,6 +15,7 @@ import { } from '@tanstack/ai/adapter-internals' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' +import { getGeminiProviderToolKind } from '../tools/gemini-provider-tool' import { buildGeminiUsage } from '../usage' import { geminiThinkingConfig } from '../text/reasoning' import { @@ -142,6 +143,24 @@ function getGroundingSources( return [...sources.values()] } +/** A part of a built-in tool run, which Gemini needs back unchanged. */ +function isServerPart(part: Part): boolean { + return Boolean( + part.toolCall || + part.toolResponse || + part.executableCode || + part.codeExecutionResult, + ) +} + +/** The tool name of a built-in tool part, when the part names its tool. */ +function serverToolName(part: Part): string | undefined { + if (part.executableCode || part.codeExecutionResult) return 'code_execution' + const type = (part.toolCall ?? part.toolResponse)?.toolType + if (!type || type === 'TOOL_TYPE_UNSPECIFIED') return undefined + return type.startsWith('GOOGLE_SEARCH') ? 'google_search' : type.toLowerCase() +} + /** Maps `chat({ toolChoice })` to the Gemini function calling config. */ function toGeminiFunctionCallingConfig( choice: ToolChoice, @@ -872,28 +891,50 @@ export class GeminiTextAdapter< let hasEmittedTextMessageStart = false let groundingMetadata: GroundingMetadata | undefined let groundingCallEmitted = false + // Gemini needs a step's built-in tool parts back only when the step can + // also call a function tool. + const keepServerParts = + options.tools?.some( + (tool) => getGeminiProviderToolKind(tool) === undefined, + ) ?? false + const serverParts: Array<{ part: Part; afterCallId?: string }> = [] + let lastCallId: string | undefined const adapterName = this.name const emitGroundingToolCall = function* (): Generator { - if (!groundingMetadata || groundingCallEmitted) return - const sources = getGroundingSources(groundingMetadata) + if (groundingCallEmitted) return + const sources = groundingMetadata + ? getGroundingSources(groundingMetadata) + : [] const hasGoogleSearchEvidence = sources.length > 0 || - (groundingMetadata.webSearchQueries?.length ?? 0) > 0 || - groundingMetadata.searchEntryPoint !== undefined - if (!hasGoogleSearchEvidence) return + (groundingMetadata?.webSearchQueries?.length ?? 0) > 0 || + groundingMetadata?.searchEntryPoint !== undefined + if (!hasGoogleSearchEvidence && serverParts.length === 0) return groundingCallEmitted = true const toolCallId = generateId(adapterName) + const toolCallName = + (!hasGoogleSearchEvidence && + (serverParts + .map(({ part }) => serverToolName(part)) + .find((name) => name !== undefined) ?? + options.tools?.find( + (tool) => getGeminiProviderToolKind(tool) !== undefined, + )?.name)) || + 'google_search' const metadata: GeminiToolCallMetadata = { providerExecuted: true, sources, - gemini: { groundingMetadata }, + gemini: { + ...(groundingMetadata && { groundingMetadata }), + ...(serverParts.length > 0 && { serverParts }), + }, } yield { type: EventType.TOOL_CALL_START, toolCallId, - toolCallName: 'google_search', + toolCallName, parentMessageId: messageId, model, timestamp: Date.now(), @@ -903,11 +944,11 @@ export class GeminiTextAdapter< yield { type: EventType.TOOL_CALL_END, toolCallId, - toolCallName: 'google_search', + toolCallName, model, timestamp: Date.now(), input: { - ...(groundingMetadata.webSearchQueries && { + ...(groundingMetadata?.webSearchQueries && { queries: groundingMetadata.webSearchQueries, }), }, @@ -945,6 +986,13 @@ export class GeminiTextAdapter< const parts = chunk.candidates[0].content.parts for (const part of parts) { + if (keepServerParts && isServerPart(part)) { + serverParts.push({ + part, + ...(lastCallId !== undefined && { afterCallId: lastCallId }), + }) + } + if (part.text) { if (part.thought) { // Emit REASONING start events on first thinking content @@ -1024,6 +1072,7 @@ export class GeminiTextAdapter< functionCall.id || `${functionCall.name}_${Date.now()}_${nextToolIndex}` const functionArgs = functionCall.args || {} + lastCallId = toolCallId // Gemini emits thoughtSignature as a Part-level sibling of // functionCall (per @google/genai Part type), not nested inside @@ -1441,6 +1490,46 @@ export class GeminiTextAdapter< .map(thoughtPart), ) } + + // Built-in tool parts go back where the model put them: after the + // function call they followed, else before the first part that is + // not a thought. Gemini rejects the step when they are missing or + // out of place. The SDK throws on them for Vertex AI. + const serverParts = this.client.vertexai + ? [] + : (msg.toolCalls ?? []).flatMap( + (toolCall) => + (toolCall.metadata as GeminiToolCallMetadata | undefined) + ?.gemini?.serverParts ?? [], + ) + if (serverParts.length > 0) { + const after = (id: string | undefined) => + serverParts + .filter(({ afterCallId }) => afterCallId === id) + .map(({ part }) => part) + const callIds = new Set( + parts.flatMap((part) => part.functionCall?.id ?? []), + ) + const first = parts.findIndex((part) => !part.thought) + const withServerParts = parts.flatMap((part, index) => [ + ...(index === first ? after(undefined) : []), + part, + ...(part.functionCall?.id !== undefined + ? after(part.functionCall.id) + : []), + ]) + if (first === -1) withServerParts.push(...after(undefined)) + // A part whose call is not in this message goes last, not lost. + withServerParts.push( + ...serverParts + .filter( + ({ afterCallId }) => + afterCallId !== undefined && !callIds.has(afterCallId), + ) + .map(({ part }) => part), + ) + parts.splice(0, parts.length, ...withServerParts) + } } return { role, @@ -1553,11 +1642,17 @@ export class GeminiTextAdapter< // them. A `toolConfig` in modelOptions wins key by key, so its own // `functionCallingConfig` wins, and a `retrievalConfig` alone keeps this one. const tools = convertToolsToProviderFormat(options.tools) + const hasFunctions = tools.some((tool) => 'functionDeclarations' in tool) const functionCallingConfig = - options.toolChoice !== undefined && - tools.some((tool) => 'functionDeclarations' in tool) + options.toolChoice !== undefined && hasFunctions ? toGeminiFunctionCallingConfig(options.toolChoice) : undefined + // The Gemini API needs this flag to mix built-in tools with function + // declarations. The SDK throws on it for Vertex AI. + const includeServerSideToolInvocations = + !this.client.vertexai && + hasFunctions && + tools.some((tool) => !('functionDeclarations' in tool)) // Vendor `GenerateContentConfig` fields are `field?: T` (no `| undefined`) // under EOPT, so spread each common option only when present rather than @@ -1573,9 +1668,15 @@ export class GeminiTextAdapter< ), config: { ...modelOpts, - ...(functionCallingConfig !== undefined && { + ...((functionCallingConfig !== undefined || + includeServerSideToolInvocations) && { toolConfig: { - functionCallingConfig, + ...(functionCallingConfig !== undefined && { + functionCallingConfig, + }), + ...(includeServerSideToolInvocations && { + includeServerSideToolInvocations, + }), ...options.modelOptions?.toolConfig, }, }), diff --git a/packages/ai-gemini/src/message-types.ts b/packages/ai-gemini/src/message-types.ts index 0217565f11..e71cc63f2c 100644 --- a/packages/ai-gemini/src/message-types.ts +++ b/packages/ai-gemini/src/message-types.ts @@ -1,5 +1,5 @@ import type { ProviderExecutedToolMetadata } from '@tanstack/ai' -import type { GroundingMetadata } from '@google/genai' +import type { GroundingMetadata, Part } from '@google/genai' /** * Gemini-specific metadata types for multimodal content parts. @@ -184,6 +184,14 @@ export interface GeminiMessageMetadataByModality { export interface GeminiToolCallMetadata extends ProviderExecutedToolMetadata { thoughtSignature?: string gemini?: { - groundingMetadata: GroundingMetadata + groundingMetadata?: GroundingMetadata + /** + * The step's built-in tool parts (`toolCall`, `toolResponse`, + * `executableCode`, `codeExecutionResult`), sent back unchanged on the + * next request. `afterCallId` is the function call each one followed. + * + * @see https://ai.google.dev/gemini-api/docs/generate-content/tool-combination + */ + serverParts?: Array<{ part: Part; afterCallId?: string }> } } diff --git a/packages/ai-gemini/tests/gemini-adapter.test.ts b/packages/ai-gemini/tests/gemini-adapter.test.ts index 6fa8656d7c..8e495e8a8a 100644 --- a/packages/ai-gemini/tests/gemini-adapter.test.ts +++ b/packages/ai-gemini/tests/gemini-adapter.test.ts @@ -10,7 +10,7 @@ import { type SafetySetting, } from '@google/genai' import { GeminiTextAdapter } from '../src/adapters/text' -import { googleSearchTool } from '../src/tools' +import { codeExecutionTool, googleSearchTool } from '../src/tools' import { createGeminiSummarize } from '../src/adapters/summarize' import type { GeminiTextProviderOptions } from '../src/adapters/text' import type { Schema } from '@google/genai' @@ -41,8 +41,11 @@ vi.mock('@google/genai', async () => { public getGenerativeModel = getGenerativeModelSpy - constructor(options: { apiKey: string }) { + public vertexai: boolean + + constructor(options: { apiKey: string; vertexai?: boolean }) { constructorSpy(options) + this.vertexai = options.vertexai === true } } @@ -801,6 +804,166 @@ describe('GeminiAdapter through AI', () => { ).toHaveLength(0) }) + describe('built-in tools beside function tools', () => { + const toolCallPart = { + toolCall: { + id: 'search_1', + toolType: 'GOOGLE_SEARCH_WEB', + args: { queries: ['utah news'] }, + }, + thoughtSignature: 'sig-search', + } + const toolResponsePart = { + toolResponse: { + id: 'search_1', + toolType: 'GOOGLE_SEARCH_WEB', + response: { search_suggestions: '
' }, + }, + thoughtSignature: 'sig-response', + } + const lookupTool: Tool = { + name: 'lookup_caucus', + description: 'Look up a caucus location', + execute: async () => ({ location: 'Library' }), + } + const textStream = () => + createStream([ + { + candidates: [ + { content: { parts: [{ text: 'Done' }] }, finishReason: 'STOP' }, + ], + }, + ]) + + it('sets the flag and sends the built-in tool parts back in place', async () => { + mocks.generateContentStreamSpy + .mockResolvedValueOnce( + createStream([ + { + candidates: [ + { + content: { + parts: [ + toolCallPart, + { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + }, + toolResponsePart, + ], + }, + finishReason: 'STOP', + }, + ], + }, + ]), + ) + .mockResolvedValueOnce(textStream()) + + const received: Array = [] + for await (const chunk of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { apiKey: 'test-key' }, + 'gemini-3.6-flash', + ), + tools: [googleSearchTool(), lookupTool], + messages: [{ role: 'user', content: 'Search and look up.' }], + })) { + received.push(chunk) + } + + expect(mocks.generateContentStreamSpy).toHaveBeenCalledTimes(2) + const [firstPayload] = mocks.generateContentStreamSpy.mock.calls[0]! + expect(firstPayload.config.toolConfig).toEqual({ + includeServerSideToolInvocations: true, + }) + expect( + received.find( + (chunk) => + chunk.type === 'TOOL_CALL_START' && + chunk.toolCallName === 'google_search', + ), + ).toMatchObject({ metadata: { providerExecuted: true } }) + + const [secondPayload] = mocks.generateContentStreamSpy.mock.calls[1]! + const modelTurn = secondPayload.contents.find( + (content: any) => content.role === 'model', + ) + expect(modelTurn.parts).toEqual([ + toolCallPart, + { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + }, + toolResponsePart, + ]) + }) + + it('emits no provider call for a code run without function tools', async () => { + mocks.generateContentStreamSpy.mockResolvedValueOnce( + createStream([ + { + candidates: [ + { + content: { + parts: [ + { executableCode: { code: 'print(2**40)' } }, + { codeExecutionResult: { output: '1099511627776' } }, + { text: '1099511627776' }, + ], + }, + finishReason: 'STOP', + }, + ], + }, + ]), + ) + + const received: Array = [] + for await (const chunk of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { apiKey: 'test-key' }, + 'gemini-3.6-flash', + ), + tools: [codeExecutionTool()], + messages: [{ role: 'user', content: 'Compute 2**40.' }], + })) { + received.push(chunk) + } + + expect( + received.filter((chunk) => chunk.type === 'TOOL_CALL_START'), + ).toHaveLength(0) + }) + + it('does not set the flag on Vertex AI, where the SDK rejects it', async () => { + mocks.generateContentStreamSpy.mockResolvedValueOnce(textStream()) + + for await (const _ of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { vertexai: true, project: 'p', location: 'us-central1' }, + 'gemini-3.6-flash', + ), + tools: [googleSearchTool(), lookupTool], + messages: [{ role: 'user', content: 'Search and look up.' }], + })) { + // consume + } + + const [payload] = mocks.generateContentStreamSpy.mock.calls[0]! + expect(payload.config.toolConfig).toBeUndefined() + }) + }) + it('emits parentMessageId on tool-first tool calls matching the assistant message id', async () => { // A functionCall part arrives before any text. parentMessageId must bind the // tool call to the same assistant message id the eventual TEXT_MESSAGE_START diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 0667c55f0f..3a1a613bac 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -167,6 +167,10 @@ export default async function globalSetup() { mock.mount('/openai-responses-usage', openaiResponsesUsageMount()) mock.mount('/provider-search-openai', openaiProviderSearchMount()) mock.mount('/provider-search-gemini', geminiProviderSearchMount()) + // Google Search beside a function tool on Gemini 3. Answers like the + // Gemini API: the flag is required, and the second step must send the + // signed toolCall/toolResponse parts back around the function call. + mock.mount('/gemini-server-tools', geminiServerToolsMount()) // Bedrock Converse prompt caching. aimock cannot replay the Converse // `vnd.amazon.eventstream` protocol, so this mount encodes the frames itself @@ -725,10 +729,7 @@ const TINY_PNG_BASE64 = * status } }`) so a validation failure surfaces as the actual missing/extra * field in the adapter's thrown error message, not a generic 500. */ -function rejectGeminiImageRequest( - res: http.ServerResponse, - message: string, -): true { +function rejectGeminiRequest(res: http.ServerResponse, message: string): true { res.statusCode = 400 res.setHeader('Content-Type', 'application/json') res.end( @@ -811,32 +812,29 @@ function geminiNativeImageMount(): Mountable { const body = await readJsonRequestBody(req) if (!body) { - return rejectGeminiImageRequest(res, 'Malformed JSON body.') + return rejectGeminiRequest(res, 'Malformed JSON body.') } const model = body.model if (typeof model !== 'string' || !NATIVE_IMAGE_MODELS.has(model)) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, `Unexpected image model ${JSON.stringify(model)}.`, ) } if (body.stream !== false) { - return rejectGeminiImageRequest(res, 'stream must be false.') + return rejectGeminiRequest(res, 'stream must be false.') } const format = asRecord(body.response_format) if (!format || format.type !== 'image') { - return rejectGeminiImageRequest( - res, - 'response_format.type must be "image".', - ) + return rejectGeminiRequest(res, 'response_format.type must be "image".') } const leaked = ROOT_FORBIDDEN.find((name) => name in body) if (leaked) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, `Field "${leaked}" reached the Interactions image request.`, ) @@ -847,7 +845,7 @@ function geminiNativeImageMount(): Mountable { ? GENERATION_FORBIDDEN.find((name) => name in generationConfig) : undefined if (leakedGeneration) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, `Field "${leakedGeneration}" reached generation_config.`, ) @@ -860,7 +858,7 @@ function geminiNativeImageMount(): Mountable { if (isEdit) { if (inputHasImage(body.input)) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, 'A chained edit must not resend an image block.', ) @@ -875,13 +873,13 @@ function geminiNativeImageMount(): Mountable { setting.type !== 'dangerous_content' || setting.threshold !== 'block_only_high' ) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, 'safety_settings must be [{ type: "dangerous_content", threshold: "block_only_high" }].', ) } if (generationConfig?.thinking_level !== 'low') { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, 'generation_config.thinking_level must be "low".', ) @@ -890,13 +888,13 @@ function geminiNativeImageMount(): Mountable { const aspectRatio = format.aspect_ratio const imageSize = format.image_size if (aspectRatio !== EXPECTED_ASPECT_RATIO) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, `${model}: response_format.aspect_ratio must be "${EXPECTED_ASPECT_RATIO}", got ${JSON.stringify(aspectRatio)}.`, ) } if (imageSize !== EXPECTED_FLASH_IMAGE_SIZE) { - return rejectGeminiImageRequest( + return rejectGeminiRequest( res, `${model}: response_format.image_size must be "${EXPECTED_FLASH_IMAGE_SIZE}", got ${JSON.stringify(imageSize)}.`, ) @@ -2191,6 +2189,101 @@ function geminiProviderSearchMount(): Mountable { } } +function geminiServerToolsMount(): Mountable { + const toolCall = { + toolCall: { + id: 'search_1', + toolType: 'GOOGLE_SEARCH_WEB', + args: { queries: ['caucus news'] }, + }, + thoughtSignature: 'c2lnLXNlYXJjaA==', + } + const functionCall = { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + } + const toolResponse = { + toolResponse: { + id: 'search_1', + toolType: 'GOOGLE_SEARCH_WEB', + response: { results: ['Caucus night is Tuesday'] }, + }, + thoughtSignature: 'c2lnLXJlc3BvbnNl', + } + return { + async handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse, + pathname: string, + ): Promise { + if ( + req.method !== 'POST' || + !pathname.endsWith(':streamGenerateContent') + ) { + return false + } + const body = await readJsonRequestBody(req) + if ( + asRecord(body?.toolConfig)?.includeServerSideToolInvocations !== true + ) { + return rejectGeminiRequest( + res, + 'Please enable tool_config.include_server_side_tool_invocations to use Built-in tools with Function calling.', + ) + } + const contents = Array.isArray(body?.contents) ? body.contents : [] + const modelTurn = contents + .map(asRecord) + .find((content) => content?.role === 'model') + const sent = Array.isArray(modelTurn?.parts) + ? modelTurn.parts.map(asRecord) + : [] + let parts: Array + if (!modelTurn) { + parts = [toolCall, functionCall, toolResponse] + } else if ( + sent.length !== 3 || + !sent[0]?.toolCall || + sent[0].thoughtSignature !== toolCall.thoughtSignature || + !sent[1]?.functionCall || + !sent[2]?.toolResponse || + sent[2].thoughtSignature !== toolResponse.thoughtSignature + ) { + return rejectGeminiRequest( + res, + 'Function call is missing a thought_signature in functionCall parts.', + ) + } else { + parts = [{ text: 'Caucus night is Tuesday at the library.' }] + } + + res.statusCode = 200 + res.setHeader('Content-Type', 'text/event-stream') + res.setHeader('Cache-Control', 'no-cache') + res.end( + `data: ${JSON.stringify({ + candidates: [ + { + content: { role: 'model', parts }, + finishReason: 'STOP', + index: 0, + }, + ], + usageMetadata: { + promptTokenCount: 1, + candidatesTokenCount: 1, + totalTokenCount: 2, + }, + })}\n\n`, + ) + return true + }, + } +} + /** * Mounts the OpenAI Decisions `/v1/decisions` response for the * `evaluate-usage-on-failure` spec: valid usage, but a `queue` answer whose diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 35cb311a76..445df11af5 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -102,6 +102,7 @@ import { Route as ApiForeignClientToolRouteImport } from './routes/api.foreign-c import { Route as ApiForeignInterruptRouteImport } from './routes/api.foreign-interrupt' import { Route as ApiGeminiImageGaModelsRouteImport } from './routes/api.gemini-image-ga-models' import { Route as ApiGeminiNativeImageWireRouteImport } from './routes/api.gemini-native-image-wire' +import { Route as ApiGeminiServerToolsWireRouteImport } from './routes/api.gemini-server-tools-wire' import { Route as ApiGenerationPersistenceResumeRouteImport } from './routes/api.generation-persistence-resume' import { Route as ApiGenerationPersistenceServerRouteImport } from './routes/api.generation-persistence-server' import { Route as ApiHarnessAgentTreeRouteImport } from './routes/api.harness-agent-tree' @@ -200,10 +201,10 @@ import { Route as ApiWebMcpPageToolsRouteImport } from './routes/api.web-mcp-pag import { Route as ApiWorldRouteImport } from './routes/api.world' import { Route as ApiWrapFetchWireRouteImport } from './routes/api.wrap-fetch-wire' import { Route as ApiAudioStreamRouteImport } from './routes/api.audio.stream' -import { Route as ApiHarnessProtocolSplatRouteImport } from './routes/api.harness-protocol.$' -import { Route as ApiHarnessSecretAnswerSplatRouteImport } from './routes/api.harness-secret-answer.$' import { Route as ApiHarnessHandoffSplatRouteImport } from './routes/api.harness-handoff.$' import { Route as ApiHarnessObservabilitySplatRouteImport } from './routes/api.harness-observability.$' +import { Route as ApiHarnessProtocolSplatRouteImport } from './routes/api.harness-protocol.$' +import { Route as ApiHarnessSecretAnswerSplatRouteImport } from './routes/api.harness-secret-answer.$' import { Route as ApiHarnessWaitsSplatRouteImport } from './routes/api.harness-waits.$' import { Route as ApiImageStreamRouteImport } from './routes/api.image.stream' import { Route as ApiTranscriptionStreamRouteImport } from './routes/api.transcription.stream' @@ -698,6 +699,12 @@ const ApiGeminiNativeImageWireRoute = path: '/api/gemini-native-image-wire', getParentRoute: () => rootRouteImport, } as any) +const ApiGeminiServerToolsWireRoute = + ApiGeminiServerToolsWireRouteImport.update({ + id: '/api/gemini-server-tools-wire', + path: '/api/gemini-server-tools-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiGenerationPersistenceResumeRoute = ApiGenerationPersistenceResumeRouteImport.update({ id: '/api/generation-persistence-resume', @@ -1212,17 +1219,6 @@ const ApiAudioStreamRoute = ApiAudioStreamRouteImport.update({ path: '/stream', getParentRoute: () => ApiAudioRoute, } as any) -const ApiHarnessProtocolSplatRoute = ApiHarnessProtocolSplatRouteImport.update({ - id: '/api/harness-protocol/$', - path: '/api/harness-protocol/$', - getParentRoute: () => rootRouteImport, -} as any) -const ApiHarnessSecretAnswerSplatRoute = - ApiHarnessSecretAnswerSplatRouteImport.update({ - id: '/api/harness-secret-answer/$', - path: '/api/harness-secret-answer/$', - getParentRoute: () => rootRouteImport, - } as any) const ApiHarnessHandoffSplatRoute = ApiHarnessHandoffSplatRouteImport.update({ id: '/api/harness-handoff/$', path: '/api/harness-handoff/$', @@ -1234,6 +1230,17 @@ const ApiHarnessObservabilitySplatRoute = path: '/api/harness-observability/$', getParentRoute: () => rootRouteImport, } as any) +const ApiHarnessProtocolSplatRoute = ApiHarnessProtocolSplatRouteImport.update({ + id: '/api/harness-protocol/$', + path: '/api/harness-protocol/$', + getParentRoute: () => rootRouteImport, +} as any) +const ApiHarnessSecretAnswerSplatRoute = + ApiHarnessSecretAnswerSplatRouteImport.update({ + id: '/api/harness-secret-answer/$', + path: '/api/harness-secret-answer/$', + getParentRoute: () => rootRouteImport, + } as any) const ApiHarnessWaitsSplatRoute = ApiHarnessWaitsSplatRouteImport.update({ id: '/api/harness-waits/$', path: '/api/harness-waits/$', @@ -1353,6 +1360,7 @@ export interface FileRoutesByFullPath { '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/gemini-image-ga-models': typeof ApiGeminiImageGaModelsRoute '/api/gemini-native-image-wire': typeof ApiGeminiNativeImageWireRoute + '/api/gemini-server-tools-wire': typeof ApiGeminiServerToolsWireRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute '/api/harness-agent-tree': typeof ApiHarnessAgentTreeRoute @@ -1452,10 +1460,10 @@ export interface FileRoutesByFullPath { '/api/wrap-fetch-wire': typeof ApiWrapFetchWireRoute '/$provider/': typeof ProviderIndexRoute '/api/audio/stream': typeof ApiAudioStreamRoute - '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute - '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-handoff/$': typeof ApiHarnessHandoffSplatRoute '/api/harness-observability/$': typeof ApiHarnessObservabilitySplatRoute + '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute + '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-waits/$': typeof ApiHarnessWaitsSplatRoute '/api/image/stream': typeof ApiImageStreamRoute '/api/transcription/stream': typeof ApiTranscriptionStreamRoute @@ -1555,6 +1563,7 @@ export interface FileRoutesByTo { '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/gemini-image-ga-models': typeof ApiGeminiImageGaModelsRoute '/api/gemini-native-image-wire': typeof ApiGeminiNativeImageWireRoute + '/api/gemini-server-tools-wire': typeof ApiGeminiServerToolsWireRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute '/api/harness-agent-tree': typeof ApiHarnessAgentTreeRoute @@ -1654,10 +1663,10 @@ export interface FileRoutesByTo { '/api/wrap-fetch-wire': typeof ApiWrapFetchWireRoute '/$provider': typeof ProviderIndexRoute '/api/audio/stream': typeof ApiAudioStreamRoute - '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute - '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-handoff/$': typeof ApiHarnessHandoffSplatRoute '/api/harness-observability/$': typeof ApiHarnessObservabilitySplatRoute + '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute + '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-waits/$': typeof ApiHarnessWaitsSplatRoute '/api/image/stream': typeof ApiImageStreamRoute '/api/transcription/stream': typeof ApiTranscriptionStreamRoute @@ -1758,6 +1767,7 @@ export interface FileRoutesById { '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/gemini-image-ga-models': typeof ApiGeminiImageGaModelsRoute '/api/gemini-native-image-wire': typeof ApiGeminiNativeImageWireRoute + '/api/gemini-server-tools-wire': typeof ApiGeminiServerToolsWireRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute '/api/harness-agent-tree': typeof ApiHarnessAgentTreeRoute @@ -1857,10 +1867,10 @@ export interface FileRoutesById { '/api/wrap-fetch-wire': typeof ApiWrapFetchWireRoute '/$provider/': typeof ProviderIndexRoute '/api/audio/stream': typeof ApiAudioStreamRoute - '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute - '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-handoff/$': typeof ApiHarnessHandoffSplatRoute '/api/harness-observability/$': typeof ApiHarnessObservabilitySplatRoute + '/api/harness-protocol/$': typeof ApiHarnessProtocolSplatRoute + '/api/harness-secret-answer/$': typeof ApiHarnessSecretAnswerSplatRoute '/api/harness-waits/$': typeof ApiHarnessWaitsSplatRoute '/api/image/stream': typeof ApiImageStreamRoute '/api/transcription/stream': typeof ApiTranscriptionStreamRoute @@ -1962,6 +1972,7 @@ export interface FileRouteTypes { | '/api/foreign-interrupt' | '/api/gemini-image-ga-models' | '/api/gemini-native-image-wire' + | '/api/gemini-server-tools-wire' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' | '/api/harness-agent-tree' @@ -2061,10 +2072,10 @@ export interface FileRouteTypes { | '/api/wrap-fetch-wire' | '/$provider/' | '/api/audio/stream' - | '/api/harness-protocol/$' - | '/api/harness-secret-answer/$' | '/api/harness-handoff/$' | '/api/harness-observability/$' + | '/api/harness-protocol/$' + | '/api/harness-secret-answer/$' | '/api/harness-waits/$' | '/api/image/stream' | '/api/transcription/stream' @@ -2164,6 +2175,7 @@ export interface FileRouteTypes { | '/api/foreign-interrupt' | '/api/gemini-image-ga-models' | '/api/gemini-native-image-wire' + | '/api/gemini-server-tools-wire' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' | '/api/harness-agent-tree' @@ -2263,10 +2275,10 @@ export interface FileRouteTypes { | '/api/wrap-fetch-wire' | '/$provider' | '/api/audio/stream' - | '/api/harness-protocol/$' - | '/api/harness-secret-answer/$' | '/api/harness-handoff/$' | '/api/harness-observability/$' + | '/api/harness-protocol/$' + | '/api/harness-secret-answer/$' | '/api/harness-waits/$' | '/api/image/stream' | '/api/transcription/stream' @@ -2366,6 +2378,7 @@ export interface FileRouteTypes { | '/api/foreign-interrupt' | '/api/gemini-image-ga-models' | '/api/gemini-native-image-wire' + | '/api/gemini-server-tools-wire' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' | '/api/harness-agent-tree' @@ -2465,10 +2478,10 @@ export interface FileRouteTypes { | '/api/wrap-fetch-wire' | '/$provider/' | '/api/audio/stream' - | '/api/harness-protocol/$' - | '/api/harness-secret-answer/$' | '/api/harness-handoff/$' | '/api/harness-observability/$' + | '/api/harness-protocol/$' + | '/api/harness-secret-answer/$' | '/api/harness-waits/$' | '/api/image/stream' | '/api/transcription/stream' @@ -2569,6 +2582,7 @@ export interface RootRouteChildren { ApiForeignInterruptRoute: typeof ApiForeignInterruptRoute ApiGeminiImageGaModelsRoute: typeof ApiGeminiImageGaModelsRoute ApiGeminiNativeImageWireRoute: typeof ApiGeminiNativeImageWireRoute + ApiGeminiServerToolsWireRoute: typeof ApiGeminiServerToolsWireRoute ApiGenerationPersistenceResumeRoute: typeof ApiGenerationPersistenceResumeRoute ApiGenerationPersistenceServerRoute: typeof ApiGenerationPersistenceServerRoute ApiHarnessAgentTreeRoute: typeof ApiHarnessAgentTreeRoute @@ -2667,10 +2681,10 @@ export interface RootRouteChildren { ApiWorldRoute: typeof ApiWorldRoute ApiWrapFetchWireRoute: typeof ApiWrapFetchWireRoute ProviderIndexRoute: typeof ProviderIndexRoute - ApiHarnessProtocolSplatRoute: typeof ApiHarnessProtocolSplatRoute - ApiHarnessSecretAnswerSplatRoute: typeof ApiHarnessSecretAnswerSplatRoute ApiHarnessHandoffSplatRoute: typeof ApiHarnessHandoffSplatRoute ApiHarnessObservabilitySplatRoute: typeof ApiHarnessObservabilitySplatRoute + ApiHarnessProtocolSplatRoute: typeof ApiHarnessProtocolSplatRoute + ApiHarnessSecretAnswerSplatRoute: typeof ApiHarnessSecretAnswerSplatRoute ApiHarnessWaitsSplatRoute: typeof ApiHarnessWaitsSplatRoute } @@ -3327,6 +3341,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiGeminiNativeImageWireRouteImport parentRoute: typeof rootRouteImport } + '/api/gemini-server-tools-wire': { + id: '/api/gemini-server-tools-wire' + path: '/api/gemini-server-tools-wire' + fullPath: '/api/gemini-server-tools-wire' + preLoaderRoute: typeof ApiGeminiServerToolsWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/generation-persistence-resume': { id: '/api/generation-persistence-resume' path: '/api/generation-persistence-resume' @@ -4013,20 +4034,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiAudioStreamRouteImport parentRoute: typeof ApiAudioRoute } - '/api/harness-protocol/$': { - id: '/api/harness-protocol/$' - path: '/api/harness-protocol/$' - fullPath: '/api/harness-protocol/$' - preLoaderRoute: typeof ApiHarnessProtocolSplatRouteImport - parentRoute: typeof rootRouteImport - } - '/api/harness-secret-answer/$': { - id: '/api/harness-secret-answer/$' - path: '/api/harness-secret-answer/$' - fullPath: '/api/harness-secret-answer/$' - preLoaderRoute: typeof ApiHarnessSecretAnswerSplatRouteImport - parentRoute: typeof rootRouteImport - } '/api/harness-handoff/$': { id: '/api/harness-handoff/$' path: '/api/harness-handoff/$' @@ -4041,6 +4048,20 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiHarnessObservabilitySplatRouteImport parentRoute: typeof rootRouteImport } + '/api/harness-protocol/$': { + id: '/api/harness-protocol/$' + path: '/api/harness-protocol/$' + fullPath: '/api/harness-protocol/$' + preLoaderRoute: typeof ApiHarnessProtocolSplatRouteImport + parentRoute: typeof rootRouteImport + } + '/api/harness-secret-answer/$': { + id: '/api/harness-secret-answer/$' + path: '/api/harness-secret-answer/$' + fullPath: '/api/harness-secret-answer/$' + preLoaderRoute: typeof ApiHarnessSecretAnswerSplatRouteImport + parentRoute: typeof rootRouteImport + } '/api/harness-waits/$': { id: '/api/harness-waits/$' path: '/api/harness-waits/$' @@ -4235,6 +4256,7 @@ const rootRouteChildren: RootRouteChildren = { ApiForeignInterruptRoute: ApiForeignInterruptRoute, ApiGeminiImageGaModelsRoute: ApiGeminiImageGaModelsRoute, ApiGeminiNativeImageWireRoute: ApiGeminiNativeImageWireRoute, + ApiGeminiServerToolsWireRoute: ApiGeminiServerToolsWireRoute, ApiGenerationPersistenceResumeRoute: ApiGenerationPersistenceResumeRoute, ApiGenerationPersistenceServerRoute: ApiGenerationPersistenceServerRoute, ApiHarnessAgentTreeRoute: ApiHarnessAgentTreeRoute, @@ -4334,10 +4356,10 @@ const rootRouteChildren: RootRouteChildren = { ApiWorldRoute: ApiWorldRoute, ApiWrapFetchWireRoute: ApiWrapFetchWireRoute, ProviderIndexRoute: ProviderIndexRoute, - ApiHarnessProtocolSplatRoute: ApiHarnessProtocolSplatRoute, - ApiHarnessSecretAnswerSplatRoute: ApiHarnessSecretAnswerSplatRoute, ApiHarnessHandoffSplatRoute: ApiHarnessHandoffSplatRoute, ApiHarnessObservabilitySplatRoute: ApiHarnessObservabilitySplatRoute, + ApiHarnessProtocolSplatRoute: ApiHarnessProtocolSplatRoute, + ApiHarnessSecretAnswerSplatRoute: ApiHarnessSecretAnswerSplatRoute, ApiHarnessWaitsSplatRoute: ApiHarnessWaitsSplatRoute, } export const routeTree = rootRouteImport diff --git a/testing/e2e/src/routes/api.gemini-server-tools-wire.ts b/testing/e2e/src/routes/api.gemini-server-tools-wire.ts new file mode 100644 index 0000000000..2c15ec0916 --- /dev/null +++ b/testing/e2e/src/routes/api.gemini-server-tools-wire.ts @@ -0,0 +1,47 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, toolDefinition } from '@tanstack/ai' +import { createGeminiChat } from '@tanstack/ai-gemini' +import { googleSearchTool } from '@tanstack/ai-gemini/tools' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' +const DUMMY_KEY = 'sk-e2e-test-dummy-key' + +const lookupCaucus = toolDefinition({ + name: 'lookup_caucus', + description: 'Look up a caucus location by county', + inputSchema: { + type: 'object', + properties: { county: { type: 'string' } }, + required: ['county'], + }, +}).server(async () => ({ location: 'Library' })) + +export const Route = createFileRoute('/api/gemini-server-tools-wire')({ + server: { + handlers: { + POST: async () => { + const adapter = createGeminiChat('gemini-3.6-flash', DUMMY_KEY, { + baseURL: `${LLMOCK_DEFAULT_BASE}/gemini-server-tools`, + }) + let text = '' + let error: string | undefined + const tools: Array = [] + try { + for await (const chunk of chat({ + threadId: 'gemini-server-tools-wire', + adapter, + messages: [{ role: 'user', content: 'Search and look up.' }], + tools: [googleSearchTool(), lookupCaucus], + })) { + if (chunk.type === 'TOOL_CALL_START') tools.push(chunk.toolCallName) + if (chunk.type === 'TEXT_MESSAGE_CONTENT') text += chunk.delta + if (chunk.type === 'RUN_ERROR') error = chunk.message + } + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught) + } + return Response.json({ ok: error === undefined, error, text, tools }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/gemini-server-tools-wire.spec.ts b/testing/e2e/tests/gemini-server-tools-wire.spec.ts new file mode 100644 index 0000000000..184722cda2 --- /dev/null +++ b/testing/e2e/tests/gemini-server-tools-wire.spec.ts @@ -0,0 +1,28 @@ +import { test, expect } from './fixtures' + +/** + * Google Search beside a function tool on Gemini 3. The + * `/gemini-server-tools` mount fails like the Gemini API when the request has + * no `includeServerSideToolInvocations`, or when the next step does not send + * the signed `toolCall` / `toolResponse` parts back around the function call. + */ +test('gemini runs Google Search and a function tool in one agent loop', async ({ + request, +}) => { + const res = await request.post('/api/gemini-server-tools-wire') + expect(res.ok()).toBe(true) + const result = (await res.json()) as { + ok: boolean + error?: string + text: string + tools: Array + } + expect(result.error).toBeUndefined() + expect(result).toMatchObject({ + ok: true, + text: 'Caucus night is Tuesday at the library.', + }) + expect(result.tools).toEqual( + expect.arrayContaining(['lookup_caucus', 'google_search']), + ) +}) From 02ad85fa886c4e1de243e4780278ad1aaf990db5 Mon Sep 17 00:00:00 2001 From: Leon Weber Date: Sat, 24 Oct 2026 18:49:34 +0400 Subject: [PATCH 2/3] fix(ai-gemini): keep built-in tool parts after text and send VALIDATED for auto Record whether text came before each built-in tool part since the last function call, and send the part back after that text, so a step ordered text, toolCall, toolResponse, functionCall replays in the same order. With includeServerSideToolInvocations, Gemini does not support AUTO mode, so toolChoice 'auto' maps to VALIDATED on those requests. Function-only requests keep AUTO, and a functionCallingConfig in modelOptions still wins. --- packages/ai-gemini/src/adapters/text.ts | 65 +++++++++++++------ packages/ai-gemini/src/message-types.ts | 6 +- .../ai-gemini/tests/gemini-adapter.test.ts | 57 ++++++++++++++++ .../ai-gemini/tests/tool-choice-wire.test.ts | 30 +++++++++ 4 files changed, 137 insertions(+), 21 deletions(-) diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index 74dd5d3581..25b113c737 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -897,8 +897,13 @@ export class GeminiTextAdapter< options.tools?.some( (tool) => getGeminiProviderToolKind(tool) === undefined, ) ?? false - const serverParts: Array<{ part: Part; afterCallId?: string }> = [] + const serverParts: Array<{ + part: Part + afterCallId?: string + afterText?: true + }> = [] let lastCallId: string | undefined + let textSinceCall = false const adapterName = this.name const emitGroundingToolCall = function* (): Generator { @@ -990,8 +995,10 @@ export class GeminiTextAdapter< serverParts.push({ part, ...(lastCallId !== undefined && { afterCallId: lastCallId }), + ...(textSinceCall && { afterText: true }), }) } + if (part.text?.trim() && !part.thought) textSinceCall = true if (part.text) { if (part.thought) { @@ -1073,6 +1080,7 @@ export class GeminiTextAdapter< `${functionCall.name}_${Date.now()}_${nextToolIndex}` const functionArgs = functionCall.args || {} lastCallId = toolCallId + textSinceCall = false // Gemini emits thoughtSignature as a Part-level sibling of // functionCall (per @google/genai Part type), not nested inside @@ -1492,9 +1500,10 @@ export class GeminiTextAdapter< } // Built-in tool parts go back where the model put them: after the - // function call they followed, else before the first part that is - // not a thought. Gemini rejects the step when they are missing or - // out of place. The SDK throws on them for Vertex AI. + // function call they followed (or before the first part that is not + // a thought), or after the text that came between. Gemini rejects the + // step when they are missing or out of place. The SDK throws on them + // for Vertex AI. const serverParts = this.client.vertexai ? [] : (msg.toolCalls ?? []).flatMap( @@ -1503,22 +1512,37 @@ export class GeminiTextAdapter< ?.gemini?.serverParts ?? [], ) if (serverParts.length > 0) { - const after = (id: string | undefined) => + const lastText = new Map() + let segment: string | undefined + parts.forEach((part, index) => { + if (part.functionCall) segment = part.functionCall.id + else if (part.text && !part.thought) lastText.set(segment, index) + }) + const after = (id: string | undefined, text: boolean) => serverParts - .filter(({ afterCallId }) => afterCallId === id) + .filter( + ({ afterCallId, afterText }) => + afterCallId === id && + (afterText === true && lastText.has(id)) === text, + ) .map(({ part }) => part) const callIds = new Set( parts.flatMap((part) => part.functionCall?.id ?? []), ) const first = parts.findIndex((part) => !part.thought) - const withServerParts = parts.flatMap((part, index) => [ - ...(index === first ? after(undefined) : []), - part, - ...(part.functionCall?.id !== undefined - ? after(part.functionCall.id) - : []), - ]) - if (first === -1) withServerParts.push(...after(undefined)) + segment = undefined + const withServerParts = parts.flatMap((part, index) => { + if (part.functionCall) segment = part.functionCall.id + return [ + ...(index === first ? after(undefined, false) : []), + part, + ...(part.functionCall?.id !== undefined + ? after(part.functionCall.id, false) + : []), + ...(lastText.get(segment) === index ? after(segment, true) : []), + ] + }) + if (first === -1) withServerParts.push(...after(undefined, false)) // A part whose call is not in this message goes last, not lost. withServerParts.push( ...serverParts @@ -1643,16 +1667,19 @@ export class GeminiTextAdapter< // `functionCallingConfig` wins, and a `retrievalConfig` alone keeps this one. const tools = convertToolsToProviderFormat(options.tools) const hasFunctions = tools.some((tool) => 'functionDeclarations' in tool) - const functionCallingConfig = - options.toolChoice !== undefined && hasFunctions - ? toGeminiFunctionCallingConfig(options.toolChoice) - : undefined // The Gemini API needs this flag to mix built-in tools with function - // declarations. The SDK throws on it for Vertex AI. + // declarations, and rejects AUTO mode with it. The SDK throws on it for + // Vertex AI. const includeServerSideToolInvocations = !this.client.vertexai && hasFunctions && tools.some((tool) => !('functionDeclarations' in tool)) + const functionCallingConfig = + options.toolChoice === 'auto' && includeServerSideToolInvocations + ? { mode: FunctionCallingConfigMode.VALIDATED } + : options.toolChoice !== undefined && hasFunctions + ? toGeminiFunctionCallingConfig(options.toolChoice) + : undefined // Vendor `GenerateContentConfig` fields are `field?: T` (no `| undefined`) // under EOPT, so spread each common option only when present rather than diff --git a/packages/ai-gemini/src/message-types.ts b/packages/ai-gemini/src/message-types.ts index e71cc63f2c..5092bab008 100644 --- a/packages/ai-gemini/src/message-types.ts +++ b/packages/ai-gemini/src/message-types.ts @@ -188,10 +188,12 @@ export interface GeminiToolCallMetadata extends ProviderExecutedToolMetadata { /** * The step's built-in tool parts (`toolCall`, `toolResponse`, * `executableCode`, `codeExecutionResult`), sent back unchanged on the - * next request. `afterCallId` is the function call each one followed. + * next request. `afterCallId` is the function call each one followed, and + * `afterText` is set when text came between that call (or the start) and + * the part. * * @see https://ai.google.dev/gemini-api/docs/generate-content/tool-combination */ - serverParts?: Array<{ part: Part; afterCallId?: string }> + serverParts?: Array<{ part: Part; afterCallId?: string; afterText?: true }> } } diff --git a/packages/ai-gemini/tests/gemini-adapter.test.ts b/packages/ai-gemini/tests/gemini-adapter.test.ts index 8e495e8a8a..b22ac2d81f 100644 --- a/packages/ai-gemini/tests/gemini-adapter.test.ts +++ b/packages/ai-gemini/tests/gemini-adapter.test.ts @@ -906,6 +906,63 @@ describe('GeminiAdapter through AI', () => { ]) }) + const functionCallPart = (id: string) => ({ + functionCall: { id, name: 'lookup_caucus', args: { county: 'Utah' } }, + }) + + it.each([ + { + label: 'at the start', + parts: [ + { text: 'Searching first.' }, + toolCallPart, + toolResponsePart, + functionCallPart('call_1'), + ], + }, + { + label: 'after a function call', + parts: [ + functionCallPart('call_1'), + { text: 'Searching next.' }, + toolCallPart, + toolResponsePart, + functionCallPart('call_2'), + ], + }, + ])( + 'sends built-in tool parts back after text $label', + async ({ parts }) => { + mocks.generateContentStreamSpy + .mockResolvedValueOnce( + createStream([ + { + candidates: [{ content: { parts }, finishReason: 'STOP' }], + }, + ]), + ) + .mockResolvedValueOnce(textStream()) + + for await (const _ of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { apiKey: 'test-key' }, + 'gemini-3.6-flash', + ), + tools: [googleSearchTool(), lookupTool], + messages: [{ role: 'user', content: 'Search and look up.' }], + })) { + // consume + } + + const [secondPayload] = mocks.generateContentStreamSpy.mock.calls[1]! + const modelTurn = secondPayload.contents.find( + (content: any) => content.role === 'model', + ) + expect(modelTurn.parts).toEqual(parts) + }, + ) + it('emits no provider call for a code run without function tools', async () => { mocks.generateContentStreamSpy.mockResolvedValueOnce( createStream([ diff --git a/packages/ai-gemini/tests/tool-choice-wire.test.ts b/packages/ai-gemini/tests/tool-choice-wire.test.ts index 25fb59397c..a33c5b5787 100644 --- a/packages/ai-gemini/tests/tool-choice-wire.test.ts +++ b/packages/ai-gemini/tests/tool-choice-wire.test.ts @@ -115,6 +115,36 @@ describe('Gemini toolConfig.functionCallingConfig', () => { expect(body).not.toHaveProperty('toolConfig') }) + it('sends VALIDATED for auto when built-in tools sit beside function tools', async () => { + const body = await sentBody( + { tools: [googleSearchTool(), weatherTool], toolChoice: 'auto' }, + {}, + ) + expect(body).toHaveProperty('toolConfig', { + functionCallingConfig: { mode: 'VALIDATED' }, + includeServerSideToolInvocations: true, + }) + }) + + it('lets modelOptions override the mode beside built-in tools', async () => { + const body = await sentBody( + { + tools: [googleSearchTool(), weatherTool], + toolChoice: 'auto', + modelOptions: { + toolConfig: { + functionCallingConfig: { mode: FunctionCallingConfigMode.ANY }, + }, + }, + }, + {}, + ) + expect(body).toHaveProperty('toolConfig', { + functionCallingConfig: { mode: 'ANY' }, + includeServerSideToolInvocations: true, + }) + }) + it('lets a functionCallingConfig in modelOptions win', async () => { const body = await sentBody( { From b960a5403ad6b0b6c96561c2fad4f7d5d5f82bc0 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:32:35 +1100 Subject: [PATCH 3/3] fix(ai-gemini): throw when Vertex cannot send built-in tool parts A Vertex follow-up that must send toolCall or toolResponse parts now throws. executableCode and codeExecutionResult parts still go back. --- .changeset/gemini-server-tool-invocations.md | 2 +- docs/adapters/gemini-advanced.md | 4 +- docs/config.json | 2 +- packages/ai-gemini/src/adapters/text.ts | 32 ++-- .../ai-gemini/tests/gemini-adapter.test.ts | 137 ++++++++++++++++++ 5 files changed, 161 insertions(+), 16 deletions(-) diff --git a/.changeset/gemini-server-tool-invocations.md b/.changeset/gemini-server-tool-invocations.md index 785254e4b9..c96edbc832 100644 --- a/.changeset/gemini-server-tool-invocations.md +++ b/.changeset/gemini-server-tool-invocations.md @@ -2,4 +2,4 @@ '@tanstack/ai-gemini': patch --- -Let Gemini 3 use a built-in tool (`googleSearchTool`, `urlContextTool`, `googleMapsTool`, `fileSearchTool`, `codeExecutionTool`) beside function tools in one `chat()` run. On the Gemini API, the adapter now sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs for this mix. A request without it failed with a 400. The adapter also keeps the signed `toolCall`, `toolResponse`, `executableCode` and `codeExecutionResult` parts of a step on its provider-executed tool call (`metadata.gemini.serverParts`) and sends them back in place on the next request. Without them, the next agent-loop step failed with "Function call is missing a thought_signature". Vertex AI does not get the flag, because `@google/genai` rejects it there. +Let Gemini 3 use a built-in tool (`googleSearchTool`, `urlContextTool`, `googleMapsTool`, `fileSearchTool`, `codeExecutionTool`) beside function tools in one `chat()` run. On the Gemini API, the adapter now sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs for this mix. A request without it failed with a 400. The adapter also keeps the signed `toolCall`, `toolResponse`, `executableCode` and `codeExecutionResult` parts of a step on its provider-executed tool call (`metadata.gemini.serverParts`) and sends them back in place on the next request. Without them, the next agent-loop step failed with "Function call is missing a thought_signature". Vertex AI does not get the flag, because `@google/genai` rejects it there. A follow-up that must send `toolCall` or `toolResponse` parts throws. Code execution parts are sent back. diff --git a/docs/adapters/gemini-advanced.md b/docs/adapters/gemini-advanced.md index 49cc4d9c6e..39f45eced6 100644 --- a/docs/adapters/gemini-advanced.md +++ b/docs/adapters/gemini-advanced.md @@ -443,8 +443,8 @@ const stream = chat({ The adapter does two things for this mix: -- **The flag.** On the Gemini API, it sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs to mix the two kinds of tools. The SDK rejects this flag on Vertex AI, so the adapter does not set it there. A `toolConfig` in `modelOptions` wins. -- **The signed parts.** Gemini returns each run of a provider tool as signed parts. The adapter keeps them on a provider-executed tool call, in `metadata.gemini.serverParts`, and sends them back on the next request. If you store messages, keep that metadata. +- **The flag.** On the Gemini API, it sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs to mix the two kinds of tools. The SDK rejects this flag on Vertex AI, so the adapter does not set it there. A `toolConfig` in `modelOptions` wins key by key. +- **The signed parts.** Gemini returns each run of a provider tool as signed parts. The adapter keeps them on a provider-executed tool call, in `metadata.gemini.serverParts`, and sends them back on the next request. If you store messages, keep that metadata. On Vertex AI, a next step that must send `toolCall` or `toolResponse` parts throws. `executableCode` and `codeExecutionResult` parts are sent back. ### `codeExecutionTool` diff --git a/docs/config.json b/docs/config.json index 85494e9dfc..2577af99fb 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1704,7 +1704,7 @@ "label": "Google Gemini: Advanced", "to": "adapters/gemini-advanced", "addedAt": "2026-10-21", - "updatedAt": "2026-10-23" + "updatedAt": "2026-10-25" }, { "label": "Google Vertex AI", diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index 25b113c737..a5d6acf2f6 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -1499,18 +1499,26 @@ export class GeminiTextAdapter< ) } - // Built-in tool parts go back where the model put them: after the - // function call they followed (or before the first part that is not - // a thought), or after the text that came between. Gemini rejects the - // step when they are missing or out of place. The SDK throws on them - // for Vertex AI. - const serverParts = this.client.vertexai - ? [] - : (msg.toolCalls ?? []).flatMap( - (toolCall) => - (toolCall.metadata as GeminiToolCallMetadata | undefined) - ?.gemini?.serverParts ?? [], - ) + // Built-in tool parts go back after the function call they followed, + // or after the text that came between. Gemini rejects a step that + // drops them. Vertex AI rejects toolCall and toolResponse parts, so + // those throw here instead of going out missing. + const storedServerParts = (msg.toolCalls ?? []).flatMap( + (toolCall) => + (toolCall.metadata as GeminiToolCallMetadata | undefined)?.gemini + ?.serverParts ?? [], + ) + if ( + this.client.vertexai && + storedServerParts.some( + ({ part }) => part.toolCall || part.toolResponse, + ) + ) { + throw new Error( + 'Vertex AI cannot send built-in toolCall or toolResponse parts. Use the Gemini API to mix those tools with function tools.', + ) + } + const serverParts = storedServerParts if (serverParts.length > 0) { const lastText = new Map() let segment: string | undefined diff --git a/packages/ai-gemini/tests/gemini-adapter.test.ts b/packages/ai-gemini/tests/gemini-adapter.test.ts index b22ac2d81f..d857fb4621 100644 --- a/packages/ai-gemini/tests/gemini-adapter.test.ts +++ b/packages/ai-gemini/tests/gemini-adapter.test.ts @@ -963,6 +963,56 @@ describe('GeminiAdapter through AI', () => { }, ) + it('sends code execution parts back beside a function tool', async () => { + const codePart = { + executableCode: { code: 'print(2**40)' }, + thoughtSignature: 'sig-code', + } + const resultPart = { + codeExecutionResult: { output: '1099511627776' }, + thoughtSignature: 'sig-result', + } + const callPart = { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + } + mocks.generateContentStreamSpy + .mockResolvedValueOnce( + createStream([ + { + candidates: [ + { + content: { parts: [codePart, callPart, resultPart] }, + finishReason: 'STOP', + }, + ], + }, + ]), + ) + .mockResolvedValueOnce(textStream()) + + for await (const _ of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { apiKey: 'test-key' }, + 'gemini-3.6-flash', + ), + tools: [codeExecutionTool(), lookupTool], + messages: [{ role: 'user', content: 'Compute and look up.' }], + })) { + // consume + } + + const [secondPayload] = mocks.generateContentStreamSpy.mock.calls[1]! + const modelTurn = secondPayload.contents.find( + (content: any) => content.role === 'model', + ) + expect(modelTurn.parts).toEqual([codePart, callPart, resultPart]) + }) + it('emits no provider call for a code run without function tools', async () => { mocks.generateContentStreamSpy.mockResolvedValueOnce( createStream([ @@ -1019,6 +1069,93 @@ describe('GeminiAdapter through AI', () => { const [payload] = mocks.generateContentStreamSpy.mock.calls[0]! expect(payload.config.toolConfig).toBeUndefined() }) + + it('throws on Vertex AI when the next step must send toolCall parts', async () => { + mocks.generateContentStreamSpy.mockResolvedValueOnce( + createStream([ + { + candidates: [ + { + content: { + parts: [ + toolCallPart, + { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + }, + toolResponsePart, + ], + }, + finishReason: 'STOP', + }, + ], + }, + ]), + ) + + await expect(async () => { + for await (const _ of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { vertexai: true, project: 'p', location: 'us-central1' }, + 'gemini-3.6-flash', + ), + tools: [googleSearchTool(), lookupTool], + messages: [{ role: 'user', content: 'Search and look up.' }], + })) { + // consume + } + }).rejects.toThrow(/toolCall or toolResponse/) + expect(mocks.generateContentStreamSpy).toHaveBeenCalledTimes(1) + }) + + it('sends code execution parts back on Vertex AI', async () => { + const codePart = { executableCode: { code: 'print(1)' } } + const resultPart = { codeExecutionResult: { output: '1' } } + const callPart = { + functionCall: { + id: 'call_1', + name: 'lookup_caucus', + args: { county: 'Salt Lake' }, + }, + } + mocks.generateContentStreamSpy + .mockResolvedValueOnce( + createStream([ + { + candidates: [ + { + content: { parts: [codePart, callPart, resultPart] }, + finishReason: 'STOP', + }, + ], + }, + ]), + ) + .mockResolvedValueOnce(textStream()) + + for await (const _ of chat({ + threadId: 'thread-1', + adapter: new GeminiTextAdapter( + { vertexai: true, project: 'p', location: 'us-central1' }, + 'gemini-3.6-flash', + ), + tools: [codeExecutionTool(), lookupTool], + messages: [{ role: 'user', content: 'Compute and look up.' }], + })) { + // consume + } + + const [secondPayload] = mocks.generateContentStreamSpy.mock.calls[1]! + const modelTurn = secondPayload.contents.find( + (content: any) => content.role === 'model', + ) + expect(modelTurn.parts).toEqual([codePart, callPart, resultPart]) + expect(secondPayload.config.toolConfig).toBeUndefined() + }) }) it('emits parentMessageId on tool-first tool calls matching the assistant message id', async () => {