fix(ai-gemini): run built-in tools beside function tools on Gemini 3

.changeset/gemini-server-tool-invocations.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai-gemini': patch
+---
+
+Let Gemini 3 use a built-in tool (`googleSearchTool`, `urlContextTool`, `googleMapsTool`, `fileSearchTool`, `codeExecutionTool`) beside function tools in one `chat()` run. On the Gemini API, the adapter now sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs for this mix. A request without it failed with a 400. The adapter also keeps the signed `toolCall`, `toolResponse`, `executableCode` and `codeExecutionResult` parts of a step on its provider-executed tool call (`metadata.gemini.serverParts`) and sends them back in place on the next request. Without them, the next agent-loop step failed with "Function call is missing a thought_signature". Vertex AI does not get the flag, because `@google/genai` rejects it there. A follow-up that must send `toolCall` or `toolResponse` parts throws. Code execution parts are sent back.

docs/adapters/gemini-advanced.md

@@ -415,6 +415,37 @@ Gemini has tools of its own, next to the function tools that you define. Import
 
 For the concept, a comparison table, and type gating, see [Provider Tools](../tools/provider-tools.md).
 
+### Provider tools beside your own tools
+
+On Gemini 3, put provider tools and your function tools in one `tools` array. The model can search the web and call your tool in the same step.
+
+```typescript
+import { chat, toolDefinition } from "@tanstack/ai";
+import { geminiText } from "@tanstack/ai-gemini";
+import { googleSearchTool } from "@tanstack/ai-gemini/tools";
+import { z } from "zod";
+
+const getOrder = toolDefinition({
+  name: "get_order",
+  description: "Get an order by its id",
+  inputSchema: z.object({ id: z.string() }),
+}).server(async ({ id }) => ({ id, status: "shipped" }));
+
+const stream = chat({
+  adapter: geminiText("gemini-3.8-flash"),
+  messages: [
+    { role: "user", content: "Where is order 42, and does the carrier report delays today?" },
+  ],
+  threadId: "thread-1",
+  tools: [googleSearchTool(), getOrder],
+});
+```
+
+The adapter does two things for this mix:
+
+- **The flag.** On the Gemini API, it sets `toolConfig.includeServerSideToolInvocations`, which Gemini needs to mix the two kinds of tools. The SDK rejects this flag on Vertex AI, so the adapter does not set it there. A `toolConfig` in `modelOptions` wins key by key.
+- **The signed parts.** Gemini returns each run of a provider tool as signed parts. The adapter keeps them on a provider-executed tool call, in `metadata.gemini.serverParts`, and sends them back on the next request. If you store messages, keep that metadata. On Vertex AI, a next step that must send `toolCall` or `toolResponse` parts throws. `executableCode` and `codeExecutionResult` parts are sent back.
+
 ### `codeExecutionTool`
 
 Gemini runs Python code in a sandbox and returns the results inline. It takes no arguments. Add it to the `tools` array.

docs/config.json

@@ -1703,7 +1703,8 @@
         {
           "label": "Google Gemini: Advanced",
           "to": "adapters/gemini-advanced",
-          "addedAt": "2026-10-21"
+          "addedAt": "2026-10-21",
+          "updatedAt": "2026-10-25"
         },
         {
           "label": "Google Vertex AI",

packages/ai-gemini/src/adapters/text.ts

@@ -15,6 +15,7 @@ import {
 } from '@tanstack/ai/adapter-internals'
 import { BaseTextAdapter } from '@tanstack/ai/adapters'
 import { convertToolsToProviderFormat } from '../tools/tool-converter'
+import { getGeminiProviderToolKind } from '../tools/gemini-provider-tool'
 import { buildGeminiUsage } from '../usage'
 import { geminiThinkingConfig } from '../text/reasoning'
 import {
@@ -142,6 +143,24 @@ function getGroundingSources(
   return [...sources.values()]
 }
 
+/** A part of a built-in tool run, which Gemini needs back unchanged. */
+function isServerPart(part: Part): boolean {
+  return Boolean(
+    part.toolCall ||
+    part.toolResponse ||
+    part.executableCode ||
+    part.codeExecutionResult,
+  )
+}
+
+/** The tool name of a built-in tool part, when the part names its tool. */
+function serverToolName(part: Part): string | undefined {
+  if (part.executableCode || part.codeExecutionResult) return 'code_execution'
+  const type = (part.toolCall ?? part.toolResponse)?.toolType
+  if (!type || type === 'TOOL_TYPE_UNSPECIFIED') return undefined
+  return type.startsWith('GOOGLE_SEARCH') ? 'google_search' : type.toLowerCase()
+}
+
 /** Maps `chat({ toolChoice })` to the Gemini function calling config. */
 function toGeminiFunctionCallingConfig(
   choice: ToolChoice,
@@ -872,28 +891,55 @@ export class GeminiTextAdapter<
     let hasEmittedTextMessageStart = false
     let groundingMetadata: GroundingMetadata | undefined
     let groundingCallEmitted = false
+    // Gemini needs a step's built-in tool parts back only when the step can
+    // also call a function tool.
+    const keepServerParts =
+      options.tools?.some(
+        (tool) => getGeminiProviderToolKind(tool) === undefined,
+      ) ?? false
+    const serverParts: Array<{
+      part: Part
+      afterCallId?: string
+      afterText?: true
+    }> = []
+    let lastCallId: string | undefined
+    let textSinceCall = false
     const adapterName = this.name
 
     const emitGroundingToolCall = function* (): Generator<AdapterYieldChunk> {
-      if (!groundingMetadata || groundingCallEmitted) return
-      const sources = getGroundingSources(groundingMetadata)
+      if (groundingCallEmitted) return
+      const sources = groundingMetadata
+        ? getGroundingSources(groundingMetadata)
+        : []
       const hasGoogleSearchEvidence =
         sources.length > 0 ||
-        (groundingMetadata.webSearchQueries?.length ?? 0) > 0 ||
-        groundingMetadata.searchEntryPoint !== undefined
-      if (!hasGoogleSearchEvidence) return
+        (groundingMetadata?.webSearchQueries?.length ?? 0) > 0 ||
+        groundingMetadata?.searchEntryPoint !== undefined
+      if (!hasGoogleSearchEvidence && serverParts.length === 0) return
 
       groundingCallEmitted = true
       const toolCallId = generateId(adapterName)
+      const toolCallName =
+        (!hasGoogleSearchEvidence &&
+          (serverParts
+            .map(({ part }) => serverToolName(part))
+            .find((name) => name !== undefined) ??
+            options.tools?.find(
+              (tool) => getGeminiProviderToolKind(tool) !== undefined,
+            )?.name)) ||
+        'google_search'
       const metadata: GeminiToolCallMetadata = {
         providerExecuted: true,
         sources,
-        gemini: { groundingMetadata },
+        gemini: {
+          ...(groundingMetadata && { groundingMetadata }),
+          ...(serverParts.length > 0 && { serverParts }),
+        },
       }
       yield {
         type: EventType.TOOL_CALL_START,
         toolCallId,
-        toolCallName: 'google_search',
+        toolCallName,
         parentMessageId: messageId,
         model,
         timestamp: Date.now(),
@@ -903,11 +949,11 @@ export class GeminiTextAdapter<
       yield {
         type: EventType.TOOL_CALL_END,
         toolCallId,
-        toolCallName: 'google_search',
+        toolCallName,
         model,
         timestamp: Date.now(),
         input: {
-          ...(groundingMetadata.webSearchQueries && {
+          ...(groundingMetadata?.webSearchQueries && {
             queries: groundingMetadata.webSearchQueries,
           }),
         },
@@ -945,6 +991,15 @@ export class GeminiTextAdapter<
         const parts = chunk.candidates[0].content.parts
 
         for (const part of parts) {
+          if (keepServerParts && isServerPart(part)) {
+            serverParts.push({
+              part,
+              ...(lastCallId !== undefined && { afterCallId: lastCallId }),
+              ...(textSinceCall && { afterText: true }),
+            })
+          }
+          if (part.text?.trim() && !part.thought) textSinceCall = true
+
           if (part.text) {
             if (part.thought) {
               // Emit REASONING start events on first thinking content
@@ -1024,6 +1079,8 @@ export class GeminiTextAdapter<
               functionCall.id ||
               `${functionCall.name}_${Date.now()}_${nextToolIndex}`
             const functionArgs = functionCall.args || {}
+            lastCallId = toolCallId
+            textSinceCall = false
 
             // Gemini emits thoughtSignature as a Part-level sibling of
             // functionCall (per @google/genai Part type), not nested inside
@@ -1441,6 +1498,70 @@ export class GeminiTextAdapter<
               .map(thoughtPart),
           )
         }
+
+        // Built-in tool parts go back after the function call they followed,
+        // or after the text that came between. Gemini rejects a step that
+        // drops them. Vertex AI rejects toolCall and toolResponse parts, so
+        // those throw here instead of going out missing.
+        const storedServerParts = (msg.toolCalls ?? []).flatMap(
+          (toolCall) =>
+            (toolCall.metadata as GeminiToolCallMetadata | undefined)?.gemini
+              ?.serverParts ?? [],
+        )
+        if (
+          this.client.vertexai &&
+          storedServerParts.some(
+            ({ part }) => part.toolCall || part.toolResponse,
+          )
+        ) {
+          throw new Error(
+            'Vertex AI cannot send built-in toolCall or toolResponse parts. Use the Gemini API to mix those tools with function tools.',
+          )
+        }
+        const serverParts = storedServerParts
+        if (serverParts.length > 0) {
+          const lastText = new Map<string | undefined, number>()
+          let segment: string | undefined
+          parts.forEach((part, index) => {
+            if (part.functionCall) segment = part.functionCall.id
+            else if (part.text && !part.thought) lastText.set(segment, index)
+          })
+          const after = (id: string | undefined, text: boolean) =>
+            serverParts
+              .filter(
+                ({ afterCallId, afterText }) =>
+                  afterCallId === id &&
+                  (afterText === true && lastText.has(id)) === text,
+              )
+              .map(({ part }) => part)
+          const callIds = new Set(
+            parts.flatMap((part) => part.functionCall?.id ?? []),
+          )
+          const first = parts.findIndex((part) => !part.thought)
+          segment = undefined
+          const withServerParts = parts.flatMap((part, index) => {
+            if (part.functionCall) segment = part.functionCall.id
+            return [
+              ...(index === first ? after(undefined, false) : []),
+              part,
+              ...(part.functionCall?.id !== undefined
+                ? after(part.functionCall.id, false)
+                : []),
+              ...(lastText.get(segment) === index ? after(segment, true) : []),
+            ]
+          })
+          if (first === -1) withServerParts.push(...after(undefined, false))
+          // A part whose call is not in this message goes last, not lost.
+          withServerParts.push(
+            ...serverParts
+              .filter(
+                ({ afterCallId }) =>
+                  afterCallId !== undefined && !callIds.has(afterCallId),
+              )
+              .map(({ part }) => part),
+          )
+          parts.splice(0, parts.length, ...withServerParts)
+        }
       }
       return {
         role,
@@ -1553,11 +1674,20 @@ export class GeminiTextAdapter<
     // them. A `toolConfig` in modelOptions wins key by key, so its own
     // `functionCallingConfig` wins, and a `retrievalConfig` alone keeps this one.
     const tools = convertToolsToProviderFormat(options.tools)
+    const hasFunctions = tools.some((tool) => 'functionDeclarations' in tool)
+    // The Gemini API needs this flag to mix built-in tools with function
+    // declarations, and rejects AUTO mode with it. The SDK throws on it for
+    // Vertex AI.
+    const includeServerSideToolInvocations =
+      !this.client.vertexai &&
+      hasFunctions &&
+      tools.some((tool) => !('functionDeclarations' in tool))
     const functionCallingConfig =
-      options.toolChoice !== undefined &&
-      tools.some((tool) => 'functionDeclarations' in tool)
-        ? toGeminiFunctionCallingConfig(options.toolChoice)
-        : undefined
+      options.toolChoice === 'auto' && includeServerSideToolInvocations
+        ? { mode: FunctionCallingConfigMode.VALIDATED }
+        : options.toolChoice !== undefined && hasFunctions
+          ? toGeminiFunctionCallingConfig(options.toolChoice)
+          : undefined
 
     // Vendor `GenerateContentConfig` fields are `field?: T` (no `| undefined`)
     // under EOPT, so spread each common option only when present rather than
@@ -1573,9 +1703,15 @@ export class GeminiTextAdapter<
       ),
       config: {
         ...modelOpts,
-        ...(functionCallingConfig !== undefined && {
+        ...((functionCallingConfig !== undefined ||
+          includeServerSideToolInvocations) && {
           toolConfig: {
-            functionCallingConfig,
+            ...(functionCallingConfig !== undefined && {
+              functionCallingConfig,
+            }),
+            ...(includeServerSideToolInvocations && {
+              includeServerSideToolInvocations,
+            }),
             ...options.modelOptions?.toolConfig,
           },
         }),

packages/ai-gemini/src/message-types.ts

@@ -1,5 +1,5 @@
 import type { ProviderExecutedToolMetadata } from '@tanstack/ai'
-import type { GroundingMetadata } from '@google/genai'
+import type { GroundingMetadata, Part } from '@google/genai'
 
 /**
  * Gemini-specific metadata types for multimodal content parts.
@@ -184,6 +184,16 @@ export interface GeminiMessageMetadataByModality {
 export interface GeminiToolCallMetadata extends ProviderExecutedToolMetadata {
   thoughtSignature?: string
   gemini?: {
-    groundingMetadata: GroundingMetadata
+    groundingMetadata?: GroundingMetadata
+    /**
+     * The step's built-in tool parts (`toolCall`, `toolResponse`,
+     * `executableCode`, `codeExecutionResult`), sent back unchanged on the
+     * next request. `afterCallId` is the function call each one followed, and
+     * `afterText` is set when text came between that call (or the start) and
+     * the part.
+     *
+     * @see https://ai.google.dev/gemini-api/docs/generate-content/tool-combination
+     */
+    serverParts?: Array<{ part: Part; afterCallId?: string; afterText?: true }>
   }
 }