From 21c3025d72cffbb5dc391a9d194200038e6898d9 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 9 Jul 2026 13:19:55 +0200 Subject: [PATCH 1/6] feat(server-utils): Add orchestrion support for Vercel AI v4 Extends the orchestrion diagnostics-channel instrumentation to cover the `ai` SDK v4 (`>=4.0.0 <5.0.0`), so the channel path now covers v4/v5/v6/v7 and produces spans identical to the OTel processor. v4 differs from v5/v6 in three load-bearing ways, all handled by reusing the existing shared core: - No `resolveLanguageModel` chokepoint: v4 calls the passed `LanguageModelV1` (`specificationVersion: 'v1'`) directly, so its `doGenerate`/`doStream` are patched at the operation start instead, gated on `'v1'` so v5/v6 models are untouched. Tool calls reuse the existing per-tool `execute` patch, and the model-call tool list is read from v4's `mode.tools`. - Token names: the shared core now reads `promptTokens`/`completionTokens` (v4) as well as `inputTokens`/`outputTokens` (v5+); the fallbacks are inert on v5+. - Streamed text delta: `accumulateChunk` reads `textDelta` (v4) as well as `delta` (v5+). The top-level `system` prompt is also lifted into `instructions` for all of v4/v5/v6, matching v7's native channel. The v4 integration-test suites now run under orchestrion as well as the OTel processor, plus a new `streamText` scenario covering the v4 `doStream` path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tracing/vercelai/scenario-stream-text.mjs | 36 +++++ .../vercelai/span-streaming-v4/test.ts | 51 ++++--- .../suites/tracing/vercelai/test.ts | 132 ++++++++++++++---- .../integrations/tracing-channel/vercel-ai.ts | 2 +- .../src/orchestrion/config/vercel-ai.ts | 29 ++-- packages/server-utils/src/vercel-ai/util.ts | 24 +++- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 11 +- .../vercel-ai-orchestrion-subscriber.ts | 62 +++++--- 8 files changed, 255 insertions(+), 92 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs new file mode 100644 index 000000000000..85b92622a2b8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs @@ -0,0 +1,36 @@ +import * as Sentry from '@sentry/node'; +import { streamText } from 'ai'; +import { convertArrayToReadableStream, MockLanguageModelV1 } from 'ai/test'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const result = streamText({ + experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true }, + model: new MockLanguageModelV1({ + doStream: async () => ({ + stream: convertArrayToReadableStream([ + { type: 'response-metadata', id: 'resp-1', modelId: 'mock-model-id' }, + { type: 'text-delta', textDelta: 'Stream ' }, + { type: 'text-delta', textDelta: 'response!' }, + { + type: 'finish', + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + }, + ]), + rawCall: { rawPrompt: null, rawSettings: {} }, + }), + }), + prompt: 'Stream me a response', + }); + + // streamText returns synchronously; drive the lazy stream to completion so the spans finish. + for await (const _part of result.textStream) { + void _part; + } + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts index b6649fb6b029..a5df3a60c6e7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -30,7 +30,18 @@ function attr(value: unknown) { return expect.objectContaining({ value }); } -describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', () => { +// In orchestrion mode the `ai` SDK is instrumented via the diagnostics-channel subscriber +// (`auto.vercelai.channel`); otherwise via the OTel span processor (`auto.vercelai.otel`). The +// `vercel.ai.*` pipeline attributes are OTel-processor-only, so they aren't asserted here. +const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; + +// v4's OTel path serializes tool-call arguments from the provider's raw JSON string (whitespace +// preserved); the channel path serializes the SDK-parsed args object (compact). Same data, different spacing. +const toolCallArgs = isOrchestrionEnabled() + ? '{\\"location\\":\\"San Francisco\\"}' + : '{ \\"location\\": \\"San Francisco\\" }'; + +describe('Vercel AI integration (streaming v4)', () => { afterAll(() => { cleanupChildProcesses(); }); @@ -49,9 +60,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), - 'vercel.ai.pipeline.name': attr('generateText'), - 'vercel.ai.streaming': attr(false), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Second span - generate_content for simple generateText @@ -66,9 +75,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), - 'vercel.ai.pipeline.name': attr('generateText.doGenerate'), - 'vercel.ai.streaming': attr(false), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Third span - invoke_agent for explicit telemetry generateText @@ -82,7 +89,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Fourth span - tool call invoke_agent @@ -96,7 +103,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Fifth span - tool call generate_content @@ -110,7 +117,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Sixth span - execute_tool @@ -124,7 +131,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), ]), @@ -149,9 +156,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), - 'vercel.ai.pipeline.name': attr('generateText'), - 'vercel.ai.streaming': attr(false), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Second span - generate_content with input/output messages @@ -170,7 +175,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Third span - explicit telemetry invoke_agent with messages @@ -188,7 +193,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Fourth span - tool call invoke_agent with messages @@ -200,7 +205,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', '[{"role":"user","content":"What is the weather in San Francisco?"}]', ), [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]', + `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, ), [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), @@ -208,7 +213,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Fifth span - tool call generate_content with available_tools @@ -225,7 +230,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), // Sixth span - execute_tool with description and input/output @@ -241,7 +246,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), ]), @@ -255,7 +260,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', attributes: expect.objectContaining({ [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), expect.objectContaining({ @@ -268,7 +273,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), expect.objectContaining({ @@ -280,7 +285,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin), }), }), ]), diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index e96c9dcf477a..ea555eb3e5af 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -24,7 +24,12 @@ import { import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; -describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { +// In orchestrion mode the `ai` SDK is instrumented via the diagnostics-channel subscriber +// (`auto.vercelai.channel`); otherwise via the OTel span processor (`auto.vercelai.otel`). +const orchestrion = isOrchestrionEnabled(); +const expectedOrigin = orchestrion ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; + +describe('Vercel AI integration (v4)', () => { afterAll(() => { cleanupChildProcesses(); }); @@ -265,17 +270,16 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let traceId: string = 'unset-trace-id'; - let spanId: string = 'unset-span-id'; + let transactionEvent: Event | undefined; + let errorEvent: Event | undefined; await createRunner() + // In orchestrion mode the tool error is captured mid-transaction, so the error and + // transaction/span envelopes can arrive in either order — assert content, not wire order. + .unordered() .expect({ transaction: transaction => { - expect(transaction.transaction).toBe('main'); - // gen_ai spans should be empty in transaction - expect(transaction.spans).toEqual([]); - traceId = transaction.contexts!.trace!.trace_id; - spanId = transaction.contexts!.trace!.span_id; + transactionEvent = transaction; }, }) .expect({ @@ -302,26 +306,45 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { expect(toolSpan!.name).toBe('execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); }, }) .expect({ event: event => { - expect(event.exception?.values).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - ); - expect(event.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(event.contexts!.trace!.trace_id).toBe(traceId); - expect(event.contexts!.trace!.span_id).toBe(spanId); + errorEvent = event; }, }) .start() .completed(); + + expect(transactionEvent).toBeDefined(); + expect(transactionEvent!.transaction).toBe('main'); + + expect(errorEvent).toBeDefined(); + expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); + // Trace id is shared between the transaction and the tool error. + expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); + + if (orchestrion) { + // The channel subscriber captures the raw tool error and tags it with the tool identity. + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toMatchObject({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); + } else { + // The OTel processor surfaces the SDK's wrapped `AI_ToolExecutionError`. + expect(errorEvent!.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); + } }); }); @@ -331,6 +354,8 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { let errorEvent: Event | undefined; const runner = createRunner() + // In orchestrion mode the tool error is captured mid-request, so envelopes can arrive in any order. + .unordered() .expect({ transaction: transaction => { transactionEvent = transaction; @@ -377,21 +402,37 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent).toBeDefined(); - expect(errorEvent!.exception?.values).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - ); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); - expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); + + if (orchestrion) { + // The channel subscriber captures the raw tool error and tags it with the tool identity. + expect(errorEvent!.tags).toMatchObject({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); + } else { + // The OTel processor surfaces the SDK's wrapped `AI_ToolExecutionError`. + expect(errorEvent!.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-late-model-id.mjs', 'instrument.mjs', (createRunner, test) => { + // The late-model-id span-naming behaviour (e.g. `generateText.doGenerate`) is specific to the OTel + // span processor. The channel subscriber names the model-call span from the model id captured at + // call time, so there is no equivalent assertion in orchestrion mode. + if (orchestrion) { + return; + } test('sets op correctly even when model ID is not available at span start', async () => { await createRunner() .expect({ transaction: { transaction: 'main' } }) @@ -641,4 +682,39 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => { }); }, ); + + createEsmAndCjsTests(__dirname, 'scenario-stream-text.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates ai spans for streamText (doStream)', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); + expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the + // subscriber normalizes to input/output token attributes. + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); + expect(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('Stream response!'); + + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); + }, + }) + .start() + .completed(); + }); + }); }); diff --git a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts index 17a8acac7a30..72ef44bcbd86 100644 --- a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts +++ b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts @@ -35,6 +35,6 @@ const _vercelAiChannelIntegration = ((options: VercelAiOptions = {}) => { /** * Auto-instrument the `ai` SDK. Supported are: * - v7 via native `ai:telemetry` tracing channel - * - v5 & v6 via orchestrion `orchestrion:ai:*` channels + * - v4, v5 & v6 via orchestrion `orchestrion:ai:*` channels */ export const vercelAiChannelIntegration = defineIntegration(_vercelAiChannelIntegration); diff --git a/packages/server-utils/src/orchestrion/config/vercel-ai.ts b/packages/server-utils/src/orchestrion/config/vercel-ai.ts index d2be04f4984f..4f9553df9d6a 100644 --- a/packages/server-utils/src/orchestrion/config/vercel-ai.ts +++ b/packages/server-utils/src/orchestrion/config/vercel-ai.ts @@ -9,31 +9,32 @@ export const vercelAiConfig = [ // `streamText` returns its result synchronously (streaming is lazy), so it's // `Sync`; the subscriber binds the span via `bindTracingChannelToSpan`, which // ends it when the (synchronous) call returns. - // Vercel AI v5: same top-level entry points as v6 - ...vercelAiEntries('>=5.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), - ...vercelAiEntries('>=5.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), - ...vercelAiEntries('>=5.0.0 <7.0.0', 'embed', 'embed', 'Async'), - ...vercelAiEntries('>=5.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + + // The majority of entrypoints are present in all versions we support + ...vercelAiEntries('>=4.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), + ...vercelAiEntries('>=4.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=4.0.0 <7.0.0', 'embed', 'embed', 'Async'), + ...vercelAiEntries('>=4.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + + // The following entry is only present in v5 and later ...vercelAiEntries('>=5.0.0 <7.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), - // This only exists in >= v6 + + // The following entry is only present in v6 and later ...vercelAiEntries('>=6.0.0 <7.0.0', 'executeToolCall', 'executeToolCall', 'Async'), ] satisfies InstrumentationConfig[]; export const vercelAiChannels = { - // Vercel AI (`ai`) v5 & v6: orchestrion injects these so the same channel-based + // Vercel AI (`ai`): orchestrion injects these so the same channel-based // integration that consumes `ai`'s native `ai:telemetry` channel (v7) can - // also instrument v5/v6. Each maps to a top-level function in `ai`'s bundle. - // v5 and v6 share the same channel names (the subscriber is version-agnostic); - // `VERCEL_AI_EXECUTE_TOOL_CALL` is v6-only (v5 has no `executeToolCall` export). + // also instrument v4/v5/v6. Each maps to a top-level function in `ai`'s bundle. + // All three versions share the same channel names (the subscriber is version-agnostic); + // `VERCEL_AI_EXECUTE_TOOL_CALL` is v6-only (v4/v5 have no `executeToolCall` export) and + // `VERCEL_AI_RESOLVE_LANGUAGE_MODEL` is v5/v6-only (v4 has no such chokepoint). VERCEL_AI_GENERATE_TEXT: 'orchestrion:ai:generateText', VERCEL_AI_STREAM_TEXT: 'orchestrion:ai:streamText', VERCEL_AI_EMBED: 'orchestrion:ai:embed', VERCEL_AI_EMBED_MANY: 'orchestrion:ai:embedMany', VERCEL_AI_EXECUTE_TOOL_CALL: 'orchestrion:ai:executeToolCall', - // `resolveLanguageModel` is the single chokepoint every model call flows - // through; we wrap it to monkey-patch `doGenerate`/`doStream` on the returned - // model (the model-call site itself is an inline call with no injectable - // definition). VERCEL_AI_RESOLVE_LANGUAGE_MODEL: 'orchestrion:ai:resolveLanguageModel', } as const; diff --git a/packages/server-utils/src/vercel-ai/util.ts b/packages/server-utils/src/vercel-ai/util.ts index dee10ecac46e..88edb7b5c6a6 100644 --- a/packages/server-utils/src/vercel-ai/util.ts +++ b/packages/server-utils/src/vercel-ai/util.ts @@ -49,6 +49,8 @@ export function safeStringify(value: unknown): string { interface StreamChunk { type?: unknown; delta?: unknown; + // v4 text-delta chunks carry the text on `textDelta`; v5+ uses `delta`. + textDelta?: unknown; id?: unknown; modelId?: unknown; toolCallId?: unknown; @@ -154,12 +156,26 @@ function accumulateChunk(state: StreamedModelCallResult, chunk: unknown): string if (!isObjectLike(chunk)) { return undefined; } - const { type, delta, id, modelId, toolCallId, toolName, input, args, finishReason, usage, providerMetadata } = - chunk as StreamChunk; + const { + type, + delta, + textDelta, + id, + modelId, + toolCallId, + toolName, + input, + args, + finishReason, + usage, + providerMetadata, + } = chunk as StreamChunk; switch (type) { - case 'text-delta': - return typeof delta === 'string' ? delta : undefined; + case 'text-delta': { + const textChunk = delta ?? textDelta; + return typeof textChunk === 'string' ? textChunk : undefined; + } case 'tool-call': state.toolCalls.push({ toolCallId, toolName, input: input ?? args }); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index 1240f44fec61..dd7fd514bef3 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -323,8 +323,8 @@ function enrichInvokeAgentFromStream( const usage = isObjectLike(final.usage) ? final.usage : undefined; if (usage) { - const input = tokenCount(usage.inputTokens) ?? tokenCount(usage.tokens); - const output = tokenCount(usage.outputTokens); + const input = tokenCount(usage.inputTokens) ?? tokenCount(usage.promptTokens) ?? tokenCount(usage.tokens); + const output = tokenCount(usage.outputTokens) ?? tokenCount(usage.completionTokens); addTokensToSpan(span, GEN_AI_USAGE_INPUT_TOKENS, input); addTokensToSpan(span, GEN_AI_USAGE_OUTPUT_TOKENS, output); addTokensToSpan(span, GEN_AI_USAGE_TOTAL_TOKENS, tokenCount(usage.totalTokens) ?? sum(input, output)); @@ -521,11 +521,12 @@ export function enrichSpanOnEnd( } // `languageModelCall` results report usage as `{ total }` objects; top-level/step results report - // flat numbers. `tokenCount` handles both. + // flat numbers. `tokenCount` handles both. v4 names tokens `promptTokens`/`completionTokens` + // (v5+ uses `inputTokens`/`outputTokens`); the fallbacks are `undefined`-inert on v5+. const usage = isObjectLike(result.usage) ? result.usage : undefined; if (usage) { - const inputTokens = tokenCount(usage.inputTokens) ?? tokenCount(usage.tokens); - const outputTokens = tokenCount(usage.outputTokens); + const inputTokens = tokenCount(usage.inputTokens) ?? tokenCount(usage.promptTokens) ?? tokenCount(usage.tokens); + const outputTokens = tokenCount(usage.outputTokens) ?? tokenCount(usage.completionTokens); const totalTokens = tokenCount(usage.totalTokens) ?? sum(inputTokens, outputTokens); if (inputTokens !== undefined) { span.setAttribute(GEN_AI_USAGE_INPUT_TOKENS, inputTokens); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 95e0cd76cb4d..76a42bae7178 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -18,30 +18,34 @@ import { import { asString, isReadableStream, tapModelCallStream } from './util'; /** - * v5 & v6 channel adapter for the Vercel AI (`ai`) SDK. + * v4, v5 & v6 channel adapter for the Vercel AI (`ai`) SDK. * * `ai` >= 7 publishes a normalized `ai:telemetry` tracing channel natively - * (consumed by `subscribeVercelAiTracingChannel`). v5/v6 have no such channel, so + * (consumed by `subscribeVercelAiTracingChannel`). v4/v5/v6 have no such channel, so * orchestrion injects `orchestrion:ai:*` channels around the top-level * functions (see `orchestrion/config/index.ts`). The injected channels carry only the * wrapped call's `{ arguments, result, error }` — NOT v7's normalized `event` * object — so this adapter reconstructs an equivalent {@link VercelAiChannelMessage} - * from v5/v6's argument/result shapes and delegates to the SAME span-building core + * from each version's argument/result shapes and delegates to the SAME span-building core * (`createSpanFromMessage` / `enrichSpanOnEnd`) the v7 subscriber uses, so the - * emitted spans are identical between v5, v6 and v7. + * emitted spans are identical between v4, v5, v6 and v7. The shared core reads both v4-style + * (`promptTokens`/`completionTokens`) and v5+-style (`inputTokens`/`outputTokens`) token names. * * The model call (`languageModelCall` / `generate_content` span) has no - * injectable definition in `ai`, so we instead wrap `resolveLanguageModel` (the - * single chokepoint every model call flows through, present in both v5 and v6) - * and monkey-patch `doGenerate`/`doStream` on the returned model. + * injectable definition in `ai`. On v5/v6 we wrap `resolveLanguageModel` (the + * single chokepoint every model call flows through) and monkey-patch + * `doGenerate`/`doStream` on the returned model. v4 has no `resolveLanguageModel` + * — the passed `LanguageModelV1` (`specificationVersion: 'v1'`) is called directly — + * so we patch its `doGenerate`/`doStream` at the operation start instead (see + * `buildOperationSpan`), gated on `'v1'` so it never touches v5/v6 models. * * Tool-call spans differ by version: v6 exposes a per-call `executeToolCall` - * function orchestrion wraps into its own channel. v5 has no such export (only a + * function orchestrion wraps into its own channel. v4/v5 have no such export (only a * batch `executeTools`), so instead we monkey-patch each tool's `execute` from * the operation's `tools`. On v6 that patch is inert: `executeToolCall` runs * `execute` inside its own tool-call span (the active async context), so the - * patch sees that span as its parent and skips — only v5 (where the parent is - * the enclosing `invoke_agent` span) emits the patched span, so v6 never + * patch sees that span as its parent and skips — only v4/v5 (where the parent is + * the enclosing `invoke_agent` span) emit the patched span, so v6 never * double-counts tool spans. */ @@ -234,6 +238,15 @@ function bindOperation( if (isObjectLike(callOptions.tools)) { patchOperationTools(callOptions.tools, options); } + // v4 has no `resolveLanguageModel` chokepoint — the passed `LanguageModelV1` + // (`specificationVersion: 'v1'`) is called directly — so patch its `doGenerate`/`doStream` + // here, at the operation start, instead. v5/v6 models are `'v2'` and are patched via + // `resolveLanguageModel`, so restricting to `'v1'` keeps this strictly additive and avoids + // double-patching. Embedding models are also `'v1'` but expose no `doGenerate`/`doStream`, so + // the patch is a no-op for `embed`/`embedMany`. + if (isRecord(callOptions.model) && callOptions.model.specificationVersion === 'v1') { + patchModelMethods(callOptions.model as PatchableModel, options); + } } return span; }; @@ -353,15 +366,10 @@ function subscribeResolveLanguageModel( if (!isObjectLike(ctx.result)) { return; } - const model = ctx.result as PatchableModel; // Patch the model's `doGenerate`/`doStream` once. The model call recovers its parent from the // active async context at call time (the operation span `bindTracingChannelToSpan` bound), which // propagates into the model call for `streamText` too, so there is nothing to capture on the model here. - if (!model[PATCHED]) { - model[PATCHED] = true; - patchModelMethod(model, 'doGenerate', options); - patchModelMethod(model, 'doStream', options); - } + patchModelMethods(ctx.result as PatchableModel, options); }, start() { /* no-op */ @@ -400,6 +408,20 @@ function resolveModelCallParent(): Span | undefined { return active && operationSpans.has(active) ? active : undefined; } +/** + * Idempotently patch a resolved model's `doGenerate`/`doStream` so each invocation emits a + * `languageModelCall` span. Shared by the v5/v6 `resolveLanguageModel` path and the v4 path (which + * patches the passed model at the operation start, as v4 has no `resolveLanguageModel`). + */ +function patchModelMethods(model: PatchableModel, options: VercelAiChannelOptions): void { + if (model[PATCHED]) { + return; + } + model[PATCHED] = true; + patchModelMethod(model, 'doGenerate', options); + patchModelMethod(model, 'doStream', options); +} + function patchModelMethod( model: PatchableModel, method: 'doGenerate' | 'doStream', @@ -426,7 +448,9 @@ function patchModelMethod( callId, provider: model.provider, modelId: model.modelId, - tools: callArgs.tools, + // v4 nests the tool list under `mode.tools` (the `LanguageModelV1` call shape); v5+ passes a + // top-level `tools` array. Reading both keeps `available_tools` populated on the model-call span. + tools: callArgs.tools ?? (isRecord(callArgs.mode) ? callArgs.mode.tools : undefined), messages: callArgs.prompt, // Inherit the enclosing operation's per-call recording flags so inputs/tools/outputs are recorded on // the model-call span whenever they are on the parent `invoke_agent` span. @@ -594,6 +618,10 @@ function buildTextMessage(type: 'generateText' | 'streamText'): MessageBuilder { functionId: asString(telemetry.functionId), ...modelFields(options.model), maxRetries: options.maxRetries, + // The `ai` SDK takes the system prompt as a top-level `system` option (all of v4/v5/v6); the + // shared core lifts `event.instructions` into the system-instructions attribute, matching v7's + // native channel (which carries it as a distinct field rather than inside the messages array). + instructions: asString(options.system), // Normalize to the message-array shape the shared core (and v7's channel) expects: a bare string // `prompt` becomes a single user message, matching the SDK's own normalization. messages: normalizePromptMessages(options), From b8a46edc83cf03fe536576d0cfa2865f6de68a9b Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 9 Jul 2026 13:22:29 +0200 Subject: [PATCH 2/6] comment cleanup --- .../suites/tracing/vercelai/span-streaming-v4/test.ts | 3 --- .../node-integration-tests/suites/tracing/vercelai/test.ts | 2 -- 2 files changed, 5 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts index a5df3a60c6e7..f28559c4b9e9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -30,9 +30,6 @@ function attr(value: unknown) { return expect.objectContaining({ value }); } -// In orchestrion mode the `ai` SDK is instrumented via the diagnostics-channel subscriber -// (`auto.vercelai.channel`); otherwise via the OTel span processor (`auto.vercelai.otel`). The -// `vercel.ai.*` pipeline attributes are OTel-processor-only, so they aren't asserted here. const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; // v4's OTel path serializes tool-call arguments from the provider's raw JSON string (whitespace diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index ea555eb3e5af..6a0823b488c4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -24,8 +24,6 @@ import { import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; import { getStringAttributeValue, isOrchestrionEnabled } from '../../../utils'; -// In orchestrion mode the `ai` SDK is instrumented via the diagnostics-channel subscriber -// (`auto.vercelai.channel`); otherwise via the OTel span processor (`auto.vercelai.otel`). const orchestrion = isOrchestrionEnabled(); const expectedOrigin = orchestrion ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; From 6d7d41ba27e7c6d797bb8ca416c8bc9270497068 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 9 Jul 2026 13:57:45 +0200 Subject: [PATCH 3/6] cover generate object --- .../tracing/vercelai/test-generate-object.ts | 50 ------------------- .../suites/tracing/vercelai/test.ts | 47 +++++++++++++++++ .../src/orchestrion/config/vercel-ai.ts | 2 + .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 14 +++++- .../vercel-ai-orchestrion-subscriber.ts | 3 +- 5 files changed, 64 insertions(+), 52 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts deleted file mode 100644 index 54c64bc2172b..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; - -describe('Vercel AI integration - generateObject', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests(__dirname, 'scenario-generate-object.mjs', 'instrument.mjs', (createRunner, test) => { - test('captures generateObject spans with schema attributes', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - const [firstSpan, secondSpan] = container.items; - - // [0] generateObject (invoke_agent) - expect(firstSpan!.name).toBe('invoke_agent'); - expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); - expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); - expect(firstSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); - expect(firstSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(firstSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(firstSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(firstSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); - expect(firstSpan!.attributes['gen_ai.request.schema']).toBeDefined(); - - // [1] generateObject.doGenerate (generate_content) - expect(secondSpan!.name).toBe('generate_content mock-model-id'); - expect(secondSpan!.status).toBe('ok'); - expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); - expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); - expect(secondSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); - expect(secondSpan!.attributes['gen_ai.system'].value).toBe('mock-provider'); - expect(secondSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); - expect(secondSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(secondSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(secondSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(secondSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); - }, - }) - .start() - .completed(); - }); - }); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 6a0823b488c4..0b91ce62a02c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -715,4 +715,51 @@ describe('Vercel AI integration (v4)', () => { .completed(); }); }); + + createEsmAndCjsTests(__dirname, 'scenario-generate-object.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('captures generateObject spans with schema attributes', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + + // generateObject (invoke_agent) + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); + expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); + expect(invokeAgentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); + expect(invokeAgentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); + expect(invokeAgentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + // The JSON schema attribute is derived from the SDK's Zod schema by the OTel span processor; + // the channel subscriber does not reconstruct it, so assert it only in OTel mode. + if (!orchestrion) { + expect(invokeAgentSpan!.attributes['gen_ai.request.schema']).toBeDefined(); + } + + // generateObject.doGenerate (generate_content) + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); + expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); + expect(generateContentSpan!.attributes['gen_ai.system'].value).toBe('mock-provider'); + expect(generateContentSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); + expect(generateContentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); + expect(generateContentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + }, + }) + .start() + .completed(); + }); + }); }); diff --git a/packages/server-utils/src/orchestrion/config/vercel-ai.ts b/packages/server-utils/src/orchestrion/config/vercel-ai.ts index 4f9553df9d6a..082be1a135bc 100644 --- a/packages/server-utils/src/orchestrion/config/vercel-ai.ts +++ b/packages/server-utils/src/orchestrion/config/vercel-ai.ts @@ -13,6 +13,7 @@ export const vercelAiConfig = [ // The majority of entrypoints are present in all versions we support ...vercelAiEntries('>=4.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), ...vercelAiEntries('>=4.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=4.0.0 <7.0.0', 'generateObject', 'generateObject', 'Async'), ...vercelAiEntries('>=4.0.0 <7.0.0', 'embed', 'embed', 'Async'), ...vercelAiEntries('>=4.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), @@ -32,6 +33,7 @@ export const vercelAiChannels = { // `VERCEL_AI_RESOLVE_LANGUAGE_MODEL` is v5/v6-only (v4 has no such chokepoint). VERCEL_AI_GENERATE_TEXT: 'orchestrion:ai:generateText', VERCEL_AI_STREAM_TEXT: 'orchestrion:ai:streamText', + VERCEL_AI_GENERATE_OBJECT: 'orchestrion:ai:generateObject', VERCEL_AI_EMBED: 'orchestrion:ai:embed', VERCEL_AI_EMBED_MANY: 'orchestrion:ai:embedMany', VERCEL_AI_EXECUTE_TOOL_CALL: 'orchestrion:ai:executeToolCall', diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index dd7fd514bef3..52cb19a5ea1d 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -102,7 +102,14 @@ const invokeAgentSpanByCallId = new Map(); // Only top-level operations own the `callId` → operationId mapping; `step`/`languageModelCall`/ // `executeTool` share the parent's `callId`, so they must not clear it. -const ROOT_OPERATION_TYPES = new Set(['generateText', 'streamText', 'embed', 'embedMany', 'rerank']); +const ROOT_OPERATION_TYPES = new Set([ + 'generateText', + 'streamText', + 'generateObject', + 'embed', + 'embedMany', + 'rerank', +]); /** Drop the per-operation `callId` maps once the owning top-level operation settles (success or error). */ export function clearOperationId(data: VercelAiChannelMessage): void { @@ -171,6 +178,7 @@ function resolveToolDescription(callId: string | undefined, toolName: string, to export type ChannelEventType = | 'generateText' | 'streamText' + | 'generateObject' | 'step' | 'languageModelCall' | 'executeTool' @@ -388,6 +396,10 @@ export function createSpanFromMessage( switch (type) { case 'generateText': case 'streamText': + case 'generateObject': + // `generateObject` builds the same `invoke_agent` span as `generateText` (non-streaming); its + // distinct `ai.generateObject` operationId rides on `event.operationId`. The JSON-schema attribute + // the OTel path derives from the SDK's Zod schema is not reconstructed on the channel path. return buildInvokeAgentSpan(event, baseAttributes, recordInputs, enableTruncation, callId, type === 'streamText'); case 'languageModelCall': return buildModelCallSpan(event, baseAttributes, recordInputs, enableTruncation, callId, modelId); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 76a42bae7178..d4600cc63bb7 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -130,6 +130,7 @@ export function subscribeVercelAiOrchestrionChannels( try { bindOperation(tracingChannel, CHANNELS.VERCEL_AI_GENERATE_TEXT, buildTextMessage('generateText'), options); bindOperation(tracingChannel, CHANNELS.VERCEL_AI_STREAM_TEXT, buildTextMessage('streamText'), options); + bindOperation(tracingChannel, CHANNELS.VERCEL_AI_GENERATE_OBJECT, buildTextMessage('generateObject'), options); bindOperation( tracingChannel, CHANNELS.VERCEL_AI_EMBED, @@ -609,7 +610,7 @@ function patchToolExecute( }; } -function buildTextMessage(type: 'generateText' | 'streamText'): MessageBuilder { +function buildTextMessage(type: 'generateText' | 'streamText' | 'generateObject'): MessageBuilder { return (options, telemetry) => ({ type, event: { From 355dfdccadd7abbf1effdcdb4212ff0b76572ae0 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 9 Jul 2026 14:05:45 +0200 Subject: [PATCH 4/6] test(node): Cover generateObject in the v6/v7 vercelai suite v6 emits generateObject spans via orchestrion; v7's native ai:telemetry channel does not publish generateObject (like embedMany), so the v7 case is skipped with a documenting comment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../v6_v7/scenario-generate-object.mjs | 35 ++++++++++++ .../suites/tracing/vercelai/v6_v7/test.ts | 54 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-generate-object.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-generate-object.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-generate-object.mjs new file mode 100644 index 000000000000..dfab9c73336b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-generate-object.mjs @@ -0,0 +1,35 @@ +import * as Sentry from '@sentry/node'; +import { generateObject } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateObject({ + experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [{ type: 'text', text: '{ "name": "John Doe", "age": 30 }' }], + warnings: [], + }), + }), + schema: z.object({ + name: z.string().describe('The name of the person'), + age: z.number().describe('The age of the person'), + }), + schemaName: 'Person', + schemaDescription: 'A person with name and age', + prompt: 'Generate a person object', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index b49783eebc2e..c6841314050e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -884,4 +884,58 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); + + createEsmTests( + __dirname, + 'scenario-generate-object.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + // ai v7's native `ai:telemetry` channel does not publish a top-level `generateObject` operation + // (like `embedMany`, it's dispatched via a path the channel never sees), so the channel-based + // integration can't surface it on v7. v4/v5/v6 use orchestrion, which injects the `generateObject` + // channel directly — v6 is exercised here, v4 in the base suite. + if (version === '7') { + return; + } + test('creates spans for generateObject', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + // Every emitted gen_ai span carries the version-appropriate origin. + container.items + .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) + .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); + expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]?.value).toBe(40); + + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( + 'ai.generateObject.doGenerate', + ); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); }); From 7179dc1785dbdfaa9b06d59b84493dab7e53099c Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 9 Jul 2026 14:39:02 +0200 Subject: [PATCH 5/6] better check --- .../suites/tracing/vercelai/v6_v7/test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index c6841314050e..6963de59bcc6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -894,10 +894,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // (like `embedMany`, it's dispatched via a path the channel never sees), so the channel-based // integration can't surface it on v7. v4/v5/v6 use orchestrion, which injects the `generateObject` // channel directly — v6 is exercised here, v4 in the base suite. - if (version === '7') { - return; - } - test('creates spans for generateObject', async () => { + test.skipIf(version === '7')('creates spans for generateObject', async () => { await createRunner() .expect({ transaction: { transaction: 'main' } }) .expect({ From ed36ecd8270ac49d975d0f39edcedab22b9b1043 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 10 Jul 2026 09:23:39 +0200 Subject: [PATCH 6/6] fix it --- .../src/vercel-ai/vercel-ai-orchestrion-subscriber.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index d4600cc63bb7..4548ecc73b35 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -245,7 +245,7 @@ function bindOperation( // `resolveLanguageModel`, so restricting to `'v1'` keeps this strictly additive and avoids // double-patching. Embedding models are also `'v1'` but expose no `doGenerate`/`doStream`, so // the patch is a no-op for `embed`/`embedMany`. - if (isRecord(callOptions.model) && callOptions.model.specificationVersion === 'v1') { + if (isObjectLike(callOptions.model) && callOptions.model.specificationVersion === 'v1') { patchModelMethods(callOptions.model as PatchableModel, options); } } @@ -451,7 +451,7 @@ function patchModelMethod( modelId: model.modelId, // v4 nests the tool list under `mode.tools` (the `LanguageModelV1` call shape); v5+ passes a // top-level `tools` array. Reading both keeps `available_tools` populated on the model-call span. - tools: callArgs.tools ?? (isRecord(callArgs.mode) ? callArgs.mode.tools : undefined), + tools: callArgs.tools ?? (isObjectLike(callArgs.mode) ? callArgs.mode.tools : undefined), messages: callArgs.prompt, // Inherit the enclosing operation's per-call recording flags so inputs/tools/outputs are recorded on // the model-call span whenever they are on the parent `invoke_agent` span.