From 9509b3bd45d7a0e68e31889f6900f7442d70adc7 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 10:45:19 +0200 Subject: [PATCH 1/7] test: Add eve e2e test app --- .github/workflows/build.yml | 2 + dev-packages/e2e-tests/.env.example | 4 + dev-packages/e2e-tests/run.ts | 2 +- .../test-applications/node-eve/.gitignore | 10 ++ .../test-applications/node-eve/agent/agent.ts | 16 ++ .../node-eve/agent/channels/eve.ts | 8 + .../node-eve/agent/instructions.md | 7 + .../node-eve/agent/instrumentation.ts | 13 ++ .../node-eve/agent/tools/fail_now.ts | 10 ++ .../node-eve/agent/tools/get_weather.ts | 10 ++ .../test-applications/node-eve/package.json | 43 ++++++ .../node-eve/playwright.config.mjs | 13 ++ .../node-eve/start-event-proxy.mjs | 6 + .../node-eve/tests/eve.test.ts | 143 ++++++++++++++++++ .../test-applications/node-eve/tsconfig.json | 13 ++ 15 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a9e79c085c5..23fb501d72b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1149,6 +1149,8 @@ jobs: REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' + # Used by the `node-eve` test app to make real model calls through OpenRouter + E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} strategy: fail-fast: false matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index c598b7cbf597..13eff7352024 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -11,3 +11,7 @@ E2E_TEST_SENTRY_ORG_SLUG= # A Sentry project slug E2E_TEST_SENTRY_PROJECT= + +# An OpenRouter API key, used by the `node-eve` test app to make real model calls. +# Only needed to run that test app locally. +E2E_OPENROUTER_API_KEY= diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 5a44ea9a9416..e2c56813a3bf 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -260,7 +260,7 @@ async function run(): Promise { await asyncExec(testCommand, { env: appEnv, cwd }); // clean up (although this is tmp, still nice to do) - await rm(tmpDirPath, { recursive: true }); + await rm(tmpDirPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 300 }); } } diff --git a/dev-packages/e2e-tests/test-applications/node-eve/.gitignore b/dev-packages/e2e-tests/test-applications/node-eve/.gitignore new file mode 100644 index 000000000000..8301375b6738 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/.gitignore @@ -0,0 +1,10 @@ +node_modules +.eve +.output +.nitro +.vercel +.data +*.tsbuildinfo +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts new file mode 100644 index 000000000000..c528086820ad --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts @@ -0,0 +1,16 @@ +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { defineAgent } from "eve"; + +// We call OpenRouter directly (rather than the default Vercel AI Gateway) so the +// e2e test needs only a single OpenRouter key. eve resolves this authored +// `LanguageModel` at runtime. +const openrouter = createOpenRouter({ + apiKey: process.env.E2E_OPENROUTER_API_KEY, +}); + +export default defineAgent({ + model: openrouter("openai/gpt-4o-mini"), + // A direct-provider model is not in the AI Gateway catalog, so eve cannot look + // up its context window for compaction. Provide it explicitly. + modelContextWindowTokens: 128_000, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts new file mode 100644 index 000000000000..14a1d557384a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts @@ -0,0 +1,8 @@ +import { none } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; + +// The test drives the agent over localhost in both dev and prod, so the channel +// is left open. Do not copy this into a real deployment. +export default eveChannel({ + auth: [none()], +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md new file mode 100644 index 000000000000..339925ffc4f7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md @@ -0,0 +1,7 @@ +You are a concise assistant used by an automated end-to-end test. + +- When the user asks about the weather in a place, call the `get_weather` tool + for that place and answer in one short sentence using its result. +- When the user asks you to trigger a failure, call the `fail_now` tool. + +Do not ask follow-up questions. diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts new file mode 100644 index 000000000000..229307fa5b7a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts @@ -0,0 +1,13 @@ +import * as Sentry from "@sentry/node"; + +// eve auto-discovers `agent/instrumentation.ts` and runs it at server startup, +// before it loads the agent (and the `ai` SDK). That is early enough for the +// Sentry SDK to install its instrumentation, so no `--import` / `NODE_OPTIONS` +// bootstrap is needed. eve's own OpenTelemetry pipeline is intentionally left +// unused: the gen_ai spans come from Sentry's `ai` instrumentation, not OTel. +Sentry.init({ + environment: "qa", + dsn: process.env.E2E_TEST_DSN, + tunnel: "http://localhost:3031/", // proxy server + tracesSampleRate: 1.0, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts new file mode 100644 index 000000000000..a62230371ee5 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +export default defineTool({ + description: "Always throws an error. Call this when the user asks to trigger a failure.", + inputSchema: z.object({}), + async execute() { + throw new Error("Intentional eve tool failure"); + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts new file mode 100644 index 000000000000..18b17904f892 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +export default defineTool({ + description: "Get the current weather for a city.", + inputSchema: z.object({ city: z.string().min(1) }), + async execute({ city }) { + return { city, condition: "Sunny", temperatureC: 22 }; + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/package.json b/dev-packages/e2e-tests/test-applications/node-eve/package.json new file mode 100644 index 000000000000..f24e782faf6a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/package.json @@ -0,0 +1,43 @@ +{ + "name": "node-eve", + "version": "0.0.0", + "private": true, + "type": "module", + "imports": { + "#*": "./agent/*" + }, + "scripts": { + "build": "EVE_TELEMETRY_DISABLED=1 eve build", + "dev": "EVE_TELEMETRY_DISABLED=1 eve dev --no-ui --port 3030", + "start": "EVE_TELEMETRY_DISABLED=1 eve start --port 3030", + "clean": "npx rimraf node_modules .eve .output pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:assert": "pnpm test:prod && pnpm test:dev", + "test:prod": "TEST_ENV=production playwright test", + "test:dev": "TEST_ENV=development playwright test" + }, + "dependencies": { + "@openrouter/ai-sdk-provider": "^3.0.0", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "ai": "^7.0.82", + "eve": "^0.52.3", + "zod": "4.5.4" + }, + "devDependencies": { + "@playwright/test": "~1.56.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/core": "file:../../packed/sentry-core-packed.tgz", + "@types/node": "24.x", + "typescript": "~5.9.0" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs new file mode 100644 index 000000000000..d46451a21501 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs @@ -0,0 +1,13 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const testEnv = process.env.TEST_ENV; + +if (!testEnv) { + throw new Error('No test env defined'); +} + +const config = getPlaywrightConfig({ + startCommand: testEnv === 'development' ? 'pnpm dev' : 'pnpm start', +}); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs new file mode 100644 index 000000000000..1c059695bb90 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-eve', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts new file mode 100644 index 000000000000..2e5b0dbea960 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -0,0 +1,143 @@ +import { expect, test } from '@playwright/test'; +import { getSpanOp, waitForError, waitForStreamedSpans } from '@sentry-internal/test-utils'; + +const APP = 'node-eve'; + +// eve serves one agent turn across two request contexts — the session API +// request (`POST /eve/v1/session`) and the internal durable-workflow request +// (`POST /.well-known/workflow/v1/flow`) — so the captured server path can be +// either one, depending on eve's workflow scheduling. +const EVE_AGENT_PATH = /(\/eve\/v1\/session|\/\.well-known\/workflow\/v1\/flow)/; + + +/** + * Drive one agent turn through eve's default HTTP channel and wait for it to + * settle, so the agent has finished and its spans have been flushed before we + * assert. eve runs the turn in a durable workflow, so the POST only needs to be + * accepted; we drain the event stream to know when the turn is done. + */ +async function runAgentTurn(baseURL: string, message: string): Promise { + const createRes = await fetch(`${baseURL}/eve/v1/session`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message }), + }); + expect(createRes.status).toBe(202); + const { sessionId } = (await createRes.json()) as { sessionId: string }; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 25_000); + try { + const streamRes = await fetch(`${baseURL}/eve/v1/session/${sessionId}/stream`, { + signal: controller.signal, + }); + const reader = streamRes.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + if (buffer.includes('"type":"session.waiting"') || buffer.includes('"type":"turn.failed"')) { + break; + } + } + await reader.cancel().catch(() => {}); + } finally { + clearTimeout(timer); + } +} + +test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_tool) for an eve turn', async ({ + baseURL, +}) => { + const genAiSpansPromise = waitForStreamedSpans(APP, spans => + ['gen_ai.invoke_agent', 'gen_ai.generate_content', 'gen_ai.execute_tool'].every(op => + spans.some(span => getSpanOp(span) === op), + ), + ); + const httpServerSpanPromise = waitForStreamedSpans(APP, spans => + spans.some( + span => + getSpanOp(span) === 'http.server' && + EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')), + ), + ); + + await runAgentTurn(baseURL!, 'What is the weather in Paris?'); + + const genAiSpans = await genAiSpansPromise; + + const invokeAgent = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const generateContent = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.generate_content'); + const executeTool = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); + + expect(invokeAgent?.attributes?.['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(invokeAgent?.attributes?.['gen_ai.operation.name']?.value).toBe('invoke_agent'); + expect(invokeAgent?.attributes?.['gen_ai.request.model']?.value).toBe('openai/gpt-4o-mini'); + expect(invokeAgent?.attributes?.['gen_ai.provider.name']?.value).toBe('openrouter'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe('number'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe('number'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.total_tokens']?.value).toBe('number'); + + expect(generateContent?.attributes?.['gen_ai.operation.name']?.value).toBe('generate_content'); + expect(generateContent?.attributes?.['gen_ai.request.model']?.value).toBe('openai/gpt-4o-mini'); + + expect(executeTool?.attributes?.['gen_ai.operation.name']?.value).toBe('execute_tool'); + expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('get_weather'); + + // The agent turn is captured as an http.server span on one of eve's two agent + // request paths (the other http.server spans — health and the event stream — + // are filtered out). + const allSpans = await httpServerSpanPromise; + const agentServerSpans = allSpans.filter( + span => + getSpanOp(span) === 'http.server' && + EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')), + ); + expect(agentServerSpans.length).toBeGreaterThanOrEqual(1); + + for (const span of agentServerSpans) { + expect(span).toMatchObject({ + // no parametrization available, so the server span name is just the method + name: 'POST', + attributes: { + 'sentry.origin': { value: 'auto.http.http_server', type: 'string' }, + 'sentry.op': { value: 'http.server', type: 'string' }, + 'sentry.segment.name.source': { value: 'url', type: 'string' }, + 'sentry.kind': { value: 'server', type: 'string' }, + 'url.path': { value: expect.stringMatching(EVE_AGENT_PATH), type: 'string' }, + 'http.request.method': { value: 'POST', type: 'string' }, + }, + }); + } +}); + +test('captures errors thrown inside an eve tool', async ({ baseURL }) => { + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value?.includes('Intentional eve tool failure') ?? false, + ); + + await runAgentTurn(baseURL!, 'Please call the tool that triggers a failure now.'); + + const error = await errorPromise; + + expect(error).toMatchObject({ + exception: { + values: [ + { + type: 'Error', + value: expect.stringContaining('Intentional eve tool failure'), + mechanism: { + type: 'auto.vercelai.channel', + handled: false, + }, + }, + ], + }, + // The tool runs inside eve's durable workflow, so the error is attributed to + // either the session request or the internal workflow request. + transaction: expect.stringMatching(EVE_AGENT_PATH), + }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json new file mode 100644 index 000000000000..79911c330c61 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "types": ["node", "eve/workflow-modules"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["agent/**/*.ts"] +} From 0832ee0f3bea54fdf69dc0f6e94834ddf113147d Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 11:24:02 +0200 Subject: [PATCH 2/7] add dataloader test --- .../test-applications/node-eve/agent/agent.ts | 8 ++++ .../node-eve/agent/instructions.md | 1 + .../node-eve/agent/instrumentation.ts | 5 +++ .../node-eve/agent/tools/count_items.ts | 17 ++++++++ .../test-applications/node-eve/package.json | 17 +++++++- .../node-eve/playwright.config.mjs | 9 ++++- .../node-eve/tests/dataloader.test.ts | 37 +++++++++++++++++ .../node-eve/tests/eve.test.ts | 40 +------------------ .../test-applications/node-eve/tests/utils.ts | 39 ++++++++++++++++++ 9 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts index c528086820ad..c47030c41dc2 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts @@ -13,4 +13,12 @@ export default defineAgent({ // A direct-provider model is not in the AI Gateway catalog, so eve cannot look // up its context window for compaction. Provide it explicitly. modelContextWindowTokens: 128_000, + build: { + // `dataloader` is instrumented by Sentry via orchestrion (a module + // transform). Keep it external so it stays a real module the transform can + // hook; if eve inlined it into the server bundle it could never be + // instrumented. (The Vercel AI SDK needs none of this — it uses a native + // diagnostics channel.) + externalDependencies: ["dataloader"], + }, }); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md index 339925ffc4f7..e67cae081e40 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md @@ -2,6 +2,7 @@ You are a concise assistant used by an automated end-to-end test. - When the user asks about the weather in a place, call the `get_weather` tool for that place and answer in one short sentence using its result. +- When the user asks to count items, call the `count_items` tool with the item names. - When the user asks you to trigger a failure, call the `fail_now` tool. Do not ask follow-up questions. diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts index 229307fa5b7a..83f2548807e8 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts @@ -10,4 +10,9 @@ Sentry.init({ dsn: process.env.E2E_TEST_DSN, tunnel: "http://localhost:3031/", // proxy server tracesSampleRate: 1.0, + // Not a default integration. It only produces spans in the "orchestrion" test + // variant, where the server is started with + // `NODE_OPTIONS=--import=@sentry/node/import` so the orchestrion module + // transform is registered before `dataloader` loads. + integrations: [Sentry.dataloaderIntegration()], }); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts new file mode 100644 index 000000000000..4614478cb0c0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts @@ -0,0 +1,17 @@ +import DataLoader from "dataloader"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +// Uses `dataloader` so the e2e test can assert Sentry's orchestrion-based +// instrumentation of it. Unlike the Vercel AI SDK (native diagnostics channel), +// orchestrion packages are only instrumented when the Sentry loader is +// registered at process start (the "orchestrion" test variant). +export default defineTool({ + description: "Count the number of letters in each given name. Call this when asked to count items.", + inputSchema: z.object({ names: z.array(z.string()).min(1) }), + async execute({ names }) { + const loader = new DataLoader(async keys => keys.map(k => k.length)); + const counts = await Promise.all(names.map(n => loader.load(n))); + return { counts }; + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/package.json b/dev-packages/e2e-tests/test-applications/node-eve/package.json index f24e782faf6a..9a8799c1fca7 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/package.json +++ b/dev-packages/e2e-tests/test-applications/node-eve/package.json @@ -10,9 +10,13 @@ "build": "EVE_TELEMETRY_DISABLED=1 eve build", "dev": "EVE_TELEMETRY_DISABLED=1 eve dev --no-ui --port 3030", "start": "EVE_TELEMETRY_DISABLED=1 eve start --port 3030", + "dev:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm dev", + "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start", "clean": "npx rimraf node_modules .eve .output pnpm-lock.yaml", "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add eve@latest ai@latest && pnpm build", "test:assert": "pnpm test:prod && pnpm test:dev", + "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert", "test:prod": "TEST_ENV=production playwright test", "test:dev": "TEST_ENV=development playwright test" }, @@ -20,6 +24,7 @@ "@openrouter/ai-sdk-provider": "^3.0.0", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", "ai": "^7.0.82", + "dataloader": "^2.2.3", "eve": "^0.52.3", "zod": "4.5.4" }, @@ -38,6 +43,16 @@ "extends": "../../package.json" }, "sentryTest": { - "optional": true + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-eve (latest)" + }, + { + "assert-command": "pnpm test:assert-orchestrion", + "label": "node-eve (orchestrion)" + } + ] } } diff --git a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs index d46451a21501..442aa7fdb901 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs +++ b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs @@ -1,13 +1,20 @@ import { getPlaywrightConfig } from '@sentry-internal/test-utils'; const testEnv = process.env.TEST_ENV; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; if (!testEnv) { throw new Error('No test env defined'); } +let startCommand = testEnv === 'development' ? 'pnpm dev' : 'pnpm start'; + +if (useOrchestrion) { + startCommand = `${startCommand}:orchestrion`; +} + const config = getPlaywrightConfig({ - startCommand: testEnv === 'development' ? 'pnpm dev' : 'pnpm start', + startCommand, }); export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts new file mode 100644 index 000000000000..5293de7e9d7b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from '@playwright/test'; +import { getSpanOp, waitForStreamedSpans } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-eve'; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +const isDataloaderSpan = (span: { attributes?: Record }): boolean => + getSpanOp(span) === 'cache.get' && span.attributes?.['sentry.origin']?.value === 'auto.db.dataloader'; + +/** + * `dataloader` is instrumented by Sentry via orchestrion (a module transform), + * unlike the Vercel AI SDK which publishes to a native diagnostics channel. + * Under eve's bundled server output the transform only runs when the Sentry + * loader is registered at process start via + * `NODE_OPTIONS=--import=@sentry/node/import` (the `node-eve (orchestrion)` + * variant, `USE_ORCHESTRION=1`). Without that bootstrap no dataloader span is + * captured, so this test is expected to fail — see `test.fail(!useOrchestrion)`. + */ +test('captures orchestrion-instrumented dataloader spans (requires the --import bootstrap)', async ({ baseURL }) => { + test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import'); + + // With orchestrion, wait for the dataloader span itself. Without it, that span + // never arrives, so anchor on the (always-present) tool-execution span and let + // the assertion below fail fast rather than time out. + const spansPromise = waitForStreamedSpans(APP, spans => + useOrchestrion ? spans.some(isDataloaderSpan) : spans.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + ); + + await runAgentTurn(baseURL!, 'Count items: apple, banana, cherry'); + + const spans = await spansPromise; + const dataloaderSpan = spans.find(isDataloaderSpan); + + expect(dataloaderSpan?.attributes?.['sentry.op']?.value).toBe('cache.get'); + expect(dataloaderSpan?.attributes?.['sentry.origin']?.value).toBe('auto.db.dataloader'); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 2e5b0dbea960..0aca978e5b95 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -1,5 +1,6 @@ import { expect, test } from '@playwright/test'; import { getSpanOp, waitForError, waitForStreamedSpans } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; const APP = 'node-eve'; @@ -9,45 +10,6 @@ const APP = 'node-eve'; // either one, depending on eve's workflow scheduling. const EVE_AGENT_PATH = /(\/eve\/v1\/session|\/\.well-known\/workflow\/v1\/flow)/; - -/** - * Drive one agent turn through eve's default HTTP channel and wait for it to - * settle, so the agent has finished and its spans have been flushed before we - * assert. eve runs the turn in a durable workflow, so the POST only needs to be - * accepted; we drain the event stream to know when the turn is done. - */ -async function runAgentTurn(baseURL: string, message: string): Promise { - const createRes = await fetch(`${baseURL}/eve/v1/session`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ message }), - }); - expect(createRes.status).toBe(202); - const { sessionId } = (await createRes.json()) as { sessionId: string }; - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 25_000); - try { - const streamRes = await fetch(`${baseURL}/eve/v1/session/${sessionId}/stream`, { - signal: controller.signal, - }); - const reader = streamRes.body!.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - if (buffer.includes('"type":"session.waiting"') || buffer.includes('"type":"turn.failed"')) { - break; - } - } - await reader.cancel().catch(() => {}); - } finally { - clearTimeout(timer); - } -} - test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_tool) for an eve turn', async ({ baseURL, }) => { diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts new file mode 100644 index 000000000000..6e3f0bd7a301 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts @@ -0,0 +1,39 @@ +import { expect } from '@playwright/test'; + +/** + * Drive one agent turn through eve's default HTTP channel and wait for it to + * settle, so the agent has finished and its spans have been flushed before we + * assert. eve runs the turn in a durable workflow, so the POST only needs to be + * accepted; we drain the event stream to know when the turn is done. + */ +export async function runAgentTurn(baseURL: string, message: string): Promise { + const createRes = await fetch(`${baseURL}/eve/v1/session`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message }), + }); + expect(createRes.status).toBe(202); + const { sessionId } = (await createRes.json()) as { sessionId: string }; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 25_000); + try { + const streamRes = await fetch(`${baseURL}/eve/v1/session/${sessionId}/stream`, { + signal: controller.signal, + }); + const reader = streamRes.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + if (buffer.includes('"type":"session.waiting"') || buffer.includes('"type":"turn.failed"')) { + break; + } + } + await reader.cancel().catch(() => {}); + } finally { + clearTimeout(timer); + } +} From acc622e420b79365b3329f3b2a60cd962e81b232 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 11:29:06 +0200 Subject: [PATCH 3/7] fixes and stuff --- .../test-applications/node-eve/agent/agent.ts | 30 +++++++++++++------ .../node-eve/agent/channels/eve.ts | 4 +-- .../node-eve/agent/instrumentation.ts | 6 ++-- .../node-eve/agent/tools/count_items.ts | 8 ++--- .../node-eve/agent/tools/fail_now.ts | 8 ++--- .../node-eve/agent/tools/get_weather.ts | 8 ++--- .../test-applications/node-eve/package.json | 2 ++ .../node-eve/tests/eve.test.ts | 22 +++++++------- 8 files changed, 50 insertions(+), 38 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts index c47030c41dc2..f4ec76970a0d 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts @@ -1,5 +1,5 @@ -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; -import { defineAgent } from "eve"; +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import { defineAgent } from 'eve'; // We call OpenRouter directly (rather than the default Vercel AI Gateway) so the // e2e test needs only a single OpenRouter key. eve resolves this authored @@ -8,17 +8,29 @@ const openrouter = createOpenRouter({ apiKey: process.env.E2E_OPENROUTER_API_KEY, }); +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + export default defineAgent({ - model: openrouter("openai/gpt-4o-mini"), + model: openrouter('openai/gpt-4o-mini'), // A direct-provider model is not in the AI Gateway catalog, so eve cannot look // up its context window for compaction. Provide it explicitly. modelContextWindowTokens: 128_000, build: { - // `dataloader` is instrumented by Sentry via orchestrion (a module - // transform). Keep it external so it stays a real module the transform can - // hook; if eve inlined it into the server bundle it could never be - // instrumented. (The Vercel AI SDK needs none of this — it uses a native - // diagnostics channel.) - externalDependencies: ["dataloader"], + // Only configure externals for orchestrion mode, to ensure everything else works without it + ...(useOrchestrion + ? { + // `dataloader` is instrumented by Sentry via orchestrion (a module + // transform). Keep it external so it stays a real module the transform can + // hook; if eve inlined it into the server bundle it could never be + // instrumented. (The Vercel AI SDK needs none of this — it uses a native + // diagnostics channel.) + // + // Do NOT add `@sentry/server-runtime-injection` here: the `--import` + // loader instruments regardless (so the "bundled ... uninstrumented" + // warning is a false positive), and externalizing it makes eve's dev + // host fail to resolve its `/register` subpath (`eve dev` only). + externalDependencies: ['dataloader'], + } + : {}), }, }); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts index 14a1d557384a..898e308d4394 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts @@ -1,5 +1,5 @@ -import { none } from "eve/channels/auth"; -import { eveChannel } from "eve/channels/eve"; +import { none } from 'eve/channels/auth'; +import { eveChannel } from 'eve/channels/eve'; // The test drives the agent over localhost in both dev and prod, so the channel // is left open. Do not copy this into a real deployment. diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts index 83f2548807e8..e2d33fd49591 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts @@ -1,4 +1,4 @@ -import * as Sentry from "@sentry/node"; +import * as Sentry from '@sentry/node'; // eve auto-discovers `agent/instrumentation.ts` and runs it at server startup, // before it loads the agent (and the `ai` SDK). That is early enough for the @@ -6,9 +6,9 @@ import * as Sentry from "@sentry/node"; // bootstrap is needed. eve's own OpenTelemetry pipeline is intentionally left // unused: the gen_ai spans come from Sentry's `ai` instrumentation, not OTel. Sentry.init({ - environment: "qa", + environment: 'qa', dsn: process.env.E2E_TEST_DSN, - tunnel: "http://localhost:3031/", // proxy server + tunnel: 'http://localhost:3031/', // proxy server tracesSampleRate: 1.0, // Not a default integration. It only produces spans in the "orchestrion" test // variant, where the server is started with diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts index 4614478cb0c0..47ec2d31acdd 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts @@ -1,13 +1,13 @@ -import DataLoader from "dataloader"; -import { defineTool } from "eve/tools"; -import { z } from "zod"; +import DataLoader from 'dataloader'; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; // Uses `dataloader` so the e2e test can assert Sentry's orchestrion-based // instrumentation of it. Unlike the Vercel AI SDK (native diagnostics channel), // orchestrion packages are only instrumented when the Sentry loader is // registered at process start (the "orchestrion" test variant). export default defineTool({ - description: "Count the number of letters in each given name. Call this when asked to count items.", + description: 'Count the number of letters in each given name. Call this when asked to count items.', inputSchema: z.object({ names: z.array(z.string()).min(1) }), async execute({ names }) { const loader = new DataLoader(async keys => keys.map(k => k.length)); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts index a62230371ee5..ec9b6aeae782 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts @@ -1,10 +1,10 @@ -import { defineTool } from "eve/tools"; -import { z } from "zod"; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; export default defineTool({ - description: "Always throws an error. Call this when the user asks to trigger a failure.", + description: 'Always throws an error. Call this when the user asks to trigger a failure.', inputSchema: z.object({}), async execute() { - throw new Error("Intentional eve tool failure"); + throw new Error('Intentional eve tool failure'); }, }); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts index 18b17904f892..10d29e22a7f7 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts @@ -1,10 +1,10 @@ -import { defineTool } from "eve/tools"; -import { z } from "zod"; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; export default defineTool({ - description: "Get the current weather for a city.", + description: 'Get the current weather for a city.', inputSchema: z.object({ city: z.string().min(1) }), async execute({ city }) { - return { city, condition: "Sunny", temperatureC: 22 }; + return { city, condition: 'Sunny', temperatureC: 22 }; }, }); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/package.json b/dev-packages/e2e-tests/test-applications/node-eve/package.json index 9a8799c1fca7..e95a1aaafe41 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/package.json +++ b/dev-packages/e2e-tests/test-applications/node-eve/package.json @@ -14,6 +14,7 @@ "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start", "clean": "npx rimraf node_modules .eve .output pnpm-lock.yaml", "test:build": "pnpm install && pnpm build", + "test:build-orchestrion": "USE_ORCHESTRION=1 pnpm test:build", "test:build-latest": "pnpm install && pnpm add eve@latest ai@latest && pnpm build", "test:assert": "pnpm test:prod && pnpm test:dev", "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert", @@ -50,6 +51,7 @@ "label": "node-eve (latest)" }, { + "build-command": "pnpm test:build-orchestrion", "assert-command": "pnpm test:assert-orchestrion", "label": "node-eve (orchestrion)" } diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 0aca978e5b95..1c089e65eb72 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -10,6 +10,14 @@ const APP = 'node-eve'; // either one, depending on eve's workflow scheduling. const EVE_AGENT_PATH = /(\/eve\/v1\/session|\/\.well-known\/workflow\/v1\/flow)/; +// The agent turn is served by a POST to one of eve's two agent paths. Requiring +// POST keeps the GET spans (health, the event stream) out even though +// EVE_AGENT_PATH also matches the stream path. +const isAgentServerSpan = (span: { attributes?: Record }): boolean => + getSpanOp(span) === 'http.server' && + span.attributes?.['http.request.method']?.value === 'POST' && + EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')); + test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_tool) for an eve turn', async ({ baseURL, }) => { @@ -18,13 +26,7 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to spans.some(span => getSpanOp(span) === op), ), ); - const httpServerSpanPromise = waitForStreamedSpans(APP, spans => - spans.some( - span => - getSpanOp(span) === 'http.server' && - EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')), - ), - ); + const httpServerSpanPromise = waitForStreamedSpans(APP, spans => spans.some(isAgentServerSpan)); await runAgentTurn(baseURL!, 'What is the weather in Paris?'); @@ -52,11 +54,7 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to // request paths (the other http.server spans — health and the event stream — // are filtered out). const allSpans = await httpServerSpanPromise; - const agentServerSpans = allSpans.filter( - span => - getSpanOp(span) === 'http.server' && - EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')), - ); + const agentServerSpans = allSpans.filter(isAgentServerSpan); expect(agentServerSpans.length).toBeGreaterThanOrEqual(1); for (const span of agentServerSpans) { From c8a3873c125db20f2af00d6e4b72ac0837f77f74 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 12:05:12 +0200 Subject: [PATCH 4/7] fix(node): Skip registration-only instrumentations in the runtime loader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registration-only orchestrion configs (native-channel libraries — ai v7, ioredis, @redis/client, mysql2, mongoose) carry a custom transform wired into the bundler plugins only. The runtime loader (`@sentry/server-runtime-injection` `register`) has no custom transforms, so transforming these modules threw `TypeError: transform is not a function`, which the diagnostics callback misreported as the always-on "`@sentry/server-runtime-injection` was bundled ... loads uninstrumented" warning — even though the libraries are correctly instrumented via their native channel (`setupOnce` / `waitForTracingChannelBinding`). Exclude registration-only configs from the runtime instrumentation set (`SENTRY_RUNTIME_INSTRUMENTATIONS`). This is lossless: at runtime the snippet would only trigger a no-op subscription to `orchestrion:*` channels these versions never publish. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../server-runtime-injection/src/register.ts | 8 ++--- .../src/orchestrion/config/index.ts | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/server-runtime-injection/src/register.ts b/packages/server-runtime-injection/src/register.ts index 881da139417f..25993f953577 100644 --- a/packages/server-runtime-injection/src/register.ts +++ b/packages/server-runtime-injection/src/register.ts @@ -3,7 +3,7 @@ import { existsSync } from 'node:fs'; import * as Module from 'node:module'; import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { SENTRY_INSTRUMENTATIONS } from '@sentry/server-utils/orchestrion/config'; +import { SENTRY_RUNTIME_INSTRUMENTATIONS } from '@sentry/server-utils/orchestrion/config'; import type { register } from 'node:module'; import ModulePatch from '@apm-js-collab/tracing-hooks'; import { initialize, load, resolve, createDiagnosticsPort } from '@apm-js-collab/tracing-hooks/hook-sync.mjs'; @@ -126,7 +126,7 @@ export function registerDiagnosticsChannelInjection(): void { // incompatibility) we warn and continue without channel injection. try { if (typeof mod.registerHooks === 'function' && stableSyncHooks) { - initialize({ instrumentations: SENTRY_INSTRUMENTATIONS }); + initialize({ instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS }); mod.registerHooks({ resolve, load }); debug.log('Registered diagnostics-channel injection via Module.registerHooks()'); } else if (typeof mod.register === 'function' && !globalAny.Bun && !globalAny.Deno) { @@ -185,7 +185,7 @@ export function registerDiagnosticsChannelInjection(): void { mod.register(hookSpecifier, { parentURL, - data: { instrumentations: SENTRY_INSTRUMENTATIONS, diagnosticsPort }, + data: { instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS, diagnosticsPort }, transferList: [diagnosticsPort], }); @@ -194,7 +194,7 @@ export function registerDiagnosticsChannelInjection(): void { // are resolved through the CJS machinery and never reach the ESM // register hook, so without this patch the file we want to instrument // loads untransformed. - new ModulePatch({ instrumentations: SENTRY_INSTRUMENTATIONS }).patch(); + new ModulePatch({ instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS }).patch(); debug.log('Registered diagnostics-channel injection via Module.register()'); } else { marker.runtimeUnavailable = true; diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index 1fec4fb2c5ad..f28ad60c56cb 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -35,6 +35,8 @@ import { vercelAiConfig } from './vercel-ai'; // Kept sorted alphabetically by module so concurrent additions insert at different // points rather than all appending to the end (fewer merge conflicts). +import { MODULE_REGISTRATION_TRANSFORM } from './registration-only'; + /** * The orchestrion code-transform configs. Every instrumentable library is here * so the transform is all-or-nothing: whenever orchestrion is enabled, all of @@ -81,6 +83,38 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...vercelAiConfig, ]; +/** + * The subset of {@link SENTRY_INSTRUMENTATIONS} the RUNTIME loader + * (`@sentry/server-runtime-injection`'s `register`, reached via `--import` or + * `Sentry.init()`) can actually apply. + * + * Registration-only configs (native-channel libraries such as `ai` v7, + * `ioredis`, `@redis/client`, `mysql2`, `mongoose`) carry the custom + * `MODULE_REGISTRATION_TRANSFORM` operator. That operator is wired into the + * BUNDLER plugins only (see `orchestrion/bundler/moduleInjectedTransform.ts`, + * applied via `bundler/options.ts`'s `customTransforms`); the runtime loader's + * `initialize()` receives no custom transforms. Attempting one of these at + * runtime therefore throws `TypeError: transform is not a function`, which the + * loader misreports as the always-on "`@sentry/server-runtime-injection` was + * bundled ... loads uninstrumented" warning even though nothing is wrong. + * + * Excluding them at runtime is correct, not just a way to silence the warning: + * these libraries publish their own tracing channels, and their integrations + * subscribe through `setupOnce()` / `waitForTracingChannelBinding`, + * independently of the module-injected snippet. That snippet only fires + * `orchestrion.module-injected`, which drives the `setup()` / + * `invokeOrchestrionInstrumentation` path; for a native-channel version that + * path subscribes to the injected `orchestrion:*` channels the library never + * publishes — a no-op. So running these at runtime would add no spans. The + * snippet earns its keep only on the BUNDLER path — notably bundler-only SDKs + * (e.g. `@sentry/cloudflare`) that discover a loaded module via that event to + * instantiate its integration factory. `@sentry/node` registers its + * integrations statically, so it does not need it. + */ +export const SENTRY_RUNTIME_INSTRUMENTATIONS: InstrumentationConfig[] = SENTRY_INSTRUMENTATIONS.filter( + config => config.transform !== MODULE_REGISTRATION_TRANSFORM, +); + /** * The unique set of package names instrumented by `SENTRY_INSTRUMENTATIONS` * merged with any caller-provided `instrumentations` (e.g. `['mysql']`). From de754a695fd8e3122567b760c730be278dce1172 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 12:49:02 +0200 Subject: [PATCH 5/7] fix spans --- .../node-eve/playwright.config.mjs | 10 ++++-- .../node-eve/tests/dataloader.test.ts | 15 +++++---- .../node-eve/tests/eve.test.ts | 31 +++++++++++-------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs index 442aa7fdb901..fabff63d3749 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs +++ b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs @@ -13,8 +13,12 @@ if (useOrchestrion) { startCommand = `${startCommand}:orchestrion`; } -const config = getPlaywrightConfig({ - startCommand, -}); +const config = getPlaywrightConfig( + { startCommand }, + // Each test drives a real OpenRouter tool-calling turn (two model calls) and + // then waits for the streamed spans to flush, which does not fit the default + // 30s test timeout when the provider is slow. + { timeout: 90_000 }, +); export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts index 5293de7e9d7b..409ad8349aae 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpans } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; import { runAgentTurn } from './utils'; const APP = 'node-eve'; @@ -20,11 +20,14 @@ const isDataloaderSpan = (span: { attributes?: Record { test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import'); - // With orchestrion, wait for the dataloader span itself. Without it, that span - // never arrives, so anchor on the (always-present) tool-execution span and let - // the assertion below fail fast rather than time out. - const spansPromise = waitForStreamedSpans(APP, spans => - useOrchestrion ? spans.some(isDataloaderSpan) : spans.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + // Accumulate the workflow trace's spans across envelopes. With orchestrion we + // wait for the dataloader span itself; without it that span never arrives, so + // anchor on the (always-present) tool-execution span and let the assertion + // below fail fast rather than time out. + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + useOrchestrion + ? spansOfTrace.some(isDataloaderSpan) + : spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), ); await runAgentTurn(baseURL!, 'Count items: apple, banana, cherry'); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 1c089e65eb72..1fee6d694312 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForError, waitForStreamedSpans } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; import { runAgentTurn } from './utils'; const APP = 'node-eve'; @@ -21,20 +21,26 @@ const isAgentServerSpan = (span: { attributes?: Record { - const genAiSpansPromise = waitForStreamedSpans(APP, spans => - ['gen_ai.invoke_agent', 'gen_ai.generate_content', 'gen_ai.execute_tool'].every(op => - spans.some(span => getSpanOp(span) === op), - ), + // The gen_ai spans and the agent http.server span share one trace, but the + // still-open invoke_agent parent flushes on a timer in a separate envelope + // from its completed children. `collectStreamedSpans` accumulates a trace's + // spans across envelopes (unlike `waitForStreamedSpans`, which sees one + // envelope at a time), so we wait until the whole trace has arrived. + const traceSpansPromise = collectStreamedSpans( + APP, + spansOfTrace => + ['gen_ai.invoke_agent', 'gen_ai.generate_content', 'gen_ai.execute_tool'].every(op => + spansOfTrace.some(span => getSpanOp(span) === op), + ) && spansOfTrace.some(isAgentServerSpan), ); - const httpServerSpanPromise = waitForStreamedSpans(APP, spans => spans.some(isAgentServerSpan)); await runAgentTurn(baseURL!, 'What is the weather in Paris?'); - const genAiSpans = await genAiSpansPromise; + const traceSpans = await traceSpansPromise; - const invokeAgent = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); - const generateContent = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.generate_content'); - const executeTool = genAiSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); + const invokeAgent = traceSpans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const generateContent = traceSpans.find(span => getSpanOp(span) === 'gen_ai.generate_content'); + const executeTool = traceSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); expect(invokeAgent?.attributes?.['sentry.origin']?.value).toBe('auto.vercelai.channel'); expect(invokeAgent?.attributes?.['gen_ai.operation.name']?.value).toBe('invoke_agent'); @@ -52,9 +58,8 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to // The agent turn is captured as an http.server span on one of eve's two agent // request paths (the other http.server spans — health and the event stream — - // are filtered out). - const allSpans = await httpServerSpanPromise; - const agentServerSpans = allSpans.filter(isAgentServerSpan); + // are not in this trace). + const agentServerSpans = traceSpans.filter(isAgentServerSpan); expect(agentServerSpans.length).toBeGreaterThanOrEqual(1); for (const span of agentServerSpans) { From cb96e27781c11e5dacc0b4094d673e3e17ddad13 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 12:49:33 +0200 Subject: [PATCH 6/7] Revert "fix(node): Skip registration-only instrumentations in the runtime loader" This reverts commit fe004a294914d66dae4ede6c5c257eddf80a4584. --- .../server-runtime-injection/src/register.ts | 8 ++--- .../src/orchestrion/config/index.ts | 34 ------------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/packages/server-runtime-injection/src/register.ts b/packages/server-runtime-injection/src/register.ts index 25993f953577..881da139417f 100644 --- a/packages/server-runtime-injection/src/register.ts +++ b/packages/server-runtime-injection/src/register.ts @@ -3,7 +3,7 @@ import { existsSync } from 'node:fs'; import * as Module from 'node:module'; import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { SENTRY_RUNTIME_INSTRUMENTATIONS } from '@sentry/server-utils/orchestrion/config'; +import { SENTRY_INSTRUMENTATIONS } from '@sentry/server-utils/orchestrion/config'; import type { register } from 'node:module'; import ModulePatch from '@apm-js-collab/tracing-hooks'; import { initialize, load, resolve, createDiagnosticsPort } from '@apm-js-collab/tracing-hooks/hook-sync.mjs'; @@ -126,7 +126,7 @@ export function registerDiagnosticsChannelInjection(): void { // incompatibility) we warn and continue without channel injection. try { if (typeof mod.registerHooks === 'function' && stableSyncHooks) { - initialize({ instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS }); + initialize({ instrumentations: SENTRY_INSTRUMENTATIONS }); mod.registerHooks({ resolve, load }); debug.log('Registered diagnostics-channel injection via Module.registerHooks()'); } else if (typeof mod.register === 'function' && !globalAny.Bun && !globalAny.Deno) { @@ -185,7 +185,7 @@ export function registerDiagnosticsChannelInjection(): void { mod.register(hookSpecifier, { parentURL, - data: { instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS, diagnosticsPort }, + data: { instrumentations: SENTRY_INSTRUMENTATIONS, diagnosticsPort }, transferList: [diagnosticsPort], }); @@ -194,7 +194,7 @@ export function registerDiagnosticsChannelInjection(): void { // are resolved through the CJS machinery and never reach the ESM // register hook, so without this patch the file we want to instrument // loads untransformed. - new ModulePatch({ instrumentations: SENTRY_RUNTIME_INSTRUMENTATIONS }).patch(); + new ModulePatch({ instrumentations: SENTRY_INSTRUMENTATIONS }).patch(); debug.log('Registered diagnostics-channel injection via Module.register()'); } else { marker.runtimeUnavailable = true; diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index f28ad60c56cb..1fec4fb2c5ad 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -35,8 +35,6 @@ import { vercelAiConfig } from './vercel-ai'; // Kept sorted alphabetically by module so concurrent additions insert at different // points rather than all appending to the end (fewer merge conflicts). -import { MODULE_REGISTRATION_TRANSFORM } from './registration-only'; - /** * The orchestrion code-transform configs. Every instrumentable library is here * so the transform is all-or-nothing: whenever orchestrion is enabled, all of @@ -83,38 +81,6 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...vercelAiConfig, ]; -/** - * The subset of {@link SENTRY_INSTRUMENTATIONS} the RUNTIME loader - * (`@sentry/server-runtime-injection`'s `register`, reached via `--import` or - * `Sentry.init()`) can actually apply. - * - * Registration-only configs (native-channel libraries such as `ai` v7, - * `ioredis`, `@redis/client`, `mysql2`, `mongoose`) carry the custom - * `MODULE_REGISTRATION_TRANSFORM` operator. That operator is wired into the - * BUNDLER plugins only (see `orchestrion/bundler/moduleInjectedTransform.ts`, - * applied via `bundler/options.ts`'s `customTransforms`); the runtime loader's - * `initialize()` receives no custom transforms. Attempting one of these at - * runtime therefore throws `TypeError: transform is not a function`, which the - * loader misreports as the always-on "`@sentry/server-runtime-injection` was - * bundled ... loads uninstrumented" warning even though nothing is wrong. - * - * Excluding them at runtime is correct, not just a way to silence the warning: - * these libraries publish their own tracing channels, and their integrations - * subscribe through `setupOnce()` / `waitForTracingChannelBinding`, - * independently of the module-injected snippet. That snippet only fires - * `orchestrion.module-injected`, which drives the `setup()` / - * `invokeOrchestrionInstrumentation` path; for a native-channel version that - * path subscribes to the injected `orchestrion:*` channels the library never - * publishes — a no-op. So running these at runtime would add no spans. The - * snippet earns its keep only on the BUNDLER path — notably bundler-only SDKs - * (e.g. `@sentry/cloudflare`) that discover a loaded module via that event to - * instantiate its integration factory. `@sentry/node` registers its - * integrations statically, so it does not need it. - */ -export const SENTRY_RUNTIME_INSTRUMENTATIONS: InstrumentationConfig[] = SENTRY_INSTRUMENTATIONS.filter( - config => config.transform !== MODULE_REGISTRATION_TRANSFORM, -); - /** * The unique set of package names instrumented by `SENTRY_INSTRUMENTATIONS` * merged with any caller-provided `instrumentations` (e.g. `['mysql']`). From 157f3bba09982d88e2ec515f84d4c1eac626b08a Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 9 Sep 2026 14:07:31 +0200 Subject: [PATCH 7/7] test inputs and outputs --- .../test-applications/node-eve/tests/eve.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 1fee6d694312..3e7adc612c8a 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -56,6 +56,16 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to expect(executeTool?.attributes?.['gen_ai.operation.name']?.value).toBe('execute_tool'); expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('get_weather'); + // Inputs and outputs are recorded with the SDK's default data collection (no + // `dataCollection` override), for both the model call and the tool call. + expect(invokeAgent?.attributes?.['gen_ai.input.messages']?.value).toContain('What is the weather in Paris?'); + expect(typeof invokeAgent?.attributes?.['gen_ai.output.messages']?.value).toBe('string'); + expect(String(invokeAgent?.attributes?.['gen_ai.output.messages']?.value ?? '')).not.toBe(''); + + expect(executeTool?.attributes?.['gen_ai.tool.call.arguments']?.value).toContain('Paris'); + // The tool returns `{ city, condition: 'Sunny', temperatureC: 22 }`. + expect(executeTool?.attributes?.['gen_ai.tool.call.result']?.value).toContain('Sunny'); + // The agent turn is captured as an http.server span on one of eve's two agent // request paths (the other http.server spans — health and the event stream — // are not in this trace).