diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a9e79c085c5..23fb501d72b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1149,6 +1149,8 @@ jobs: REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }} E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' + # Used by the `node-eve` test app to make real model calls through OpenRouter + E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} strategy: fail-fast: false matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index c598b7cbf597..13eff7352024 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -11,3 +11,7 @@ E2E_TEST_SENTRY_ORG_SLUG= # A Sentry project slug E2E_TEST_SENTRY_PROJECT= + +# An OpenRouter API key, used by the `node-eve` test app to make real model calls. +# Only needed to run that test app locally. +E2E_OPENROUTER_API_KEY= diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 5a44ea9a9416..e2c56813a3bf 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -260,7 +260,7 @@ async function run(): Promise { await asyncExec(testCommand, { env: appEnv, cwd }); // clean up (although this is tmp, still nice to do) - await rm(tmpDirPath, { recursive: true }); + await rm(tmpDirPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 300 }); } } diff --git a/dev-packages/e2e-tests/test-applications/node-eve/.gitignore b/dev-packages/e2e-tests/test-applications/node-eve/.gitignore new file mode 100644 index 000000000000..8301375b6738 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/.gitignore @@ -0,0 +1,10 @@ +node_modules +.eve +.output +.nitro +.vercel +.data +*.tsbuildinfo +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts new file mode 100644 index 000000000000..d4bab74b9da6 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts @@ -0,0 +1,41 @@ +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import { defineAgent } from 'eve'; + +const apiKey = process.env.E2E_OPENROUTER_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENROUTER_API_KEY is not set'); +} + +// We call OpenRouter directly (rather than the default Vercel AI Gateway) so the +// e2e test needs only a single OpenRouter key. eve resolves this authored +// `LanguageModel` at runtime. +const openrouter = createOpenRouter({ + apiKey, +}); + +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +export default defineAgent({ + model: openrouter('openai/gpt-4o-mini'), + // A direct-provider model is not in the AI Gateway catalog, so eve cannot look + // up its context window for compaction. Provide it explicitly. + modelContextWindowTokens: 128_000, + build: { + // Only configure externals for orchestrion mode, to ensure everything else works without it + ...(useOrchestrion + ? { + // `dataloader` is instrumented by Sentry via orchestrion (a module + // transform). Keep it external so it stays a real module the transform can + // hook; if eve inlined it into the server bundle it could never be + // instrumented. (The Vercel AI SDK needs none of this — it uses a native + // diagnostics channel.) + // + // Do NOT add `@sentry/server-runtime-injection` here: the `--import` + // loader instruments regardless (so the "bundled ... uninstrumented" + // warning is a false positive), and externalizing it makes eve's dev + // host fail to resolve its `/register` subpath (`eve dev` only). + externalDependencies: ['dataloader'], + } + : {}), + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts new file mode 100644 index 000000000000..898e308d4394 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts @@ -0,0 +1,8 @@ +import { none } from 'eve/channels/auth'; +import { eveChannel } from 'eve/channels/eve'; + +// The test drives the agent over localhost in both dev and prod, so the channel +// is left open. Do not copy this into a real deployment. +export default eveChannel({ + auth: [none()], +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md new file mode 100644 index 000000000000..e67cae081e40 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md @@ -0,0 +1,8 @@ +You are a concise assistant used by an automated end-to-end test. + +- When the user asks about the weather in a place, call the `get_weather` tool + for that place and answer in one short sentence using its result. +- When the user asks to count items, call the `count_items` tool with the item names. +- When the user asks you to trigger a failure, call the `fail_now` tool. + +Do not ask follow-up questions. diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts new file mode 100644 index 000000000000..e2d33fd49591 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts @@ -0,0 +1,18 @@ +import * as Sentry from '@sentry/node'; + +// eve auto-discovers `agent/instrumentation.ts` and runs it at server startup, +// before it loads the agent (and the `ai` SDK). That is early enough for the +// Sentry SDK to install its instrumentation, so no `--import` / `NODE_OPTIONS` +// bootstrap is needed. eve's own OpenTelemetry pipeline is intentionally left +// unused: the gen_ai spans come from Sentry's `ai` instrumentation, not OTel. +Sentry.init({ + environment: 'qa', + dsn: process.env.E2E_TEST_DSN, + tunnel: 'http://localhost:3031/', // proxy server + tracesSampleRate: 1.0, + // Not a default integration. It only produces spans in the "orchestrion" test + // variant, where the server is started with + // `NODE_OPTIONS=--import=@sentry/node/import` so the orchestrion module + // transform is registered before `dataloader` loads. + integrations: [Sentry.dataloaderIntegration()], +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts new file mode 100644 index 000000000000..47ec2d31acdd --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts @@ -0,0 +1,17 @@ +import DataLoader from 'dataloader'; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; + +// Uses `dataloader` so the e2e test can assert Sentry's orchestrion-based +// instrumentation of it. Unlike the Vercel AI SDK (native diagnostics channel), +// orchestrion packages are only instrumented when the Sentry loader is +// registered at process start (the "orchestrion" test variant). +export default defineTool({ + description: 'Count the number of letters in each given name. Call this when asked to count items.', + inputSchema: z.object({ names: z.array(z.string()).min(1) }), + async execute({ names }) { + const loader = new DataLoader(async keys => keys.map(k => k.length)); + const counts = await Promise.all(names.map(n => loader.load(n))); + return { counts }; + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts new file mode 100644 index 000000000000..ec9b6aeae782 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts @@ -0,0 +1,10 @@ +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; + +export default defineTool({ + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + inputSchema: z.object({}), + async execute() { + throw new Error('Intentional eve tool failure'); + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts new file mode 100644 index 000000000000..10d29e22a7f7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts @@ -0,0 +1,10 @@ +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; + +export default defineTool({ + description: 'Get the current weather for a city.', + inputSchema: z.object({ city: z.string().min(1) }), + async execute({ city }) { + return { city, condition: 'Sunny', temperatureC: 22 }; + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/package.json b/dev-packages/e2e-tests/test-applications/node-eve/package.json new file mode 100644 index 000000000000..e95a1aaafe41 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/package.json @@ -0,0 +1,60 @@ +{ + "name": "node-eve", + "version": "0.0.0", + "private": true, + "type": "module", + "imports": { + "#*": "./agent/*" + }, + "scripts": { + "build": "EVE_TELEMETRY_DISABLED=1 eve build", + "dev": "EVE_TELEMETRY_DISABLED=1 eve dev --no-ui --port 3030", + "start": "EVE_TELEMETRY_DISABLED=1 eve start --port 3030", + "dev:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm dev", + "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start", + "clean": "npx rimraf node_modules .eve .output pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-orchestrion": "USE_ORCHESTRION=1 pnpm test:build", + "test:build-latest": "pnpm install && pnpm add eve@latest ai@latest && pnpm build", + "test:assert": "pnpm test:prod && pnpm test:dev", + "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert", + "test:prod": "TEST_ENV=production playwright test", + "test:dev": "TEST_ENV=development playwright test" + }, + "dependencies": { + "@openrouter/ai-sdk-provider": "^3.0.0", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "ai": "^7.0.82", + "dataloader": "^2.2.3", + "eve": "^0.52.3", + "zod": "4.5.4" + }, + "devDependencies": { + "@playwright/test": "~1.56.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/core": "file:../../packed/sentry-core-packed.tgz", + "@types/node": "24.x", + "typescript": "~5.9.0" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-eve (latest)" + }, + { + "build-command": "pnpm test:build-orchestrion", + "assert-command": "pnpm test:assert-orchestrion", + "label": "node-eve (orchestrion)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs new file mode 100644 index 000000000000..fabff63d3749 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs @@ -0,0 +1,24 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const testEnv = process.env.TEST_ENV; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +if (!testEnv) { + throw new Error('No test env defined'); +} + +let startCommand = testEnv === 'development' ? 'pnpm dev' : 'pnpm start'; + +if (useOrchestrion) { + startCommand = `${startCommand}:orchestrion`; +} + +const config = getPlaywrightConfig( + { startCommand }, + // Each test drives a real OpenRouter tool-calling turn (two model calls) and + // then waits for the streamed spans to flush, which does not fit the default + // 30s test timeout when the provider is slow. + { timeout: 90_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs new file mode 100644 index 000000000000..1c059695bb90 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-eve', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts new file mode 100644 index 000000000000..409ad8349aae --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-eve'; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +const isDataloaderSpan = (span: { attributes?: Record }): boolean => + getSpanOp(span) === 'cache.get' && span.attributes?.['sentry.origin']?.value === 'auto.db.dataloader'; + +/** + * `dataloader` is instrumented by Sentry via orchestrion (a module transform), + * unlike the Vercel AI SDK which publishes to a native diagnostics channel. + * Under eve's bundled server output the transform only runs when the Sentry + * loader is registered at process start via + * `NODE_OPTIONS=--import=@sentry/node/import` (the `node-eve (orchestrion)` + * variant, `USE_ORCHESTRION=1`). Without that bootstrap no dataloader span is + * captured, so this test is expected to fail — see `test.fail(!useOrchestrion)`. + */ +test('captures orchestrion-instrumented dataloader spans (requires the --import bootstrap)', async ({ baseURL }) => { + test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import'); + + // Accumulate the workflow trace's spans across envelopes. With orchestrion we + // wait for the dataloader span itself; without it that span never arrives, so + // anchor on the (always-present) tool-execution span and let the assertion + // below fail fast rather than time out. + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + useOrchestrion + ? spansOfTrace.some(isDataloaderSpan) + : spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + ); + + await runAgentTurn(baseURL!, 'Count items: apple, banana, cherry'); + + const spans = await spansPromise; + const dataloaderSpan = spans.find(isDataloaderSpan); + + expect(dataloaderSpan?.attributes?.['sentry.op']?.value).toBe('cache.get'); + expect(dataloaderSpan?.attributes?.['sentry.origin']?.value).toBe('auto.db.dataloader'); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts new file mode 100644 index 000000000000..3e7adc612c8a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -0,0 +1,118 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-eve'; + +// eve serves one agent turn across two request contexts — the session API +// request (`POST /eve/v1/session`) and the internal durable-workflow request +// (`POST /.well-known/workflow/v1/flow`) — so the captured server path can be +// either one, depending on eve's workflow scheduling. +const EVE_AGENT_PATH = /(\/eve\/v1\/session|\/\.well-known\/workflow\/v1\/flow)/; + +// The agent turn is served by a POST to one of eve's two agent paths. Requiring +// POST keeps the GET spans (health, the event stream) out even though +// EVE_AGENT_PATH also matches the stream path. +const isAgentServerSpan = (span: { attributes?: Record }): boolean => + getSpanOp(span) === 'http.server' && + span.attributes?.['http.request.method']?.value === 'POST' && + EVE_AGENT_PATH.test(String(span.attributes?.['url.path']?.value ?? '')); + +test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_tool) for an eve turn', async ({ + baseURL, +}) => { + // The gen_ai spans and the agent http.server span share one trace, but the + // still-open invoke_agent parent flushes on a timer in a separate envelope + // from its completed children. `collectStreamedSpans` accumulates a trace's + // spans across envelopes (unlike `waitForStreamedSpans`, which sees one + // envelope at a time), so we wait until the whole trace has arrived. + const traceSpansPromise = collectStreamedSpans( + APP, + spansOfTrace => + ['gen_ai.invoke_agent', 'gen_ai.generate_content', 'gen_ai.execute_tool'].every(op => + spansOfTrace.some(span => getSpanOp(span) === op), + ) && spansOfTrace.some(isAgentServerSpan), + ); + + await runAgentTurn(baseURL!, 'What is the weather in Paris?'); + + const traceSpans = await traceSpansPromise; + + const invokeAgent = traceSpans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const generateContent = traceSpans.find(span => getSpanOp(span) === 'gen_ai.generate_content'); + const executeTool = traceSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); + + expect(invokeAgent?.attributes?.['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(invokeAgent?.attributes?.['gen_ai.operation.name']?.value).toBe('invoke_agent'); + expect(invokeAgent?.attributes?.['gen_ai.request.model']?.value).toBe('openai/gpt-4o-mini'); + expect(invokeAgent?.attributes?.['gen_ai.provider.name']?.value).toBe('openrouter'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe('number'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe('number'); + expect(typeof invokeAgent?.attributes?.['gen_ai.usage.total_tokens']?.value).toBe('number'); + + expect(generateContent?.attributes?.['gen_ai.operation.name']?.value).toBe('generate_content'); + expect(generateContent?.attributes?.['gen_ai.request.model']?.value).toBe('openai/gpt-4o-mini'); + + expect(executeTool?.attributes?.['gen_ai.operation.name']?.value).toBe('execute_tool'); + expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('get_weather'); + + // Inputs and outputs are recorded with the SDK's default data collection (no + // `dataCollection` override), for both the model call and the tool call. + expect(invokeAgent?.attributes?.['gen_ai.input.messages']?.value).toContain('What is the weather in Paris?'); + expect(typeof invokeAgent?.attributes?.['gen_ai.output.messages']?.value).toBe('string'); + expect(String(invokeAgent?.attributes?.['gen_ai.output.messages']?.value ?? '')).not.toBe(''); + + expect(executeTool?.attributes?.['gen_ai.tool.call.arguments']?.value).toContain('Paris'); + // The tool returns `{ city, condition: 'Sunny', temperatureC: 22 }`. + expect(executeTool?.attributes?.['gen_ai.tool.call.result']?.value).toContain('Sunny'); + + // The agent turn is captured as an http.server span on one of eve's two agent + // request paths (the other http.server spans — health and the event stream — + // are not in this trace). + const agentServerSpans = traceSpans.filter(isAgentServerSpan); + expect(agentServerSpans.length).toBeGreaterThanOrEqual(1); + + for (const span of agentServerSpans) { + expect(span).toMatchObject({ + // no parametrization available, so the server span name is just the method + name: 'POST', + attributes: { + 'sentry.origin': { value: 'auto.http.http_server', type: 'string' }, + 'sentry.op': { value: 'http.server', type: 'string' }, + 'sentry.segment.name.source': { value: 'url', type: 'string' }, + 'sentry.kind': { value: 'server', type: 'string' }, + 'url.path': { value: expect.stringMatching(EVE_AGENT_PATH), type: 'string' }, + 'http.request.method': { value: 'POST', type: 'string' }, + }, + }); + } +}); + +test('captures errors thrown inside an eve tool', async ({ baseURL }) => { + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value?.includes('Intentional eve tool failure') ?? false, + ); + + await runAgentTurn(baseURL!, 'Please call the tool that triggers a failure now.'); + + const error = await errorPromise; + + expect(error).toMatchObject({ + exception: { + values: [ + { + type: 'Error', + value: expect.stringContaining('Intentional eve tool failure'), + mechanism: { + type: 'auto.vercelai.channel', + handled: false, + }, + }, + ], + }, + // The tool runs inside eve's durable workflow, so the error is attributed to + // either the session request or the internal workflow request. + transaction: expect.stringMatching(EVE_AGENT_PATH), + }); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts new file mode 100644 index 000000000000..6e3f0bd7a301 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts @@ -0,0 +1,39 @@ +import { expect } from '@playwright/test'; + +/** + * Drive one agent turn through eve's default HTTP channel and wait for it to + * settle, so the agent has finished and its spans have been flushed before we + * assert. eve runs the turn in a durable workflow, so the POST only needs to be + * accepted; we drain the event stream to know when the turn is done. + */ +export async function runAgentTurn(baseURL: string, message: string): Promise { + const createRes = await fetch(`${baseURL}/eve/v1/session`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message }), + }); + expect(createRes.status).toBe(202); + const { sessionId } = (await createRes.json()) as { sessionId: string }; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 25_000); + try { + const streamRes = await fetch(`${baseURL}/eve/v1/session/${sessionId}/stream`, { + signal: controller.signal, + }); + const reader = streamRes.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + if (buffer.includes('"type":"session.waiting"') || buffer.includes('"type":"turn.failed"')) { + break; + } + } + await reader.cancel().catch(() => {}); + } finally { + clearTimeout(timer); + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json new file mode 100644 index 000000000000..79911c330c61 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "types": ["node", "eve/workflow-modules"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["agent/**/*.ts"] +}