test(eve): Add e2e test for node-eve

.github/workflows/build.yml

@@ -1149,6 +1149,8 @@ jobs:
       REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
       E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
+      # Used by the `node-eve` test app to make real model calls through OpenRouter
+      E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.job_build.outputs.e2e-matrix-optional) }}

dev-packages/e2e-tests/.env.example

@@ -11,3 +11,7 @@ E2E_TEST_SENTRY_ORG_SLUG=
 
 # A Sentry project slug
 E2E_TEST_SENTRY_PROJECT=
+
+# An OpenRouter API key, used by the `node-eve` test app to make real model calls.
+# Only needed to run that test app locally.
+E2E_OPENROUTER_API_KEY=

dev-packages/e2e-tests/run.ts

@@ -260,7 +260,7 @@ async function run(): Promise<void> {
     await asyncExec(testCommand, { env: appEnv, cwd });
 
     // clean up (although this is tmp, still nice to do)
-    await rm(tmpDirPath, { recursive: true });
+    await rm(tmpDirPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 300 });
   }
 }
 

dev-packages/e2e-tests/test-applications/node-eve/.gitignore

@@ -0,0 +1,10 @@
+node_modules
+.eve
+.output
+.nitro
+.vercel
+.data
+*.tsbuildinfo
+results.junit.xml
+test-results
+playwright-report

dev-packages/e2e-tests/test-applications/node-eve/agent/agent.ts

@@ -0,0 +1,36 @@
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+import { defineAgent } from 'eve';
+
+// We call OpenRouter directly (rather than the default Vercel AI Gateway) so the
+// e2e test needs only a single OpenRouter key. eve resolves this authored
+// `LanguageModel` at runtime.
+const openrouter = createOpenRouter({
+  apiKey: process.env.E2E_OPENROUTER_API_KEY,
+});
+
+const useOrchestrion = process.env.USE_ORCHESTRION === '1';
+
+export default defineAgent({
+  model: openrouter('openai/gpt-4o-mini'),
+  // A direct-provider model is not in the AI Gateway catalog, so eve cannot look
+  // up its context window for compaction. Provide it explicitly.
+  modelContextWindowTokens: 128_000,
+  build: {
+    // Only configure externals for orchestrion mode, to ensure everything else works without it
+    ...(useOrchestrion
+      ? {
+          // `dataloader` is instrumented by Sentry via orchestrion (a module
+          // transform). Keep it external so it stays a real module the transform can
+          // hook; if eve inlined it into the server bundle it could never be
+          // instrumented. (The Vercel AI SDK needs none of this — it uses a native
+          // diagnostics channel.)
+          //
+          // Do NOT add `@sentry/server-runtime-injection` here: the `--import`
+          // loader instruments regardless (so the "bundled ... uninstrumented"
+          // warning is a false positive), and externalizing it makes eve's dev
+          // host fail to resolve its `/register` subpath (`eve dev` only).
+          externalDependencies: ['dataloader'],
+        }
+      : {}),
+  },
+});

dev-packages/e2e-tests/test-applications/node-eve/agent/channels/eve.ts

@@ -0,0 +1,8 @@
+import { none } from 'eve/channels/auth';
+import { eveChannel } from 'eve/channels/eve';
+
+// The test drives the agent over localhost in both dev and prod, so the channel
+// is left open. Do not copy this into a real deployment.
+export default eveChannel({
+  auth: [none()],
+});

dev-packages/e2e-tests/test-applications/node-eve/agent/instructions.md

@@ -0,0 +1,8 @@
+You are a concise assistant used by an automated end-to-end test.
+
+- When the user asks about the weather in a place, call the `get_weather` tool
+  for that place and answer in one short sentence using its result.
+- When the user asks to count items, call the `count_items` tool with the item names.
+- When the user asks you to trigger a failure, call the `fail_now` tool.
+
+Do not ask follow-up questions.

dev-packages/e2e-tests/test-applications/node-eve/agent/instrumentation.ts

@@ -0,0 +1,18 @@
+import * as Sentry from '@sentry/node';
+
+// eve auto-discovers `agent/instrumentation.ts` and runs it at server startup,
+// before it loads the agent (and the `ai` SDK). That is early enough for the
+// Sentry SDK to install its instrumentation, so no `--import` / `NODE_OPTIONS`
+// bootstrap is needed. eve's own OpenTelemetry pipeline is intentionally left
+// unused: the gen_ai spans come from Sentry's `ai` instrumentation, not OTel.
+Sentry.init({
+  environment: 'qa',
+  dsn: process.env.E2E_TEST_DSN,
+  tunnel: 'http://localhost:3031/', // proxy server
+  tracesSampleRate: 1.0,
+  // Not a default integration. It only produces spans in the "orchestrion" test
+  // variant, where the server is started with
+  // `NODE_OPTIONS=--import=@sentry/node/import` so the orchestrion module
+  // transform is registered before `dataloader` loads.
+  integrations: [Sentry.dataloaderIntegration()],
+});

dev-packages/e2e-tests/test-applications/node-eve/agent/tools/count_items.ts

@@ -0,0 +1,17 @@
+import DataLoader from 'dataloader';
+import { defineTool } from 'eve/tools';
+import { z } from 'zod';
+
+// Uses `dataloader` so the e2e test can assert Sentry's orchestrion-based
+// instrumentation of it. Unlike the Vercel AI SDK (native diagnostics channel),
+// orchestrion packages are only instrumented when the Sentry loader is
+// registered at process start (the "orchestrion" test variant).
+export default defineTool({
+  description: 'Count the number of letters in each given name. Call this when asked to count items.',
+  inputSchema: z.object({ names: z.array(z.string()).min(1) }),
+  async execute({ names }) {
+    const loader = new DataLoader<string, number>(async keys => keys.map(k => k.length));
+    const counts = await Promise.all(names.map(n => loader.load(n)));
+    return { counts };
+  },
+});

dev-packages/e2e-tests/test-applications/node-eve/agent/tools/fail_now.ts

@@ -0,0 +1,10 @@
+import { defineTool } from 'eve/tools';
+import { z } from 'zod';
+
+export default defineTool({
+  description: 'Always throws an error. Call this when the user asks to trigger a failure.',
+  inputSchema: z.object({}),
+  async execute() {
+    throw new Error('Intentional eve tool failure');
+  },
+});

dev-packages/e2e-tests/test-applications/node-eve/agent/tools/get_weather.ts

@@ -0,0 +1,10 @@
+import { defineTool } from 'eve/tools';
+import { z } from 'zod';
+
+export default defineTool({
+  description: 'Get the current weather for a city.',
+  inputSchema: z.object({ city: z.string().min(1) }),
+  async execute({ city }) {
+    return { city, condition: 'Sunny', temperatureC: 22 };
+  },
+});

dev-packages/e2e-tests/test-applications/node-eve/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "node-eve",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "imports": {
+    "#*": "./agent/*"
+  },
+  "scripts": {
+    "build": "EVE_TELEMETRY_DISABLED=1 eve build",
+    "dev": "EVE_TELEMETRY_DISABLED=1 eve dev --no-ui --port 3030",
+    "start": "EVE_TELEMETRY_DISABLED=1 eve start --port 3030",
+    "dev:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm dev",
+    "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start",
+    "clean": "npx rimraf node_modules .eve .output pnpm-lock.yaml",
+    "test:build": "pnpm install && pnpm build",
+    "test:build-orchestrion": "USE_ORCHESTRION=1 pnpm test:build",
+    "test:build-latest": "pnpm install && pnpm add eve@latest ai@latest && pnpm build",
+    "test:assert": "pnpm test:prod && pnpm test:dev",
+    "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert",
+    "test:prod": "TEST_ENV=production playwright test",
+    "test:dev": "TEST_ENV=development playwright test"
+  },
+  "dependencies": {
+    "@openrouter/ai-sdk-provider": "^3.0.0",
+    "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
+    "ai": "^7.0.82",
+    "dataloader": "^2.2.3",
+    "eve": "^0.52.3",
+    "zod": "4.5.4"
+  },
+  "devDependencies": {
+    "@playwright/test": "~1.56.0",
+    "@sentry-internal/test-utils": "link:../../../test-utils",
+    "@sentry/core": "file:../../packed/sentry-core-packed.tgz",
+    "@types/node": "24.x",
+    "typescript": "~5.9.0"
+  },
+  "engines": {
+    "node": "24.x"
+  },
+  "volta": {
+    "node": "24.15.0",
+    "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true,
+    "optionalVariants": [
+      {
+        "build-command": "pnpm test:build-latest",
+        "label": "node-eve (latest)"
+      },
+      {
+        "build-command": "pnpm test:build-orchestrion",
+        "assert-command": "pnpm test:assert-orchestrion",
+        "label": "node-eve (orchestrion)"
+      }
+    ]
+  }
+}

dev-packages/e2e-tests/test-applications/node-eve/playwright.config.mjs

@@ -0,0 +1,24 @@
+import { getPlaywrightConfig } from '@sentry-internal/test-utils';
+
+const testEnv = process.env.TEST_ENV;
+const useOrchestrion = process.env.USE_ORCHESTRION === '1';
+
+if (!testEnv) {
+  throw new Error('No test env defined');
+}
+
+let startCommand = testEnv === 'development' ? 'pnpm dev' : 'pnpm start';
+
+if (useOrchestrion) {
+  startCommand = `${startCommand}:orchestrion`;
+}
+
+const config = getPlaywrightConfig(
+  { startCommand },
+  // Each test drives a real OpenRouter tool-calling turn (two model calls) and
+  // then waits for the streamed spans to flush, which does not fit the default
+  // 30s test timeout when the provider is slow.
+  { timeout: 90_000 },
+);
+
+export default config;

dev-packages/e2e-tests/test-applications/node-eve/start-event-proxy.mjs

@@ -0,0 +1,6 @@
+import { startEventProxyServer } from '@sentry-internal/test-utils';
+
+startEventProxyServer({
+  port: 3031,
+  proxyServerName: 'node-eve',
+});

dev-packages/e2e-tests/test-applications/node-eve/tests/dataloader.test.ts

@@ -0,0 +1,40 @@
+import { expect, test } from '@playwright/test';
+import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
+import { runAgentTurn } from './utils';
+
+const APP = 'node-eve';
+const useOrchestrion = process.env.USE_ORCHESTRION === '1';
+
+const isDataloaderSpan = (span: { attributes?: Record<string, { value?: unknown }> }): boolean =>
+  getSpanOp(span) === 'cache.get' && span.attributes?.['sentry.origin']?.value === 'auto.db.dataloader';
+
+/**
+ * `dataloader` is instrumented by Sentry via orchestrion (a module transform),
+ * unlike the Vercel AI SDK which publishes to a native diagnostics channel.
+ * Under eve's bundled server output the transform only runs when the Sentry
+ * loader is registered at process start via
+ * `NODE_OPTIONS=--import=@sentry/node/import` (the `node-eve (orchestrion)`
+ * variant, `USE_ORCHESTRION=1`). Without that bootstrap no dataloader span is
+ * captured, so this test is expected to fail — see `test.fail(!useOrchestrion)`.
+ */
+test('captures orchestrion-instrumented dataloader spans (requires the --import bootstrap)', async ({ baseURL }) => {
+  test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import');
+
+  // Accumulate the workflow trace's spans across envelopes. With orchestrion we
+  // wait for the dataloader span itself; without it that span never arrives, so
+  // anchor on the (always-present) tool-execution span and let the assertion
+  // below fail fast rather than time out.
+  const spansPromise = collectStreamedSpans(APP, spansOfTrace =>
+    useOrchestrion
+      ? spansOfTrace.some(isDataloaderSpan)
+      : spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'),
+  );
+
+  await runAgentTurn(baseURL!, 'Count items: apple, banana, cherry');
+
+  const spans = await spansPromise;
+  const dataloaderSpan = spans.find(isDataloaderSpan);
+
+  expect(dataloaderSpan?.attributes?.['sentry.op']?.value).toBe('cache.get');
+  expect(dataloaderSpan?.attributes?.['sentry.origin']?.value).toBe('auto.db.dataloader');
+});