From c237b437ea426ea81603de80cc61beeb6eb175c7 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Tue, 1 Sep 2026 12:26:47 +0200 Subject: [PATCH] test(e2e): Port cloudflare-agent to span streaming Co-Authored-By: Claude Fable 5 --- .../tests/ai-streaming.test.ts | 27 +- .../cloudflare-agent/tests/callable.test.ts | 253 +++++++++--------- .../tests/chat-conversation.test.ts | 34 ++- .../cloudflare-agent/tests/chat-rpc.test.ts | 24 +- .../cloudflare-agent/worker/index.ts | 1 - 5 files changed, 175 insertions(+), 164 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/ai-streaming.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/ai-streaming.test.ts index e09239d6107b..a2d3c37b66dc 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/ai-streaming.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/ai-streaming.test.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan, waitForTransaction } from '@sentry-internal/test-utils'; +import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; import type { SerializedStreamedSpan } from '@sentry/core'; // Drives Workers AI through the real Cloudflare Agents SDK + Vercel AI SDK + `workers-ai-provider` @@ -26,32 +26,41 @@ function assertGenAiStreamingSpan(span: SerializedStreamedSpan): void { expect(span.attributes['gen_ai.conversation.id']?.value).toMatch(/^[0-9a-f]{32}$/); } +// With span streaming, URL-sourced `http.server` spans are named by method only, so the request +// segments below are identified by their `url.path` attribute. + test('captures Workers AI streaming output when driven via an Agent', async ({ request, baseURL }) => { const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); - const transactionPromise = waitForTransaction( + const requestSpanPromise = waitForStreamedSpan( 'cloudflare-agent', - transactionEvent => transactionEvent.transaction === 'GET /agents/my-agent/test', + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-agent/test', ); const response = await request.get(`${baseURL}/agents/my-agent/test`); expect(response.ok()).toBe(true); - const [span, transaction] = await Promise.all([spanPromise, transactionPromise]); - expect(span.trace_id).toBe(transaction.contexts?.trace?.trace_id); + const [span, requestSpan] = await Promise.all([spanPromise, requestSpanPromise]); + expect(span.trace_id).toBe(requestSpan.trace_id); assertGenAiStreamingSpan(span); }); test('captures Workers AI streaming output when driven via an AIChatAgent', async ({ request, baseURL }) => { const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); - const transactionPromise = waitForTransaction( + const requestSpanPromise = waitForStreamedSpan( 'cloudflare-agent', - transactionEvent => transactionEvent.transaction === 'GET /agents/my-chat-agent/test', + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-chat-agent/test', ); const response = await request.get(`${baseURL}/agents/my-chat-agent/test`); expect(response.ok()).toBe(true); - const [span, transaction] = await Promise.all([spanPromise, transactionPromise]); - expect(span.trace_id).toBe(transaction.contexts?.trace?.trace_id); + const [span, requestSpan] = await Promise.all([spanPromise, requestSpanPromise]); + expect(span.trace_id).toBe(requestSpan.trace_id); assertGenAiStreamingSpan(span); }); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/callable.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/callable.test.ts index c5540d717d4b..1051cf8282c5 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/callable.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/callable.test.ts @@ -1,21 +1,29 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; + +// The agent request segment is the Durable Object's `http.server` span. It has a parent because +// the worker propagates its trace over the RPC binding; the worker's own segment for the same URL +// does not. With span streaming the name is the method only, so the segment is picked by `url.path`. test('@callable() methods work correctly with Sentry instrumentAgentWithSentry', async ({ page, baseURL }) => { - const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { - return ( - transactionEvent.transaction === 'GET /agents/my-agent/user-123' && - transactionEvent.contexts?.trace?.parent_span_id !== undefined - ); - }); + const requestSpanPromise = waitForStreamedSpan( + 'cloudflare-agent', + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-agent/user-123' && + span.parent_span_id !== undefined, + ); - // The greet() call goes over the websocket, so its storage spans land in a webSocketMessage - // transaction. Filter for the one carrying our put span — control messages produce their own - // webSocketMessage transactions without storage spans. - const storageTransactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { + // The greet() call goes over the websocket, so its storage spans are children of the `greet` rpc + // span inside a webSocketMessage segment. Control messages produce their own webSocketMessage + // segments without storage spans. Streamed children arrive before their segment, so collect until + // the segment that closes the trace of a `greet` call has arrived. + const storageSpansPromise = collectStreamedSpans('cloudflare-agent', spans => { + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet'); return ( - transactionEvent.transaction === 'webSocketMessage' && - (transactionEvent.spans ?? []).some(span => span.description === 'durable_object_storage_put') + !!rpcSpan && + spans.some(span => span.is_segment && span.name === 'webSocketMessage' && span.trace_id === rpcSpan.trace_id) ); }); @@ -25,110 +33,91 @@ test('@callable() methods work correctly with Sentry instrumentAgentWithSentry', await page.getByRole('button', { name: 'Call Agent' }).click(); await expect(page.getByText('Hello, World!')).toBeVisible(); - const transaction = await transactionPromise; - - expect(transaction).toEqual({ - contexts: { - trace: { - parent_span_id: expect.stringMatching(/[a-f0-9]{16}/), - span_id: expect.stringMatching(/[a-f0-9]{16}/), - trace_id: expect.stringMatching(/[a-f0-9]{32}/), - data: expect.any(Object), - op: 'http.server', - status: 'ok', - origin: 'auto.http.cloudflare', - }, - cloud_resource: { 'cloud.provider': 'cloudflare' }, - culture: { timezone: expect.any(String) }, - runtime: { name: 'cloudflare' }, - }, - spans: [], + const requestSpan = await requestSpanPromise; + + expect(requestSpan).toEqual({ + trace_id: expect.stringMatching(/[a-f0-9]{32}/), + span_id: expect.stringMatching(/[a-f0-9]{16}/), + parent_span_id: expect.stringMatching(/[a-f0-9]{16}/), + name: 'GET', start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - transaction: 'GET /agents/my-agent/user-123', - type: 'transaction', - request: { - cookies: {}, - headers: expect.any(Object), - method: 'GET', - url: expect.stringContaining('/agents/my-agent/user-123'), - query_string: expect.any(String), - }, - transaction_info: { source: 'url' }, - platform: 'javascript', - event_id: expect.stringMatching(/[a-f0-9]{32}/), - environment: expect.any(String), - user: { - ip_address: '127.0.0.1', - }, - release: expect.any(String), - sdk: { - integrations: expect.any(Array), - name: 'sentry.javascript.cloudflare', - version: expect.any(String), - packages: expect.any(Array), - }, + end_timestamp: expect.any(Number), + status: 'ok', + is_segment: true, + attributes: expect.objectContaining({ + 'sentry.op': { value: 'http.server', type: 'string' }, + 'sentry.origin': { value: 'auto.http.cloudflare', type: 'string' }, + 'sentry.segment.name.source': { value: 'url', type: 'string' }, + 'http.request.method': { value: 'GET', type: 'string' }, + 'url.path': { value: '/agents/my-agent/user-123', type: 'string' }, + 'sentry.environment': { value: expect.any(String), type: 'string' }, + }), }); // greet() touches 6 storage keys: 2 user ops + 3 framework-internal keys (cf_, __ps_, /) that // must be filtered + 1 allowlisted cf_ key. Spans carry no key attribute, so filtering can only // be verified by count — exactly these 3 storage spans (in execution order) should survive, and // any framework-internal span leaking through shows up as an extra entry here. - const storageTransaction = await storageTransactionPromise; + const spans = await storageSpansPromise; + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet')!; - const storageSpans = (storageTransaction.spans ?? []).filter( - span => span.origin === 'auto.db.cloudflare.durable_object', - ); + const storageSpans = spans + .filter( + span => + span.parent_span_id === rpcSpan.span_id && + span.attributes['sentry.origin']?.value === 'auto.db.cloudflare.durable_object', + ) + .sort((a, b) => a.start_timestamp - b.start_timestamp); expect(storageSpans).toEqual([ - expect.objectContaining({ - data: { - 'db.operation.name': 'put', - 'db.system.name': 'cloudflare.durable_object.storage', - 'sentry.op': 'db', - 'sentry.origin': 'auto.db.cloudflare.durable_object', - }, - description: 'durable_object_storage_put', - op: 'db', - origin: 'auto.db.cloudflare.durable_object', - parent_span_id: expect.stringMatching(/[a-f0-9]{16}/), - span_id: expect.stringMatching(/[a-f0-9]{16}/), + { + name: 'durable_object_storage_put', + attributes: expect.objectContaining({ + 'db.operation.name': { value: 'put', type: 'string' }, + 'db.system.name': { value: 'cloudflare.durable_object.storage', type: 'string' }, + 'sentry.op': { value: 'db', type: 'string' }, + 'sentry.origin': { value: 'auto.db.cloudflare.durable_object', type: 'string' }, + }), + parent_span_id: rpcSpan.span_id, + span_id: expect.stringMatching(/^[a-f0-9]{16}$/), start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.stringMatching(/[a-f0-9]{32}/), - }), - expect.objectContaining({ - data: { - 'db.operation.name': 'get', - 'db.system.name': 'cloudflare.durable_object.storage', - 'sentry.op': 'db', - 'sentry.origin': 'auto.db.cloudflare.durable_object', - }, - description: 'durable_object_storage_get', - op: 'db', - origin: 'auto.db.cloudflare.durable_object', - parent_span_id: expect.stringMatching(/[a-f0-9]{16}/), - span_id: expect.stringMatching(/[a-f0-9]{16}/), + end_timestamp: expect.any(Number), + status: 'ok', + is_segment: false, + trace_id: rpcSpan.trace_id, + }, + { + name: 'durable_object_storage_get', + attributes: expect.objectContaining({ + 'db.operation.name': { value: 'get', type: 'string' }, + 'db.system.name': { value: 'cloudflare.durable_object.storage', type: 'string' }, + 'sentry.op': { value: 'db', type: 'string' }, + 'sentry.origin': { value: 'auto.db.cloudflare.durable_object', type: 'string' }, + }), + parent_span_id: rpcSpan.span_id, + span_id: expect.stringMatching(/^[a-f0-9]{16}$/), start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.stringMatching(/[a-f0-9]{32}/), - }), - expect.objectContaining({ - data: { - 'db.operation.name': 'get', - 'db.system.name': 'cloudflare.durable_object.storage', - 'sentry.op': 'db', - 'sentry.origin': 'auto.db.cloudflare.durable_object', - }, - description: 'durable_object_storage_get', - op: 'db', - origin: 'auto.db.cloudflare.durable_object', - parent_span_id: expect.stringMatching(/[a-f0-9]{16}/), - span_id: expect.stringMatching(/[a-f0-9]{16}/), + end_timestamp: expect.any(Number), + status: 'ok', + is_segment: false, + trace_id: rpcSpan.trace_id, + }, + { + name: 'durable_object_storage_get', + attributes: expect.objectContaining({ + 'db.operation.name': { value: 'get', type: 'string' }, + 'db.system.name': { value: 'cloudflare.durable_object.storage', type: 'string' }, + 'sentry.op': { value: 'db', type: 'string' }, + 'sentry.origin': { value: 'auto.db.cloudflare.durable_object', type: 'string' }, + }), + parent_span_id: rpcSpan.span_id, + span_id: expect.stringMatching(/^[a-f0-9]{16}$/), start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.stringMatching(/[a-f0-9]{32}/), - }), + end_timestamp: expect.any(Number), + status: 'ok', + is_segment: false, + trace_id: rpcSpan.trace_id, + }, ]); }); @@ -136,12 +125,15 @@ test('does not emit db.query spans for the agents runtime `cf_`-prefixed interna page, baseURL, }) => { - const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { - return ( - transactionEvent.transaction === 'GET /agents/my-agent/user-123' && - transactionEvent.contexts?.trace?.parent_span_id !== undefined - ); - }); + const spansPromise = collectStreamedSpans('cloudflare-agent', spans => + spans.some( + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-agent/user-123' && + span.parent_span_id !== undefined, + ), + ); await page.goto(baseURL!); @@ -149,22 +141,33 @@ test('does not emit db.query spans for the agents runtime `cf_`-prefixed interna await page.getByRole('button', { name: 'Call Agent' }).click(); await expect(page.getByText('Hello, World!')).toBeVisible(); - const transaction = await transactionPromise; + const spans = await spansPromise; + const requestSpan = spans.find( + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-agent/user-123' && + span.parent_span_id !== undefined, + )!; // The agents runtime constantly queries its own `cf_agents_*` / `cf_agent_*` bookkeeping tables. // These are framework internals and are filtered out by default, so no such span should leak. - const internalTableSpans = (transaction.spans ?? []).filter( - span => span.op === 'db.query' && /\bcf_/.test((span.data?.['db.query.summary'] as string) ?? ''), + const internalTableSpans = spans.filter( + span => + span.trace_id === requestSpan.trace_id && + getSpanOp(span) === 'db.query' && + /\bcf_/.test(String(span.attributes['db.query.summary']?.value ?? '')), ); expect(internalTableSpans).toEqual([]); }); test('creates an rpc span named after the @callable() method', async ({ page, baseURL }) => { - const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { + const spansPromise = collectStreamedSpans('cloudflare-agent', spans => { + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet'); return ( - transactionEvent.transaction === 'webSocketMessage' && - (transactionEvent.spans ?? []).some(span => span.op === 'rpc' && span.description === 'greet') + !!rpcSpan && + spans.some(span => span.is_segment && span.name === 'webSocketMessage' && span.trace_id === rpcSpan.trace_id) ); }); @@ -174,19 +177,13 @@ test('creates an rpc span named after the @callable() method', async ({ page, ba await page.getByRole('button', { name: 'Call Agent' }).click(); await expect(page.getByText('Hello, World!')).toBeVisible(); - const transaction = await transactionPromise; + const spans = await spansPromise; + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet')!; - const rpcSpans = (transaction.spans ?? []).filter(span => span.op === 'rpc'); + const rpcSpans = spans.filter(span => getSpanOp(span) === 'rpc' && span.trace_id === rpcSpan.trace_id); expect(rpcSpans).toHaveLength(1); - expect(rpcSpans[0]).toEqual( - expect.objectContaining({ - op: 'rpc', - description: 'greet', - origin: 'auto.faas.cloudflare.agents', - data: expect.objectContaining({ - 'gen_ai.agent.name': 'MyBaseAgent', - }), - }), - ); + expect(rpcSpan.attributes['sentry.op']?.value).toBe('rpc'); + expect(rpcSpan.attributes['sentry.origin']?.value).toBe('auto.faas.cloudflare.agents'); + expect(rpcSpan.attributes['gen_ai.agent.name']?.value).toBe('MyBaseAgent'); }); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-conversation.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-conversation.test.ts index 17eb088b815d..06db60704ce5 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-conversation.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-conversation.test.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan, waitForTransaction } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; import { callRpc, sendChatMessage } from './agent-socket'; const AGENT_INSTANCE = 'chat-conv-instance'; @@ -11,11 +11,15 @@ const AGENT_INSTANCE = 'chat-conv-instance'; const UUID_PATTERN = /^[0-9a-f]{32}$/; test('stamps the conversation id on gen_ai spans created inside a chat turn', async ({ baseURL }) => { - const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); - const transactionPromise = waitForTransaction( - 'cloudflare-agent', - transactionEvent => transactionEvent.transaction === 'webSocketMessage', - ); + // The gen_ai span is streamed before the webSocketMessage segment that owns it, so collect until + // the segment of the same trace has arrived; that also proves the span ran inside the chat turn. + const spansPromise = collectStreamedSpans('cloudflare-agent', spans => { + const genAiSpan = spans.find(span => getSpanOp(span) === 'gen_ai.chat'); + return ( + !!genAiSpan && + spans.some(span => span.is_segment && span.name === 'webSocketMessage' && span.trace_id === genAiSpan.trace_id) + ); + }); await sendChatMessage(baseURL!, { binding: 'my-chat-agent', @@ -23,8 +27,8 @@ test('stamps the conversation id on gen_ai spans created inside a chat turn', as prompt: 'What is the capital of France?', }); - const [genAiSpan, transaction] = await Promise.all([spanPromise, transactionPromise]); - expect(genAiSpan.trace_id).toBe(transaction.contexts?.trace?.trace_id); + const spans = await spansPromise; + const genAiSpan = spans.find(span => getSpanOp(span) === 'gen_ai.chat')!; expect(genAiSpan.attributes['gen_ai.conversation.id']?.value).toMatch(UUID_PATTERN); }); @@ -48,17 +52,21 @@ test('a conversation id set manually inside onChatMessage wins over the SDK-mint // which handler the agent uses for its AI work. test('a conversation id set manually inside onRequest wins over the SDK-minted one', async ({ request, baseURL }) => { const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); - const transactionPromise = waitForTransaction( + // With span streaming, URL-sourced `http.server` spans are named by method only, so the request + // segment is identified by its `url.path` attribute. + const requestSpanPromise = waitForStreamedSpan( 'cloudflare-agent', - transactionEvent => - transactionEvent.transaction === 'GET /agents/my-manual-chat-agent/chat-manual-request-instance', + span => + getSpanOp(span) === 'http.server' && + span.is_segment && + span.attributes['url.path']?.value === '/agents/my-manual-chat-agent/chat-manual-request-instance', ); const response = await request.get(`${baseURL}/agents/my-manual-chat-agent/chat-manual-request-instance`); expect(response.ok()).toBe(true); - const [genAiSpan, transaction] = await Promise.all([spanPromise, transactionPromise]); - expect(genAiSpan.trace_id).toBe(transaction.contexts?.trace?.trace_id); + const [genAiSpan, requestSpan] = await Promise.all([spanPromise, requestSpanPromise]); + expect(genAiSpan.trace_id).toBe(requestSpan.trace_id); expect(genAiSpan.attributes['gen_ai.conversation.id']?.value).toBe('conv_manual_e2e'); }); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-rpc.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-rpc.test.ts index 64dc206d004b..fc7e593341cf 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-rpc.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-rpc.test.ts @@ -1,27 +1,25 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; import { callRpc } from './agent-socket'; const AGENT_INSTANCE = 'chat-rpc-instance'; test('creates an rpc span for a @callable() invocation on an AIChatAgent', async ({ baseURL }) => { - const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { + // The rpc span is streamed before the webSocketMessage segment that owns it, so collect until the + // segment of the same trace has arrived. + const spansPromise = collectStreamedSpans('cloudflare-agent', spans => { + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet'); return ( - transactionEvent.transaction === 'webSocketMessage' && - (transactionEvent.spans ?? []).some(span => span.op === 'rpc' && span.description === 'greet') + !!rpcSpan && + spans.some(span => span.is_segment && span.name === 'webSocketMessage' && span.trace_id === rpcSpan.trace_id) ); }); await callRpc(baseURL!, { binding: 'my-chat-agent', instance: AGENT_INSTANCE, method: 'greet', args: ['World'] }); - const transaction = await transactionPromise; + const spans = await spansPromise; + const rpcSpan = spans.find(span => getSpanOp(span) === 'rpc' && span.name === 'greet')!; - const rpcSpan = (transaction.spans ?? []).find(span => span.op === 'rpc' && span.description === 'greet'); - expect(rpcSpan).toEqual( - expect.objectContaining({ - op: 'rpc', - description: 'greet', - origin: 'auto.faas.cloudflare.agents', - }), - ); + expect(rpcSpan.attributes['sentry.op']?.value).toBe('rpc'); + expect(rpcSpan.attributes['sentry.origin']?.value).toBe('auto.faas.cloudflare.agents'); }); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-agent/worker/index.ts b/dev-packages/e2e-tests/test-applications/cloudflare-agent/worker/index.ts index 66ea74f98cf3..f230af50eba0 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-agent/worker/index.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-agent/worker/index.ts @@ -8,7 +8,6 @@ import { MockAi } from './mocks'; const MODEL = '@cf/meta/llama-3.1-8b-instruct'; const sentryOptions = (env: Env) => ({ - traceLifecycle: 'static' as const, dsn: env.E2E_TEST_DSN, tunnel: `http://localhost:3031/`, tracesSampleRate: 1,