From 99049b2ea3398b3967be98cff246d33ae344fcc4 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 13 Jul 2026 16:34:37 -0400 Subject: [PATCH 1/3] feat(strands): log inline media as attachments Convert Strands document, image, and video bytes into Braintrust attachments while preserving the SDK payload shape. Reuse attachment references across agent and model spans with bounded caching to avoid duplicate uploads and unbounded retention. Add real SDK end-to-end coverage for document and image blocks, and share minimal media fixtures across scenarios. --- e2e/helpers/media-fixtures.mjs | 5 + .../openai-instrumentation/scenario.impl.mjs | 12 +- .../strands-agent-sdk-v160.cassette.json | 188 +++++++++++- ...ds-agent-sdk-v160-auto-hook.span-tree.json | 157 ++++++++++ ...nds-agent-sdk-v160-auto-hook.span-tree.txt | 131 +++++++++ ...ands-agent-sdk-v160-wrapped.span-tree.json | 157 ++++++++++ ...rands-agent-sdk-v160-wrapped.span-tree.txt | 131 +++++++++ .../assertions.ts | 98 +++++++ .../scenario.impl.mjs | 57 +++- js/src/instrumentation/plugins/eve-plugin.ts | 2 +- .../plugins/strands-agent-sdk-plugin.test.ts | 80 +++++- .../plugins/strands-agent-sdk-plugin.ts | 268 +++++++++++++++++- js/src/{prompt-cache => }/lru-cache.test.ts | 0 js/src/{prompt-cache => }/lru-cache.ts | 0 js/src/prompt-cache/cache-config.ts | 2 +- js/src/prompt-cache/parameters-cache.ts | 2 +- js/src/prompt-cache/prompt-cache.test.ts | 2 +- js/src/prompt-cache/prompt-cache.ts | 2 +- js/src/vendor-sdk-types/strands-agent-sdk.ts | 15 + 19 files changed, 1283 insertions(+), 26 deletions(-) create mode 100644 e2e/helpers/media-fixtures.mjs rename js/src/{prompt-cache => }/lru-cache.test.ts (100%) rename js/src/{prompt-cache => }/lru-cache.ts (100%) diff --git a/e2e/helpers/media-fixtures.mjs b/e2e/helpers/media-fixtures.mjs new file mode 100644 index 000000000..3563676a7 --- /dev/null +++ b/e2e/helpers/media-fixtures.mjs @@ -0,0 +1,5 @@ +export const MINIMAL_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; + +export const MINIMAL_PDF_BASE64 = + "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="; diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index a442a1fe2..076c39492 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -1,3 +1,7 @@ +import { + MINIMAL_PDF_BASE64, + MINIMAL_PNG_BASE64, +} from "../../helpers/media-fixtures.mjs"; import { collectAsync, runOperation, @@ -9,10 +13,6 @@ const EMBEDDING_MODEL = "text-embedding-3-small"; const MODERATION_MODEL = "omni-moderation-2024-09-26"; const ROOT_NAME = "openai-instrumentation-root"; const SCENARIO_NAME = "openai-instrumentation"; -const BASE64_IMAGE = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; -const BASE64_PDF = - "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="; const CHAT_TOOLS = [ { type: "function", @@ -212,7 +212,7 @@ export async function runOpenAIInstrumentationScenario(options) { { type: "image_url", image_url: { - url: `data:image/png;base64,${BASE64_IMAGE}`, + url: `data:image/png;base64,${MINIMAL_PNG_BASE64}`, }, }, ], @@ -241,7 +241,7 @@ export async function runOpenAIInstrumentationScenario(options) { { type: "file", file: { - file_data: `data:application/pdf;base64,${BASE64_PDF}`, + file_data: `data:application/pdf;base64,${MINIMAL_PDF_BASE64}`, filename: "document.pdf", }, }, diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json index 729e3aebc..cd2316b5f 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json @@ -222,6 +222,186 @@ }, { "callIndex": 2, + "id": "c07157ac5ff242e3", + "matchKey": "POST api.openai.com/v1/chat/completions", + "recordedAt": "2026-07-13T20:21:32.098Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_completion_tokens": 160, + "messages": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "file": { + "file_data": "data:application/pdf;base64,JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg==", + "filename": "strands-regression.pdf" + }, + "type": "file" + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"hAJ7zY0Pc\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"STR\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"5OvUWL5U\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ANDS\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"FReEmTU\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_DOCUMENT\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"k1\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_OK\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"0eg8ZOnh\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ghLMrXZfdd\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"HwqVb\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[],\"usage\":{\"prompt_tokens\":247,\"completion_tokens\":5,\"total_tokens\":252,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"ITw3NBS0a\"}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a1aafe97ee6dc45b-YYZ", + "connection": "keep-alive", + "content-type": "text/event-stream; charset=utf-8", + "date": "Mon, 13 Jul 2026 20:21:32 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "423", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999215", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3c5f893b9d4f48df88f0b20dafe5b21b" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 3, + "id": "ad37817153d52151", + "matchKey": "POST api.openai.com/v1/chat/completions", + "recordedAt": "2026-07-13T20:21:32.722Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_completion_tokens": 160, + "messages": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + }, + "type": "image_url" + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"8ncoAeH8n\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"STR\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"X1oUdVnz\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ANDS\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Essg5i5\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_IMAGE\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"5Uffg\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_OK\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YZberS3E\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"IQjxpJsARl\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"N6B0z\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[],\"usage\":{\"prompt_tokens\":8525,\"completion_tokens\":5,\"total_tokens\":8530,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"JW6tDUJ\"}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a1aafe9c2bd4c45b-YYZ", + "connection": "keep-alive", + "content-type": "text/event-stream; charset=utf-8", + "date": "Mon, 13 Jul 2026 20:21:32 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "452", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-input-images": "50000", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-input-images": "49999", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999217", + "x-ratelimit-reset-input-images": "1ms", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_68698c61c0e24e598828c1b4b51d7792" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 4, "id": "7f682068be947c96", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:00.867Z", @@ -304,7 +484,7 @@ } }, { - "callIndex": 3, + "callIndex": 5, "id": "f1affedbdb5d1da0", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:01.798Z", @@ -385,7 +565,7 @@ } }, { - "callIndex": 4, + "callIndex": 6, "id": "da2ccaeedff7ef8b", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:02.321Z", @@ -474,7 +654,7 @@ } }, { - "callIndex": 5, + "callIndex": 7, "id": "bb7637a47f479eeb", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:03.435Z", @@ -604,7 +784,7 @@ } }, { - "callIndex": 6, + "callIndex": 8, "id": "85b37d48b99f35cb", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:04.202Z", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json index f45ed1f21..e9d2b6a9b 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json @@ -114,6 +114,163 @@ "operation": "agent-invoke" } }, + { + "name": "strands-agent-document-operation", + "children": [ + { + "name": "Agent: document-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "input": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_DOCUMENT_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 247, + "outputTokens": 5, + "totalTokens": 252 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "document-agent", + "strands.agent.name": "document-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "metadata": { + "operation": "agent-document" + } + }, + { + "name": "strands-agent-image-operation", + "children": [ + { + "name": "Agent: image-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "input": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_IMAGE_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 8525, + "outputTokens": 5, + "totalTokens": 8530 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "image-agent", + "strands.agent.name": "image-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "metadata": { + "operation": "agent-image" + } + }, { "name": "strands-agent-stream-operation", "children": [ diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt index e07b7d0a6..b142b80f2 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt @@ -92,6 +92,137 @@ span_tree: │ "prompt_tokens": 127, │ "tokens": 133 │ } + ├── strands-agent-document-operation + │ metadata: { + │ "operation": "agent-document" + │ } + │ └── Agent: document-agent [task] + │ input: [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 247, + │ "outputTokens": 5, + │ "totalTokens": 252 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "document-agent", + │ "strands.agent.name": "document-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + ├── strands-agent-image-operation + │ metadata: { + │ "operation": "agent-image" + │ } + │ └── Agent: image-agent [task] + │ input: [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_IMAGE_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 8525, + │ "outputTokens": 5, + │ "totalTokens": 8530 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "image-agent", + │ "strands.agent.name": "image-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } ├── strands-agent-stream-operation │ metadata: { │ "operation": "agent-stream" diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json index ef147ae2b..18b076066 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json @@ -114,6 +114,163 @@ "operation": "agent-invoke" } }, + { + "name": "strands-agent-document-operation", + "children": [ + { + "name": "Agent: document-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "input": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_DOCUMENT_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 247, + "outputTokens": 5, + "totalTokens": 252 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "document-agent", + "strands.agent.name": "document-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "metadata": { + "operation": "agent-document" + } + }, + { + "name": "strands-agent-image-operation", + "children": [ + { + "name": "Agent: image-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "input": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_IMAGE_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 8525, + "outputTokens": 5, + "totalTokens": 8530 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "image-agent", + "strands.agent.name": "image-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "metadata": { + "operation": "agent-image" + } + }, { "name": "strands-agent-stream-operation", "children": [ diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt index 998ea28c2..ae7106d36 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt @@ -92,6 +92,137 @@ span_tree: │ "prompt_tokens": 127, │ "tokens": 133 │ } + ├── strands-agent-document-operation + │ metadata: { + │ "operation": "agent-document" + │ } + │ └── Agent: document-agent [task] + │ input: [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 247, + │ "outputTokens": 5, + │ "totalTokens": 252 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "document-agent", + │ "strands.agent.name": "document-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + ├── strands-agent-image-operation + │ metadata: { + │ "operation": "agent-image" + │ } + │ └── Agent: image-agent [task] + │ input: [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_IMAGE_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 8525, + │ "outputTokens": 5, + │ "totalTokens": 8530 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "image-agent", + │ "strands.agent.name": "image-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } ├── strands-agent-stream-operation │ metadata: { │ "operation": "agent-stream" diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts index 16f4bed37..3d89a3945 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts @@ -53,6 +53,8 @@ const SNAPSHOT_METADATA_KEYS = [ const OPERATION_NAMES = [ "strands-agent-invoke-operation", + "strands-agent-document-operation", + "strands-agent-image-operation", "strands-agent-stream-operation", "strands-graph-invoke-operation", "strands-overlap-parent-operation", @@ -172,6 +174,8 @@ function summarize(events: CapturedLogEvent[]): SpanTreeEntry[] { ROOT_NAME, ...OPERATION_NAMES, "Agent: weather-agent", + "Agent: document-agent", + "Agent: image-agent", "Agent: stream-agent", "Agent: graph-researcher", "Agent: graph-writer", @@ -206,6 +210,28 @@ function findLatestChild( return findChildSpans(events, name, parentId).at(-1); } +function findAttachmentReference( + value: unknown, +): Record | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + if ( + !Array.isArray(value) && + "type" in value && + value.type === "braintrust_attachment" + ) { + return value as Record; + } + for (const child of Object.values(value)) { + const attachment = findAttachmentReference(child); + if (attachment) { + return attachment; + } + } + return undefined; +} + export function defineStrandsAgentSDKInstrumentationAssertions(options: { name: string; expectOverlapParentProbe?: boolean; @@ -315,6 +341,78 @@ export function defineStrandsAgentSDKInstrumentationAssertions(options: { expect(streamModels.length).toBeGreaterThanOrEqual(1); }); + test("captures document bytes as attachments", testConfig, () => { + expect(setupError).toBeUndefined(); + const documentOperation = findLatestSpan( + events, + "strands-agent-document-operation", + ); + const documentAgent = findLatestChild( + events, + "Agent: document-agent", + documentOperation?.span.id, + ); + const documentModel = findLatestChild( + events, + `Strands model: ${MODEL_NAME}`, + documentAgent?.span.id, + ); + + const agentAttachment = findAttachmentReference(documentAgent?.input); + const modelAttachment = findAttachmentReference(documentModel?.input); + expect(agentAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "application/pdf", + filename: "strands-regression.pdf", + key: expect.any(String), + }); + expect(modelAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "application/pdf", + filename: "strands-regression.pdf", + key: expect.any(String), + }); + expect(modelAttachment?.key).toBe(agentAttachment?.key); + expect(JSON.stringify(documentAgent?.output)).toContain( + "STRANDS_DOCUMENT_OK", + ); + }); + + test("captures image bytes as attachments", testConfig, () => { + expect(setupError).toBeUndefined(); + const imageOperation = findLatestSpan( + events, + "strands-agent-image-operation", + ); + const imageAgent = findLatestChild( + events, + "Agent: image-agent", + imageOperation?.span.id, + ); + const imageModel = findLatestChild( + events, + `Strands model: ${MODEL_NAME}`, + imageAgent?.span.id, + ); + + const agentAttachment = findAttachmentReference(imageAgent?.input); + const modelAttachment = findAttachmentReference(imageModel?.input); + expect(agentAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "image/png", + filename: "image.png", + key: expect.any(String), + }); + expect(modelAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "image/png", + filename: "image.png", + key: expect.any(String), + }); + expect(modelAttachment?.key).toBe(agentAttachment?.key); + expect(JSON.stringify(imageAgent?.output)).toContain("STRANDS_IMAGE_OK"); + }); + test("captures graph and swarm orchestration spans", testConfig, () => { expect(setupError).toBeUndefined(); const graphOperation = findLatestSpan( diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs index 2d02a1146..a80fe7f35 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs @@ -1,4 +1,9 @@ +import { Buffer } from "node:buffer"; import { wrapStrandsAgentSDK } from "braintrust"; +import { + MINIMAL_PDF_BASE64, + MINIMAL_PNG_BASE64, +} from "../../helpers/media-fixtures.mjs"; import { collectAsync, runOperation, @@ -187,7 +192,8 @@ async function runStrandsAgentSDKInstrumentationScenario( options = {}, ) { const strands = options.decorateSDK ? options.decorateSDK(sdk) : sdk; - const { Agent, Graph, Swarm, tool } = strands; + const { Agent, DocumentBlock, Graph, ImageBlock, Swarm, TextBlock, tool } = + strands; const lookupWeather = tool({ name: "lookup_weather", description: "Return a deterministic weather report for one city.", @@ -228,6 +234,55 @@ async function runStrandsAgentSDKInstrumentationScenario( }, ); + await runOperation( + "strands-agent-document-operation", + "agent-document", + async () => { + const agent = new Agent({ + id: "document-agent", + name: "document-agent", + model: createOpenAIModel(OpenAIModel), + printer: false, + systemPrompt: "Reply exactly STRANDS_DOCUMENT_OK.", + }); + const documentBytes = Buffer.from(MINIMAL_PDF_BASE64, "base64"); + const result = await agent.invoke([ + new DocumentBlock({ + format: "pdf", + name: "strands-regression.pdf", + source: { bytes: documentBytes }, + }), + new TextBlock("Reply exactly STRANDS_DOCUMENT_OK."), + ]); + + expectContains(result, "STRANDS_DOCUMENT_OK", "agent document"); + }, + ); + + await runOperation( + "strands-agent-image-operation", + "agent-image", + async () => { + const agent = new Agent({ + id: "image-agent", + name: "image-agent", + model: createOpenAIModel(OpenAIModel), + printer: false, + systemPrompt: "Reply exactly STRANDS_IMAGE_OK.", + }); + const imageBytes = Buffer.from(MINIMAL_PNG_BASE64, "base64"); + const result = await agent.invoke([ + new ImageBlock({ + format: "png", + source: { bytes: imageBytes }, + }), + new TextBlock("Reply exactly STRANDS_IMAGE_OK."), + ]); + + expectContains(result, "STRANDS_IMAGE_OK", "agent image"); + }, + ); + await runOperation( "strands-agent-stream-operation", "agent-stream", diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index f57b87a37..66846624e 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -8,7 +8,7 @@ import { withCurrent, } from "../../logger"; import type { Span } from "../../logger"; -import { LRUCache } from "../../prompt-cache/lru-cache"; +import { LRUCache } from "../../lru-cache"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import type { diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts index 6542c432e..b8156110d 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts @@ -33,17 +33,23 @@ const { vi.mock("../../isomorph", () => ({ default: { + getEnv: vi.fn(), newAsyncLocalStorage: mockNewAsyncLocalStorage, newTracingChannel: vi.fn(), }, })); -vi.mock("../../logger", () => ({ - startSpan: (...args: unknown[]) => mockStartSpan(...args), - withCurrent: (...args: unknown[]) => mockWithCurrent(...args), -})); +vi.mock("../../logger", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + startSpan: (...args: unknown[]) => mockStartSpan(...args), + withCurrent: (...args: unknown[]) => mockWithCurrent(...args), + }; +}); import iso from "../../isomorph"; +import { Attachment } from "../../logger"; import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; import { StrandsAgentSDKPlugin } from "./strands-agent-sdk-plugin"; @@ -273,6 +279,72 @@ describe("StrandsAgentSDKPlugin", () => { expect(rootSpan?.end).toHaveBeenCalledTimes(1); }); + it.each([ + ["binary objects", new Uint8Array([1, 2, 3])], + ["base64 strings", "AQID"], + ])( + "converts media from %s to one attachment shared by agent and model spans", + async (_description, bytes) => { + const plugin = new StrandsAgentSDKPlugin(); + plugin.enable(); + + const handlers = handlersByName.get( + "orchestrion:@strands-agents/sdk:Agent.stream", + ); + const documentBlock = { + format: "pdf", + name: "tiny.pdf", + source: { bytes, type: "documentSourceBytes" }, + type: "documentBlock", + }; + const model = { + modelId: "gpt-4o-mini", + getConfig: () => ({ modelId: "gpt-4o-mini" }), + }; + const agent = { + messages: [{ content: [documentBlock], role: "user", type: "message" }], + model, + stream: vi.fn(), + }; + const stream = makeAgentStream([ + { agent, model, type: "beforeModelCallEvent" }, + { + result: { + lastMessage: { role: "assistant", content: [{ text: "done" }] }, + stopReason: "end_turn", + }, + type: "agentResultEvent", + }, + ]); + const event = { + arguments: [[documentBlock], undefined], + result: stream, + self: agent, + }; + + handlers.start(event); + handlers.end(event); + await consume(stream); + + const rootSpan = spans.find((span) => span.args.name === "Strands Agent"); + const modelSpan = spans.find( + (span) => span.args.name === "Strands model: gpt-4o-mini", + ); + const rootAttachment = + rootSpan?.args.event.input[0].document.source.bytes; + const modelAttachment = + modelSpan?.args.event.input[0].content[0].document.source.bytes; + + expect(rootAttachment).toBeInstanceOf(Attachment); + expect(rootAttachment.reference).toMatchObject({ + content_type: "application/pdf", + filename: "tiny.pdf", + type: "braintrust_attachment", + }); + expect(modelAttachment).toBe(rootAttachment); + }, + ); + it("parents nested agent spans under active graph nodes", async () => { const plugin = new StrandsAgentSDKPlugin(); plugin.enable(); diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts index 0907fc972..a35b9c09f 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts @@ -3,10 +3,17 @@ import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; -import { startSpan, withCurrent } from "../../logger"; +import { + Attachment, + BaseAttachment, + startSpan, + withCurrent, +} from "../../logger"; import type { Span } from "../../logger"; +import { LRUCache } from "../../lru-cache"; import { getCurrentUnixTimestamp } from "../../util"; import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { convertDataToBlob } from "../../wrappers/attachment-utils"; import { bindAutoInstrumentationSuppressionToStart, runWithAutoInstrumentationSuppressed, @@ -23,6 +30,7 @@ import type { StrandsBeforeNodeCallEvent, StrandsBeforeToolCallEvent, StrandsContentBlock, + StrandsMediaBlock, StrandsModel, StrandsModelMetrics, StrandsModelStreamUpdateEvent, @@ -39,6 +47,7 @@ import type { type AgentStreamState = { activeModel?: ModelSpanState; activeTools: Map; + attachmentCache: StrandsAttachmentCache; finalized: boolean; metadata: Record; span: Span; @@ -81,6 +90,13 @@ type MultiAgentStreamChannel = type ActiveChildParents = WeakMap>; +const MAX_STRANDS_STRING_ATTACHMENT_CACHE_ENTRIES = 32; + +type StrandsAttachmentCache = { + objects: WeakMap>; + strings: LRUCache>; +}; + export class StrandsAgentSDKPlugin extends BasePlugin { private readonly activeChildParents: ActiveChildParents = new WeakMap(); @@ -255,11 +271,16 @@ function startAgentStream( const parentSpan = agent ? getOnlyChildParent(activeChildParents, agent) : undefined; + const attachmentCache = createStrandsAttachmentCache(); + const input = processStrandsInputAttachments( + event.arguments[0], + attachmentCache, + ); const span = parentSpan ? withCurrent(parentSpan, () => startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: formatAgentSpanName(agent), @@ -268,7 +289,7 @@ function startAgentStream( ) : startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: formatAgentSpanName(agent), @@ -277,6 +298,7 @@ function startAgentStream( return { activeTools: new Map(), + attachmentCache, finalized: false, metadata, span, @@ -301,11 +323,12 @@ function startMultiAgentStream( const parentSpan = orchestrator ? getOnlyChildParent(activeChildParents, orchestrator) : undefined; + const input = processStrandsInputAttachments(event.arguments[0]); const span = parentSpan ? withCurrent(parentSpan, () => startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: @@ -315,7 +338,7 @@ function startMultiAgentStream( ) : startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: operation === "Graph.stream" ? "Strands Graph" : "Strands Swarm", @@ -443,7 +466,10 @@ function startModelSpan( startSpan({ event: { input: Array.isArray(event.agent?.messages) - ? event.agent.messages + ? processStrandsInputAttachments( + event.agent.messages, + state.attachmentCache, + ) : undefined, metadata, }, @@ -737,6 +763,7 @@ function finalizeAgentStream( ...(output !== undefined ? { output } : {}), }); state.span.end(); + state.attachmentCache.strings.clear(); } function finalizeMultiAgentStream( @@ -934,6 +961,235 @@ function extractNodeResultOutput(result: StrandsNodeResultEvent["result"]) { return result; } +const STRANDS_MEDIA_TYPES: Record = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + mkv: "video/x-matroska", + mov: "video/quicktime", + mp4: "video/mp4", + webm: "video/webm", + flv: "video/x-flv", + mpeg: "video/mpeg", + mpg: "video/mpeg", + wmv: "video/x-ms-wmv", + "3gp": "video/3gpp", + pdf: "application/pdf", + csv: "text/csv", + doc: "application/msword", + docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + xls: "application/vnd.ms-excel", + xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + html: "text/html", + txt: "text/plain", + md: "text/markdown", + json: "application/json", + xml: "application/xml", +}; + +function createStrandsAttachmentCache(): StrandsAttachmentCache { + return { + objects: new WeakMap(), + strings: new LRUCache({ + max: MAX_STRANDS_STRING_ATTACHMENT_CACHE_ENTRIES, + }), + }; +} + +function processStrandsInputAttachments( + input: unknown, + cache = createStrandsAttachmentCache(), +): unknown { + try { + return processStrandsInputNode(input, cache); + } catch (error) { + logInstrumentationError("Strands Agent SDK input attachments", error); + return input; + } +} + +function processStrandsInputNode( + value: unknown, + cache: StrandsAttachmentCache, +): unknown { + if (value instanceof BaseAttachment) { + return value; + } + if (Array.isArray(value)) { + return value.map((child) => processStrandsInputNode(child, cache)); + } + if (!isObject(value)) { + return value; + } + + const directMedia = processDirectStrandsMediaBlock(value, cache); + if (directMedia !== undefined) { + return directMedia; + } + + const wrappedMedia = processWrappedStrandsMediaBlock(value, cache); + if (wrappedMedia !== undefined) { + return wrappedMedia; + } + + if (value.type === "message" && Array.isArray(value.content)) { + return { + role: value.role, + content: value.content.map((child) => + processStrandsInputNode(child, cache), + ), + ...(value.metadata !== undefined ? { metadata: value.metadata } : {}), + }; + } + + if (typeof value.toJSON === "function") { + return processStrandsInputNode(value.toJSON(), cache); + } + + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [ + key, + processStrandsInputNode(child, cache), + ]), + ); +} + +function processDirectStrandsMediaBlock( + block: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + if (!isStrandsMediaBlock(block)) { + return undefined; + } + const mediaKey = + block.type === "imageBlock" + ? "image" + : block.type === "videoBlock" + ? "video" + : "document"; + + return createStrandsMediaAttachment(mediaKey, block, cache); +} + +function isStrandsMediaBlock( + block: Record, +): block is StrandsMediaBlock { + return ( + (block.type === "imageBlock" || + block.type === "videoBlock" || + block.type === "documentBlock") && + typeof block.format === "string" && + isObject(block.source) + ); +} + +function processWrappedStrandsMediaBlock( + block: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + for (const mediaKey of ["image", "video", "document"] as const) { + if (!Object.hasOwn(block, mediaKey) || !isObject(block[mediaKey])) { + continue; + } + const processed = createStrandsMediaAttachment( + mediaKey, + block[mediaKey], + cache, + ); + if (processed !== undefined) { + return processed; + } + } + return undefined; +} + +function createStrandsMediaAttachment( + mediaKey: "image" | "video" | "document", + media: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + const format = media.format; + const source = media.source; + if ( + typeof format !== "string" || + !isObject(source) || + !Object.hasOwn(source, "bytes") + ) { + return undefined; + } + + const contentType = STRANDS_MEDIA_TYPES[format.toLowerCase()]; + if (!contentType) { + return undefined; + } + const filename = + mediaKey === "document" && + typeof media.name === "string" && + media.name.length > 0 + ? media.name + : `${mediaKey}.${format.toLowerCase()}`; + const attachment = getOrCreateStrandsAttachment( + source.bytes, + filename, + contentType, + cache, + ); + if (!attachment) { + return undefined; + } + const { type: _type, ...serializedMedia } = media; + const { type: _sourceType, ...serializedSource } = source; + + return { + [mediaKey]: { + ...serializedMedia, + source: { + ...serializedSource, + bytes: attachment, + }, + }, + }; +} + +function getOrCreateStrandsAttachment( + data: unknown, + filename: string, + contentType: string, + cache: StrandsAttachmentCache, +): Attachment | undefined { + const key = `${contentType}\0${filename}`; + const attachments = + typeof data === "string" + ? cache.strings.get(data) + : isObject(data) + ? cache.objects.get(data) + : undefined; + const cached = attachments?.get(key); + if (cached) { + return cached; + } + + const blob = convertDataToBlob(data, contentType); + if (!blob) { + return undefined; + } + const attachment = new Attachment({ + data: blob, + filename, + contentType, + }); + const updatedAttachments = attachments ?? new Map(); + updatedAttachments.set(key, attachment); + if (typeof data === "string") { + cache.strings.set(data, updatedAttachments); + } else if (isObject(data)) { + cache.objects.set(data, updatedAttachments); + } + return attachment; +} + function normalizeContentBlocks(blocks: StrandsContentBlock[]): unknown { const text = blocks .map((block) => (typeof block.text === "string" ? block.text : undefined)) diff --git a/js/src/prompt-cache/lru-cache.test.ts b/js/src/lru-cache.test.ts similarity index 100% rename from js/src/prompt-cache/lru-cache.test.ts rename to js/src/lru-cache.test.ts diff --git a/js/src/prompt-cache/lru-cache.ts b/js/src/lru-cache.ts similarity index 100% rename from js/src/prompt-cache/lru-cache.ts rename to js/src/lru-cache.ts diff --git a/js/src/prompt-cache/cache-config.ts b/js/src/prompt-cache/cache-config.ts index 4d4a34d9a..4134855f8 100644 --- a/js/src/prompt-cache/cache-config.ts +++ b/js/src/prompt-cache/cache-config.ts @@ -1,7 +1,7 @@ import { debugLogger } from "../debug-logger"; import iso from "../isomorph"; import { canUseDiskCache, DiskCache } from "./disk-cache"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; type CacheMode = "mixed" | "memory" | "disk" | "none"; diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts index 57d637983..240da544b 100644 --- a/js/src/prompt-cache/parameters-cache.ts +++ b/js/src/prompt-cache/parameters-cache.ts @@ -1,5 +1,5 @@ import { RemoteEvalParameters } from "../logger"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; interface ParametersKey { diff --git a/js/src/prompt-cache/prompt-cache.test.ts b/js/src/prompt-cache/prompt-cache.test.ts index 615763c5e..628ddfcde 100644 --- a/js/src/prompt-cache/prompt-cache.test.ts +++ b/js/src/prompt-cache/prompt-cache.test.ts @@ -7,7 +7,7 @@ import { beforeEach, describe, it, afterEach, expect } from "vitest"; import type { PromptKey } from "./prompt-cache"; import { DiskCache } from "./disk-cache"; import { configureNode } from "../node/config"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; describe("PromptCache", () => { configureNode(); diff --git a/js/src/prompt-cache/prompt-cache.ts b/js/src/prompt-cache/prompt-cache.ts index e106a32cf..3b9d2ebee 100644 --- a/js/src/prompt-cache/prompt-cache.ts +++ b/js/src/prompt-cache/prompt-cache.ts @@ -1,5 +1,5 @@ import { Prompt } from "../logger"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; /** diff --git a/js/src/vendor-sdk-types/strands-agent-sdk.ts b/js/src/vendor-sdk-types/strands-agent-sdk.ts index b4069e2f8..16eaa6e87 100644 --- a/js/src/vendor-sdk-types/strands-agent-sdk.ts +++ b/js/src/vendor-sdk-types/strands-agent-sdk.ts @@ -111,6 +111,21 @@ export type StrandsContentBlock = { [key: string]: unknown; }; +export interface StrandsMediaBlock { + type: "imageBlock" | "videoBlock" | "documentBlock"; + format: string; + name?: string; + source: { + type?: string; + bytes?: string | Uint8Array; + [key: string]: unknown; + }; + citations?: unknown; + context?: unknown; + toJSON?: () => unknown; + [key: string]: unknown; +} + export interface StrandsToolUse { name: string; toolUseId: string; From dc2de7a917b939e2c062dde07afaafe1949f3111 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 13 Jul 2026 16:41:01 -0400 Subject: [PATCH 2/3] fix ci --- .changeset/strands-media-attachments.md | 5 +++++ js/dev/authorize.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/strands-media-attachments.md diff --git a/.changeset/strands-media-attachments.md b/.changeset/strands-media-attachments.md new file mode 100644 index 000000000..283774239 --- /dev/null +++ b/.changeset/strands-media-attachments.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(strands): Log inline document, image, and video bytes as Braintrust attachments. diff --git a/js/dev/authorize.ts b/js/dev/authorize.ts index 00cf8896d..4f7592d17 100644 --- a/js/dev/authorize.ts +++ b/js/dev/authorize.ts @@ -1,7 +1,7 @@ import { Request, Response, NextFunction } from "express"; import { IncomingHttpHeaders } from "http"; import createError from "http-errors"; -import { LRUCache } from "../src/prompt-cache/lru-cache"; +import { LRUCache } from "../src/lru-cache"; import { BraintrustState, LoginOptions, loginToState } from "../src/logger"; export interface RequestContext { From 2f2bd5ccbe53970aa5d0dd3d15161ae0550ebf40 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 13 Jul 2026 16:44:08 -0400 Subject: [PATCH 3/3] typing fix --- .../instrumentation/plugins/strands-agent-sdk-plugin.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts index a35b9c09f..43669cd92 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts @@ -1090,14 +1090,11 @@ function processWrappedStrandsMediaBlock( cache: StrandsAttachmentCache, ): unknown | undefined { for (const mediaKey of ["image", "video", "document"] as const) { - if (!Object.hasOwn(block, mediaKey) || !isObject(block[mediaKey])) { + const media = block[mediaKey]; + if (!Object.hasOwn(block, mediaKey) || !isObject(media)) { continue; } - const processed = createStrandsMediaAttachment( - mediaKey, - block[mediaKey], - cache, - ); + const processed = createStrandsMediaAttachment(mediaKey, media, cache); if (processed !== undefined) { return processed; }