diff --git a/.changeset/strands-media-attachments.md b/.changeset/strands-media-attachments.md new file mode 100644 index 000000000..283774239 --- /dev/null +++ b/.changeset/strands-media-attachments.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(strands): Log inline document, image, and video bytes as Braintrust attachments. diff --git a/e2e/helpers/media-fixtures.mjs b/e2e/helpers/media-fixtures.mjs new file mode 100644 index 000000000..3563676a7 --- /dev/null +++ b/e2e/helpers/media-fixtures.mjs @@ -0,0 +1,5 @@ +export const MINIMAL_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; + +export const MINIMAL_PDF_BASE64 = + "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="; diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index a442a1fe2..076c39492 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -1,3 +1,7 @@ +import { + MINIMAL_PDF_BASE64, + MINIMAL_PNG_BASE64, +} from "../../helpers/media-fixtures.mjs"; import { collectAsync, runOperation, @@ -9,10 +13,6 @@ const EMBEDDING_MODEL = "text-embedding-3-small"; const MODERATION_MODEL = "omni-moderation-2024-09-26"; const ROOT_NAME = "openai-instrumentation-root"; const SCENARIO_NAME = "openai-instrumentation"; -const BASE64_IMAGE = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; -const BASE64_PDF = - "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="; const CHAT_TOOLS = [ { type: "function", @@ -212,7 +212,7 @@ export async function runOpenAIInstrumentationScenario(options) { { type: "image_url", image_url: { - url: `data:image/png;base64,${BASE64_IMAGE}`, + url: `data:image/png;base64,${MINIMAL_PNG_BASE64}`, }, }, ], @@ -241,7 +241,7 @@ export async function runOpenAIInstrumentationScenario(options) { { type: "file", file: { - file_data: `data:application/pdf;base64,${BASE64_PDF}`, + file_data: `data:application/pdf;base64,${MINIMAL_PDF_BASE64}`, filename: "document.pdf", }, }, diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json index 729e3aebc..cd2316b5f 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__cassettes__/strands-agent-sdk-v160.cassette.json @@ -222,6 +222,186 @@ }, { "callIndex": 2, + "id": "c07157ac5ff242e3", + "matchKey": "POST api.openai.com/v1/chat/completions", + "recordedAt": "2026-07-13T20:21:32.098Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_completion_tokens": 160, + "messages": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "file": { + "file_data": "data:application/pdf;base64,JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg==", + "filename": "strands-regression.pdf" + }, + "type": "file" + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"hAJ7zY0Pc\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"STR\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"5OvUWL5U\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ANDS\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"FReEmTU\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_DOCUMENT\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"k1\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_OK\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"0eg8ZOnh\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ghLMrXZfdd\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"HwqVb\"}", + "data: {\"id\":\"chatcmpl-E1HRbNm3TLJJtzCmkg21mrSv8eVB7\",\"object\":\"chat.completion.chunk\",\"created\":1783974091,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[],\"usage\":{\"prompt_tokens\":247,\"completion_tokens\":5,\"total_tokens\":252,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"ITw3NBS0a\"}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a1aafe97ee6dc45b-YYZ", + "connection": "keep-alive", + "content-type": "text/event-stream; charset=utf-8", + "date": "Mon, 13 Jul 2026 20:21:32 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "423", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999215", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3c5f893b9d4f48df88f0b20dafe5b21b" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 3, + "id": "ad37817153d52151", + "matchKey": "POST api.openai.com/v1/chat/completions", + "recordedAt": "2026-07-13T20:21:32.722Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_completion_tokens": 160, + "messages": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + }, + "type": "image_url" + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"8ncoAeH8n\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"STR\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"X1oUdVnz\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ANDS\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Essg5i5\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_IMAGE\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"5Uffg\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"_OK\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YZberS3E\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"IQjxpJsARl\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"N6B0z\"}", + "data: {\"id\":\"chatcmpl-E1HRcrILLgMUPTLS9F8d4RCGqbUWw\",\"object\":\"chat.completion.chunk\",\"created\":1783974092,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_844ddd95ff\",\"choices\":[],\"usage\":{\"prompt_tokens\":8525,\"completion_tokens\":5,\"total_tokens\":8530,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"JW6tDUJ\"}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a1aafe9c2bd4c45b-YYZ", + "connection": "keep-alive", + "content-type": "text/event-stream; charset=utf-8", + "date": "Mon, 13 Jul 2026 20:21:32 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "452", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-input-images": "50000", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-input-images": "49999", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999217", + "x-ratelimit-reset-input-images": "1ms", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_68698c61c0e24e598828c1b4b51d7792" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 4, "id": "7f682068be947c96", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:00.867Z", @@ -304,7 +484,7 @@ } }, { - "callIndex": 3, + "callIndex": 5, "id": "f1affedbdb5d1da0", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:01.798Z", @@ -385,7 +565,7 @@ } }, { - "callIndex": 4, + "callIndex": 6, "id": "da2ccaeedff7ef8b", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:02.321Z", @@ -474,7 +654,7 @@ } }, { - "callIndex": 5, + "callIndex": 7, "id": "bb7637a47f479eeb", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:03.435Z", @@ -604,7 +784,7 @@ } }, { - "callIndex": 6, + "callIndex": 8, "id": "85b37d48b99f35cb", "matchKey": "POST api.openai.com/v1/chat/completions", "recordedAt": "2026-06-18T14:20:04.202Z", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json index f45ed1f21..e9d2b6a9b 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.json @@ -114,6 +114,163 @@ "operation": "agent-invoke" } }, + { + "name": "strands-agent-document-operation", + "children": [ + { + "name": "Agent: document-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "input": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_DOCUMENT_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 247, + "outputTokens": 5, + "totalTokens": 252 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "document-agent", + "strands.agent.name": "document-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "metadata": { + "operation": "agent-document" + } + }, + { + "name": "strands-agent-image-operation", + "children": [ + { + "name": "Agent: image-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "input": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_IMAGE_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 8525, + "outputTokens": 5, + "totalTokens": 8530 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "image-agent", + "strands.agent.name": "image-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "metadata": { + "operation": "agent-image" + } + }, { "name": "strands-agent-stream-operation", "children": [ diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt index e07b7d0a6..b142b80f2 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-auto-hook.span-tree.txt @@ -92,6 +92,137 @@ span_tree: │ "prompt_tokens": 127, │ "tokens": 133 │ } + ├── strands-agent-document-operation + │ metadata: { + │ "operation": "agent-document" + │ } + │ └── Agent: document-agent [task] + │ input: [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 247, + │ "outputTokens": 5, + │ "totalTokens": 252 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "document-agent", + │ "strands.agent.name": "document-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + ├── strands-agent-image-operation + │ metadata: { + │ "operation": "agent-image" + │ } + │ └── Agent: image-agent [task] + │ input: [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_IMAGE_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 8525, + │ "outputTokens": 5, + │ "totalTokens": 8530 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "image-agent", + │ "strands.agent.name": "image-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } ├── strands-agent-stream-operation │ metadata: { │ "operation": "agent-stream" diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json index ef147ae2b..18b076066 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.json @@ -114,6 +114,163 @@ "operation": "agent-invoke" } }, + { + "name": "strands-agent-document-operation", + "children": [ + { + "name": "Agent: document-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "input": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_DOCUMENT_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 247, + "outputTokens": 5, + "totalTokens": 252 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "document-agent", + "strands.agent.name": "document-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 247, + "tokens": 252 + } + } + ], + "metadata": { + "operation": "agent-document" + } + }, + { + "name": "strands-agent-image-operation", + "children": [ + { + "name": "Agent: image-agent", + "type": "task", + "children": [ + { + "name": "Strands model: gpt-4o-mini-2024-07-18", + "type": "llm", + "children": [], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.operation": "model.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "input": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "output": { + "content": [ + { + "text": "STRANDS_IMAGE_OK." + } + ], + "metadata": { + "usage": { + "inputTokens": 8525, + "outputTokens": 5, + "totalTokens": 8530 + } + }, + "role": "assistant" + }, + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "strands.agent.id": "image-agent", + "strands.agent.name": "image-agent", + "strands.operation": "Agent.stream", + "strands.stop_reason": "endTurn" + }, + "metrics": { + "completion_tokens": 5, + "duration": "", + "prompt_tokens": 8525, + "tokens": 8530 + } + } + ], + "metadata": { + "operation": "agent-image" + } + }, { "name": "strands-agent-stream-operation", "children": [ diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt index 998ea28c2..ae7106d36 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v160-wrapped.span-tree.txt @@ -92,6 +92,137 @@ span_tree: │ "prompt_tokens": 127, │ "tokens": 133 │ } + ├── strands-agent-document-operation + │ metadata: { + │ "operation": "agent-document" + │ } + │ └── Agent: document-agent [task] + │ input: [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 247, + │ "outputTokens": 5, + │ "totalTokens": 252 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "document-agent", + │ "strands.agent.name": "document-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 247, + │ "tokens": 252 + │ } + ├── strands-agent-image-operation + │ metadata: { + │ "operation": "agent-image" + │ } + │ └── Agent: image-agent [task] + │ input: [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "STRANDS_IMAGE_OK." + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 8525, + │ "outputTokens": 5, + │ "totalTokens": 8530 + │ } + │ }, + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.agent.id": "image-agent", + │ "strands.agent.name": "image-agent", + │ "strands.operation": "Agent.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } + │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "strands.operation": "model.stream", + │ "strands.stop_reason": "endTurn" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "duration": "", + │ "prompt_tokens": 8525, + │ "tokens": 8530 + │ } ├── strands-agent-stream-operation │ metadata: { │ "operation": "agent-stream" diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts index 16f4bed37..3d89a3945 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts @@ -53,6 +53,8 @@ const SNAPSHOT_METADATA_KEYS = [ const OPERATION_NAMES = [ "strands-agent-invoke-operation", + "strands-agent-document-operation", + "strands-agent-image-operation", "strands-agent-stream-operation", "strands-graph-invoke-operation", "strands-overlap-parent-operation", @@ -172,6 +174,8 @@ function summarize(events: CapturedLogEvent[]): SpanTreeEntry[] { ROOT_NAME, ...OPERATION_NAMES, "Agent: weather-agent", + "Agent: document-agent", + "Agent: image-agent", "Agent: stream-agent", "Agent: graph-researcher", "Agent: graph-writer", @@ -206,6 +210,28 @@ function findLatestChild( return findChildSpans(events, name, parentId).at(-1); } +function findAttachmentReference( + value: unknown, +): Record | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + if ( + !Array.isArray(value) && + "type" in value && + value.type === "braintrust_attachment" + ) { + return value as Record; + } + for (const child of Object.values(value)) { + const attachment = findAttachmentReference(child); + if (attachment) { + return attachment; + } + } + return undefined; +} + export function defineStrandsAgentSDKInstrumentationAssertions(options: { name: string; expectOverlapParentProbe?: boolean; @@ -315,6 +341,78 @@ export function defineStrandsAgentSDKInstrumentationAssertions(options: { expect(streamModels.length).toBeGreaterThanOrEqual(1); }); + test("captures document bytes as attachments", testConfig, () => { + expect(setupError).toBeUndefined(); + const documentOperation = findLatestSpan( + events, + "strands-agent-document-operation", + ); + const documentAgent = findLatestChild( + events, + "Agent: document-agent", + documentOperation?.span.id, + ); + const documentModel = findLatestChild( + events, + `Strands model: ${MODEL_NAME}`, + documentAgent?.span.id, + ); + + const agentAttachment = findAttachmentReference(documentAgent?.input); + const modelAttachment = findAttachmentReference(documentModel?.input); + expect(agentAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "application/pdf", + filename: "strands-regression.pdf", + key: expect.any(String), + }); + expect(modelAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "application/pdf", + filename: "strands-regression.pdf", + key: expect.any(String), + }); + expect(modelAttachment?.key).toBe(agentAttachment?.key); + expect(JSON.stringify(documentAgent?.output)).toContain( + "STRANDS_DOCUMENT_OK", + ); + }); + + test("captures image bytes as attachments", testConfig, () => { + expect(setupError).toBeUndefined(); + const imageOperation = findLatestSpan( + events, + "strands-agent-image-operation", + ); + const imageAgent = findLatestChild( + events, + "Agent: image-agent", + imageOperation?.span.id, + ); + const imageModel = findLatestChild( + events, + `Strands model: ${MODEL_NAME}`, + imageAgent?.span.id, + ); + + const agentAttachment = findAttachmentReference(imageAgent?.input); + const modelAttachment = findAttachmentReference(imageModel?.input); + expect(agentAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "image/png", + filename: "image.png", + key: expect.any(String), + }); + expect(modelAttachment).toMatchObject({ + type: "braintrust_attachment", + content_type: "image/png", + filename: "image.png", + key: expect.any(String), + }); + expect(modelAttachment?.key).toBe(agentAttachment?.key); + expect(JSON.stringify(imageAgent?.output)).toContain("STRANDS_IMAGE_OK"); + }); + test("captures graph and swarm orchestration spans", testConfig, () => { expect(setupError).toBeUndefined(); const graphOperation = findLatestSpan( diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs index 2d02a1146..a80fe7f35 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/scenario.impl.mjs @@ -1,4 +1,9 @@ +import { Buffer } from "node:buffer"; import { wrapStrandsAgentSDK } from "braintrust"; +import { + MINIMAL_PDF_BASE64, + MINIMAL_PNG_BASE64, +} from "../../helpers/media-fixtures.mjs"; import { collectAsync, runOperation, @@ -187,7 +192,8 @@ async function runStrandsAgentSDKInstrumentationScenario( options = {}, ) { const strands = options.decorateSDK ? options.decorateSDK(sdk) : sdk; - const { Agent, Graph, Swarm, tool } = strands; + const { Agent, DocumentBlock, Graph, ImageBlock, Swarm, TextBlock, tool } = + strands; const lookupWeather = tool({ name: "lookup_weather", description: "Return a deterministic weather report for one city.", @@ -228,6 +234,55 @@ async function runStrandsAgentSDKInstrumentationScenario( }, ); + await runOperation( + "strands-agent-document-operation", + "agent-document", + async () => { + const agent = new Agent({ + id: "document-agent", + name: "document-agent", + model: createOpenAIModel(OpenAIModel), + printer: false, + systemPrompt: "Reply exactly STRANDS_DOCUMENT_OK.", + }); + const documentBytes = Buffer.from(MINIMAL_PDF_BASE64, "base64"); + const result = await agent.invoke([ + new DocumentBlock({ + format: "pdf", + name: "strands-regression.pdf", + source: { bytes: documentBytes }, + }), + new TextBlock("Reply exactly STRANDS_DOCUMENT_OK."), + ]); + + expectContains(result, "STRANDS_DOCUMENT_OK", "agent document"); + }, + ); + + await runOperation( + "strands-agent-image-operation", + "agent-image", + async () => { + const agent = new Agent({ + id: "image-agent", + name: "image-agent", + model: createOpenAIModel(OpenAIModel), + printer: false, + systemPrompt: "Reply exactly STRANDS_IMAGE_OK.", + }); + const imageBytes = Buffer.from(MINIMAL_PNG_BASE64, "base64"); + const result = await agent.invoke([ + new ImageBlock({ + format: "png", + source: { bytes: imageBytes }, + }), + new TextBlock("Reply exactly STRANDS_IMAGE_OK."), + ]); + + expectContains(result, "STRANDS_IMAGE_OK", "agent image"); + }, + ); + await runOperation( "strands-agent-stream-operation", "agent-stream", diff --git a/js/dev/authorize.ts b/js/dev/authorize.ts index 00cf8896d..4f7592d17 100644 --- a/js/dev/authorize.ts +++ b/js/dev/authorize.ts @@ -1,7 +1,7 @@ import { Request, Response, NextFunction } from "express"; import { IncomingHttpHeaders } from "http"; import createError from "http-errors"; -import { LRUCache } from "../src/prompt-cache/lru-cache"; +import { LRUCache } from "../src/lru-cache"; import { BraintrustState, LoginOptions, loginToState } from "../src/logger"; export interface RequestContext { diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index f57b87a37..66846624e 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -8,7 +8,7 @@ import { withCurrent, } from "../../logger"; import type { Span } from "../../logger"; -import { LRUCache } from "../../prompt-cache/lru-cache"; +import { LRUCache } from "../../lru-cache"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import type { diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts index 6542c432e..b8156110d 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts @@ -33,17 +33,23 @@ const { vi.mock("../../isomorph", () => ({ default: { + getEnv: vi.fn(), newAsyncLocalStorage: mockNewAsyncLocalStorage, newTracingChannel: vi.fn(), }, })); -vi.mock("../../logger", () => ({ - startSpan: (...args: unknown[]) => mockStartSpan(...args), - withCurrent: (...args: unknown[]) => mockWithCurrent(...args), -})); +vi.mock("../../logger", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + startSpan: (...args: unknown[]) => mockStartSpan(...args), + withCurrent: (...args: unknown[]) => mockWithCurrent(...args), + }; +}); import iso from "../../isomorph"; +import { Attachment } from "../../logger"; import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; import { StrandsAgentSDKPlugin } from "./strands-agent-sdk-plugin"; @@ -273,6 +279,72 @@ describe("StrandsAgentSDKPlugin", () => { expect(rootSpan?.end).toHaveBeenCalledTimes(1); }); + it.each([ + ["binary objects", new Uint8Array([1, 2, 3])], + ["base64 strings", "AQID"], + ])( + "converts media from %s to one attachment shared by agent and model spans", + async (_description, bytes) => { + const plugin = new StrandsAgentSDKPlugin(); + plugin.enable(); + + const handlers = handlersByName.get( + "orchestrion:@strands-agents/sdk:Agent.stream", + ); + const documentBlock = { + format: "pdf", + name: "tiny.pdf", + source: { bytes, type: "documentSourceBytes" }, + type: "documentBlock", + }; + const model = { + modelId: "gpt-4o-mini", + getConfig: () => ({ modelId: "gpt-4o-mini" }), + }; + const agent = { + messages: [{ content: [documentBlock], role: "user", type: "message" }], + model, + stream: vi.fn(), + }; + const stream = makeAgentStream([ + { agent, model, type: "beforeModelCallEvent" }, + { + result: { + lastMessage: { role: "assistant", content: [{ text: "done" }] }, + stopReason: "end_turn", + }, + type: "agentResultEvent", + }, + ]); + const event = { + arguments: [[documentBlock], undefined], + result: stream, + self: agent, + }; + + handlers.start(event); + handlers.end(event); + await consume(stream); + + const rootSpan = spans.find((span) => span.args.name === "Strands Agent"); + const modelSpan = spans.find( + (span) => span.args.name === "Strands model: gpt-4o-mini", + ); + const rootAttachment = + rootSpan?.args.event.input[0].document.source.bytes; + const modelAttachment = + modelSpan?.args.event.input[0].content[0].document.source.bytes; + + expect(rootAttachment).toBeInstanceOf(Attachment); + expect(rootAttachment.reference).toMatchObject({ + content_type: "application/pdf", + filename: "tiny.pdf", + type: "braintrust_attachment", + }); + expect(modelAttachment).toBe(rootAttachment); + }, + ); + it("parents nested agent spans under active graph nodes", async () => { const plugin = new StrandsAgentSDKPlugin(); plugin.enable(); diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts index 0907fc972..43669cd92 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts @@ -3,10 +3,17 @@ import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; -import { startSpan, withCurrent } from "../../logger"; +import { + Attachment, + BaseAttachment, + startSpan, + withCurrent, +} from "../../logger"; import type { Span } from "../../logger"; +import { LRUCache } from "../../lru-cache"; import { getCurrentUnixTimestamp } from "../../util"; import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { convertDataToBlob } from "../../wrappers/attachment-utils"; import { bindAutoInstrumentationSuppressionToStart, runWithAutoInstrumentationSuppressed, @@ -23,6 +30,7 @@ import type { StrandsBeforeNodeCallEvent, StrandsBeforeToolCallEvent, StrandsContentBlock, + StrandsMediaBlock, StrandsModel, StrandsModelMetrics, StrandsModelStreamUpdateEvent, @@ -39,6 +47,7 @@ import type { type AgentStreamState = { activeModel?: ModelSpanState; activeTools: Map; + attachmentCache: StrandsAttachmentCache; finalized: boolean; metadata: Record; span: Span; @@ -81,6 +90,13 @@ type MultiAgentStreamChannel = type ActiveChildParents = WeakMap>; +const MAX_STRANDS_STRING_ATTACHMENT_CACHE_ENTRIES = 32; + +type StrandsAttachmentCache = { + objects: WeakMap>; + strings: LRUCache>; +}; + export class StrandsAgentSDKPlugin extends BasePlugin { private readonly activeChildParents: ActiveChildParents = new WeakMap(); @@ -255,11 +271,16 @@ function startAgentStream( const parentSpan = agent ? getOnlyChildParent(activeChildParents, agent) : undefined; + const attachmentCache = createStrandsAttachmentCache(); + const input = processStrandsInputAttachments( + event.arguments[0], + attachmentCache, + ); const span = parentSpan ? withCurrent(parentSpan, () => startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: formatAgentSpanName(agent), @@ -268,7 +289,7 @@ function startAgentStream( ) : startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: formatAgentSpanName(agent), @@ -277,6 +298,7 @@ function startAgentStream( return { activeTools: new Map(), + attachmentCache, finalized: false, metadata, span, @@ -301,11 +323,12 @@ function startMultiAgentStream( const parentSpan = orchestrator ? getOnlyChildParent(activeChildParents, orchestrator) : undefined; + const input = processStrandsInputAttachments(event.arguments[0]); const span = parentSpan ? withCurrent(parentSpan, () => startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: @@ -315,7 +338,7 @@ function startMultiAgentStream( ) : startSpan({ event: { - input: event.arguments[0], + input, metadata, }, name: operation === "Graph.stream" ? "Strands Graph" : "Strands Swarm", @@ -443,7 +466,10 @@ function startModelSpan( startSpan({ event: { input: Array.isArray(event.agent?.messages) - ? event.agent.messages + ? processStrandsInputAttachments( + event.agent.messages, + state.attachmentCache, + ) : undefined, metadata, }, @@ -737,6 +763,7 @@ function finalizeAgentStream( ...(output !== undefined ? { output } : {}), }); state.span.end(); + state.attachmentCache.strings.clear(); } function finalizeMultiAgentStream( @@ -934,6 +961,232 @@ function extractNodeResultOutput(result: StrandsNodeResultEvent["result"]) { return result; } +const STRANDS_MEDIA_TYPES: Record = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + mkv: "video/x-matroska", + mov: "video/quicktime", + mp4: "video/mp4", + webm: "video/webm", + flv: "video/x-flv", + mpeg: "video/mpeg", + mpg: "video/mpeg", + wmv: "video/x-ms-wmv", + "3gp": "video/3gpp", + pdf: "application/pdf", + csv: "text/csv", + doc: "application/msword", + docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + xls: "application/vnd.ms-excel", + xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + html: "text/html", + txt: "text/plain", + md: "text/markdown", + json: "application/json", + xml: "application/xml", +}; + +function createStrandsAttachmentCache(): StrandsAttachmentCache { + return { + objects: new WeakMap(), + strings: new LRUCache({ + max: MAX_STRANDS_STRING_ATTACHMENT_CACHE_ENTRIES, + }), + }; +} + +function processStrandsInputAttachments( + input: unknown, + cache = createStrandsAttachmentCache(), +): unknown { + try { + return processStrandsInputNode(input, cache); + } catch (error) { + logInstrumentationError("Strands Agent SDK input attachments", error); + return input; + } +} + +function processStrandsInputNode( + value: unknown, + cache: StrandsAttachmentCache, +): unknown { + if (value instanceof BaseAttachment) { + return value; + } + if (Array.isArray(value)) { + return value.map((child) => processStrandsInputNode(child, cache)); + } + if (!isObject(value)) { + return value; + } + + const directMedia = processDirectStrandsMediaBlock(value, cache); + if (directMedia !== undefined) { + return directMedia; + } + + const wrappedMedia = processWrappedStrandsMediaBlock(value, cache); + if (wrappedMedia !== undefined) { + return wrappedMedia; + } + + if (value.type === "message" && Array.isArray(value.content)) { + return { + role: value.role, + content: value.content.map((child) => + processStrandsInputNode(child, cache), + ), + ...(value.metadata !== undefined ? { metadata: value.metadata } : {}), + }; + } + + if (typeof value.toJSON === "function") { + return processStrandsInputNode(value.toJSON(), cache); + } + + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [ + key, + processStrandsInputNode(child, cache), + ]), + ); +} + +function processDirectStrandsMediaBlock( + block: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + if (!isStrandsMediaBlock(block)) { + return undefined; + } + const mediaKey = + block.type === "imageBlock" + ? "image" + : block.type === "videoBlock" + ? "video" + : "document"; + + return createStrandsMediaAttachment(mediaKey, block, cache); +} + +function isStrandsMediaBlock( + block: Record, +): block is StrandsMediaBlock { + return ( + (block.type === "imageBlock" || + block.type === "videoBlock" || + block.type === "documentBlock") && + typeof block.format === "string" && + isObject(block.source) + ); +} + +function processWrappedStrandsMediaBlock( + block: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + for (const mediaKey of ["image", "video", "document"] as const) { + const media = block[mediaKey]; + if (!Object.hasOwn(block, mediaKey) || !isObject(media)) { + continue; + } + const processed = createStrandsMediaAttachment(mediaKey, media, cache); + if (processed !== undefined) { + return processed; + } + } + return undefined; +} + +function createStrandsMediaAttachment( + mediaKey: "image" | "video" | "document", + media: Record, + cache: StrandsAttachmentCache, +): unknown | undefined { + const format = media.format; + const source = media.source; + if ( + typeof format !== "string" || + !isObject(source) || + !Object.hasOwn(source, "bytes") + ) { + return undefined; + } + + const contentType = STRANDS_MEDIA_TYPES[format.toLowerCase()]; + if (!contentType) { + return undefined; + } + const filename = + mediaKey === "document" && + typeof media.name === "string" && + media.name.length > 0 + ? media.name + : `${mediaKey}.${format.toLowerCase()}`; + const attachment = getOrCreateStrandsAttachment( + source.bytes, + filename, + contentType, + cache, + ); + if (!attachment) { + return undefined; + } + const { type: _type, ...serializedMedia } = media; + const { type: _sourceType, ...serializedSource } = source; + + return { + [mediaKey]: { + ...serializedMedia, + source: { + ...serializedSource, + bytes: attachment, + }, + }, + }; +} + +function getOrCreateStrandsAttachment( + data: unknown, + filename: string, + contentType: string, + cache: StrandsAttachmentCache, +): Attachment | undefined { + const key = `${contentType}\0${filename}`; + const attachments = + typeof data === "string" + ? cache.strings.get(data) + : isObject(data) + ? cache.objects.get(data) + : undefined; + const cached = attachments?.get(key); + if (cached) { + return cached; + } + + const blob = convertDataToBlob(data, contentType); + if (!blob) { + return undefined; + } + const attachment = new Attachment({ + data: blob, + filename, + contentType, + }); + const updatedAttachments = attachments ?? new Map(); + updatedAttachments.set(key, attachment); + if (typeof data === "string") { + cache.strings.set(data, updatedAttachments); + } else if (isObject(data)) { + cache.objects.set(data, updatedAttachments); + } + return attachment; +} + function normalizeContentBlocks(blocks: StrandsContentBlock[]): unknown { const text = blocks .map((block) => (typeof block.text === "string" ? block.text : undefined)) diff --git a/js/src/prompt-cache/lru-cache.test.ts b/js/src/lru-cache.test.ts similarity index 100% rename from js/src/prompt-cache/lru-cache.test.ts rename to js/src/lru-cache.test.ts diff --git a/js/src/prompt-cache/lru-cache.ts b/js/src/lru-cache.ts similarity index 100% rename from js/src/prompt-cache/lru-cache.ts rename to js/src/lru-cache.ts diff --git a/js/src/prompt-cache/cache-config.ts b/js/src/prompt-cache/cache-config.ts index 4d4a34d9a..4134855f8 100644 --- a/js/src/prompt-cache/cache-config.ts +++ b/js/src/prompt-cache/cache-config.ts @@ -1,7 +1,7 @@ import { debugLogger } from "../debug-logger"; import iso from "../isomorph"; import { canUseDiskCache, DiskCache } from "./disk-cache"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; type CacheMode = "mixed" | "memory" | "disk" | "none"; diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts index 57d637983..240da544b 100644 --- a/js/src/prompt-cache/parameters-cache.ts +++ b/js/src/prompt-cache/parameters-cache.ts @@ -1,5 +1,5 @@ import { RemoteEvalParameters } from "../logger"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; interface ParametersKey { diff --git a/js/src/prompt-cache/prompt-cache.test.ts b/js/src/prompt-cache/prompt-cache.test.ts index 615763c5e..628ddfcde 100644 --- a/js/src/prompt-cache/prompt-cache.test.ts +++ b/js/src/prompt-cache/prompt-cache.test.ts @@ -7,7 +7,7 @@ import { beforeEach, describe, it, afterEach, expect } from "vitest"; import type { PromptKey } from "./prompt-cache"; import { DiskCache } from "./disk-cache"; import { configureNode } from "../node/config"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; describe("PromptCache", () => { configureNode(); diff --git a/js/src/prompt-cache/prompt-cache.ts b/js/src/prompt-cache/prompt-cache.ts index e106a32cf..3b9d2ebee 100644 --- a/js/src/prompt-cache/prompt-cache.ts +++ b/js/src/prompt-cache/prompt-cache.ts @@ -1,5 +1,5 @@ import { Prompt } from "../logger"; -import { LRUCache } from "./lru-cache"; +import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; /** diff --git a/js/src/vendor-sdk-types/strands-agent-sdk.ts b/js/src/vendor-sdk-types/strands-agent-sdk.ts index b4069e2f8..16eaa6e87 100644 --- a/js/src/vendor-sdk-types/strands-agent-sdk.ts +++ b/js/src/vendor-sdk-types/strands-agent-sdk.ts @@ -111,6 +111,21 @@ export type StrandsContentBlock = { [key: string]: unknown; }; +export interface StrandsMediaBlock { + type: "imageBlock" | "videoBlock" | "documentBlock"; + format: string; + name?: string; + source: { + type?: string; + bytes?: string | Uint8Array; + [key: string]: unknown; + }; + citations?: unknown; + context?: unknown; + toJSON?: () => unknown; + [key: string]: unknown; +} + export interface StrandsToolUse { name: string; toolUseId: string;