From 4ef3513500f102fa3f8672c0c1be38e2d8a49d64 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 13 Jul 2026 15:46:38 +0200 Subject: [PATCH 1/3] feat: Add `langsmith` tracing --- .changeset/sixty-seals-judge.md | 5 + .gitleaks.toml | 10 + e2e/config/pr-comment-scenarios.json | 10 + .../langsmith-v0-3-30-auto-esm.span-tree.json | 202 ++++++ .../langsmith-v0-3-30-auto-esm.span-tree.txt | 160 +++++ .../langsmith-v0-3-30-wrapped.span-tree.json | 202 ++++++ .../langsmith-v0-3-30-wrapped.span-tree.txt | 160 +++++ .../langsmith-v0-5-11-auto-esm.span-tree.json | 205 ++++++ .../langsmith-v0-5-11-auto-esm.span-tree.txt | 163 +++++ .../langsmith-v0-5-11-wrapped.span-tree.json | 205 ++++++ .../langsmith-v0-5-11-wrapped.span-tree.txt | 163 +++++ .../langsmith-v0-8-1-auto-cjs.span-tree.json | 205 ++++++ .../langsmith-v0-8-1-auto-cjs.span-tree.txt | 163 +++++ .../langsmith-v0-8-1-auto-esm.span-tree.json | 235 +++++++ .../langsmith-v0-8-1-auto-esm.span-tree.txt | 189 ++++++ .../langsmith-v0-8-1-wrapped.span-tree.json | 205 ++++++ .../langsmith-v0-8-1-wrapped.span-tree.txt | 163 +++++ .../langsmith-instrumentation/assertions.ts | 159 +++++ .../langsmith-instrumentation/package.json | 20 + .../langsmith-instrumentation/pnpm-lock.yaml | 615 ++++++++++++++++++ .../scenario.auto.cjs | 21 + .../scenario.auto.mjs | 17 + .../scenario.impl.mjs | 299 +++++++++ .../scenario.test.ts | 106 +++ .../scenario.wrapper.ts | 16 + js/README.md | 43 ++ js/src/auto-instrumentations/configs/all.ts | 2 + .../configs/langsmith.ts | 36 + js/src/auto-instrumentations/index.ts | 1 + js/src/exports.ts | 5 + .../instrumentation/braintrust-plugin.test.ts | 38 ++ js/src/instrumentation/braintrust-plugin.ts | 14 + js/src/instrumentation/config.ts | 3 + .../plugins/langsmith-channels.ts | 30 + .../plugins/langsmith-plugin.test.ts | 249 +++++++ .../plugins/langsmith-plugin.ts | 557 ++++++++++++++++ js/src/vendor-sdk-types/langsmith.ts | 77 +++ js/src/wrappers/langsmith.test.ts | 172 +++++ js/src/wrappers/langsmith.ts | 352 ++++++++++ 39 files changed, 5477 insertions(+) create mode 100644 .changeset/sixty-seals-judge.md create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.json create mode 100644 e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.txt create mode 100644 e2e/scenarios/langsmith-instrumentation/assertions.ts create mode 100644 e2e/scenarios/langsmith-instrumentation/package.json create mode 100644 e2e/scenarios/langsmith-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/langsmith-instrumentation/scenario.auto.cjs create mode 100644 e2e/scenarios/langsmith-instrumentation/scenario.auto.mjs create mode 100644 e2e/scenarios/langsmith-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/langsmith-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/langsmith-instrumentation/scenario.wrapper.ts create mode 100644 js/src/auto-instrumentations/configs/langsmith.ts create mode 100644 js/src/instrumentation/plugins/langsmith-channels.ts create mode 100644 js/src/instrumentation/plugins/langsmith-plugin.test.ts create mode 100644 js/src/instrumentation/plugins/langsmith-plugin.ts create mode 100644 js/src/vendor-sdk-types/langsmith.ts create mode 100644 js/src/wrappers/langsmith.test.ts create mode 100644 js/src/wrappers/langsmith.ts diff --git a/.changeset/sixty-seals-judge.md b/.changeset/sixty-seals-judge.md new file mode 100644 index 000000000..a20431d32 --- /dev/null +++ b/.changeset/sixty-seals-judge.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add `langsmith` tracing diff --git a/.gitleaks.toml b/.gitleaks.toml index e155b52e3..bf994b888 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -4,6 +4,16 @@ title = "Braintrust SDK gitleaks config" # Use the default gitleaks ruleset as the base useDefault = true +[[allowlists]] +description = "LangSmith E2E variant identifiers are not API keys" +condition = "AND" +targetRules = ["generic-api-key"] +regexes = ['''^langsmith-v0-(3-30|5-11|8-1)$'''] +paths = [ + '''^e2e/config/pr-comment-scenarios\.json$''', + '''^e2e/scenarios/langsmith-instrumentation/scenario\.test\.ts$''', +] + [[rules]] id = "braintrust-api-key" description = "Braintrust API key" diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 30ada879e..845716169 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -139,6 +139,16 @@ "label": "LangGraph Auto-Instrumentation", "metadataScenario": "langgraph-auto-instrumentation" }, + { + "scenarioDirName": "langsmith-instrumentation", + "label": "LangSmith Instrumentation", + "metadataScenario": "langsmith-instrumentation", + "variants": [ + { "variantKey": "langsmith-v0-3-30", "label": "v0.3.30" }, + { "variantKey": "langsmith-v0-5-11", "label": "v0.5.11" }, + { "variantKey": "langsmith-v0-8-1", "label": "v0.8.1" } + ] + }, { "scenarioDirName": "mistral-instrumentation", "label": "Mistral Instrumentation", diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.json new file mode 100644 index 000000000..0dcd682ed --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.json @@ -0,0 +1,202 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.txt new file mode 100644 index 000000000..b365a2866 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-auto-esm.span-tree.txt @@ -0,0 +1,160 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.json new file mode 100644 index 000000000..0dcd682ed --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.json @@ -0,0 +1,202 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.txt new file mode 100644 index 000000000..b365a2866 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-3-30-wrapped.span-tree.txt @@ -0,0 +1,160 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.json new file mode 100644 index 000000000..dbc726ba2 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.json @@ -0,0 +1,205 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "customer_id": "customer-123", + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_id": "customer-123", + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider", + "scenario": "langsmith-instrumentation" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.txt new file mode 100644 index 000000000..aae5df2b8 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-auto-esm.span-tree.txt @@ -0,0 +1,163 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "customer_id": "customer-123", +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider", +│ "scenario": "langsmith-instrumentation" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.json new file mode 100644 index 000000000..dbc726ba2 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.json @@ -0,0 +1,205 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "customer_id": "customer-123", + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_id": "customer-123", + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider", + "scenario": "langsmith-instrumentation" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.txt new file mode 100644 index 000000000..aae5df2b8 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-5-11-wrapped.span-tree.txt @@ -0,0 +1,163 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "customer_id": "customer-123", +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider", +│ "scenario": "langsmith-instrumentation" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.json new file mode 100644 index 000000000..dbc726ba2 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.json @@ -0,0 +1,205 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "customer_id": "customer-123", + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_id": "customer-123", + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider", + "scenario": "langsmith-instrumentation" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.txt new file mode 100644 index 000000000..aae5df2b8 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-cjs.span-tree.txt @@ -0,0 +1,163 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "customer_id": "customer-123", +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider", +│ "scenario": "langsmith-instrumentation" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.json new file mode 100644 index 000000000..a21ed05c5 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.json @@ -0,0 +1,235 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "customer_id": "customer-123", + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_id": "customer-123", + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider", + "scenario": "langsmith-instrumentation" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "langchain-dedupe", + "type": "task", + "children": [], + "input": { + "input": "provider-free" + }, + "output": { + "output": "provider-free!" + }, + "metadata": { + "braintrust": { + "integration_name": "langchain-js", + "sdk_language": "javascript" + }, + "metadata": {}, + "name": "langchain-dedupe", + "run_id": "", + "serialized": { + "id": [ + "langchain_core", + "runnables", + "RunnableLambda" + ], + "lc": 1, + "type": "not_implemented" + }, + "tags": [] + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.txt new file mode 100644 index 000000000..5503ccbb8 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-auto-esm.span-tree.txt @@ -0,0 +1,189 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "customer_id": "customer-123", +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider", +│ "scenario": "langsmith-instrumentation" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +├── offline-document-enrichment [task] +│ input: { +│ "document_count": 1 +│ } +│ output: { +│ "enriched_documents": 1 +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── extract-keywords [tool] +│ input: { +│ "document": "Refund requests are accepted within 30 days." +│ } +│ output: { +│ "keywords": [ +│ "refund", +│ "30 days" +│ ] +│ } +└── langchain-dedupe [task] + input: { + "input": "provider-free" + } + output: { + "output": "provider-free!" + } + metadata: { + "braintrust": { + "integration_name": "langchain-js", + "sdk_language": "javascript" + }, + "metadata": {}, + "name": "langchain-dedupe", + "run_id": "", + "serialized": { + "id": [ + "langchain_core", + "runnables", + "RunnableLambda" + ], + "lc": 1, + "type": "not_implemented" + }, + "tags": [] + } diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.json b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.json new file mode 100644 index 000000000..dbc726ba2 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.json @@ -0,0 +1,205 @@ +{ + "span_tree": [ + { + "name": "support-assistant", + "type": "task", + "children": [ + { + "name": "retrieve-context", + "type": "tool", + "children": [], + "input": { + "query": "What is the refund window?" + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "tags": [ + "knowledge-base" + ], + "metadata": { + "customer_id": "customer-123", + "data_source": "support-kb", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": { + "args": [ + { + "max_tokens": 64, + "messages": [ + { + "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", + "role": "system" + }, + { + "content": "What is the refund window?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "temperature": 0 + }, + {} + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Refund requests are accepted within 30 days of purchase.", + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion" + }, + "tags": [ + "openai", + "chat-completion" + ], + "metadata": { + "customer_id": "customer-123", + "customer_tier": "enterprise", + "deployment": "local-fixture", + "max_tokens": 64, + "model": "gpt-4o-mini", + "provider": "openai", + "scenario": "langsmith-instrumentation", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 4, + "prompt_tokens": 18, + "tokens": 25 + } + } + ], + "input": { + "question": "What is the refund window?" + }, + "output": { + "answer": "Refund requests are accepted within 30 days of purchase.", + "source": "Refund policy" + }, + "tags": [ + "support" + ], + "metadata": { + "customer_id": "customer-123", + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "lookup-customer-record", + "type": "tool", + "children": [], + "metadata": { + "scenario": "langsmith-instrumentation" + }, + "error": "Error: customer record unavailable" + }, + { + "name": "manual-rag-pipeline", + "type": "task", + "children": [ + { + "name": "embed-query", + "type": "llm", + "children": [], + "input": { + "text": "Which policy applies?" + }, + "output": { + "embedding": [ + 0.1, + 0.2 + ] + }, + "tags": [ + "manual" + ], + "metadata": { + "model": "manual-model", + "provider": "manual-provider", + "scenario": "langsmith-instrumentation" + }, + "metrics": { + "completion_tokens": 0, + "prompt_cached_tokens": 1, + "prompt_tokens": 3, + "time_to_first_token": 0, + "tokens": 3 + } + } + ], + "input": { + "question": "Which policy applies?" + }, + "output": { + "matched_document": "Refund policy" + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "direct-search-index", + "type": "tool", + "children": [], + "input": { + "query": "refund window", + "top_k": 3 + }, + "output": { + "documents": [ + "Refund policy" + ] + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + }, + { + "name": "offline-document-enrichment", + "type": "task", + "children": [ + { + "name": "extract-keywords", + "type": "tool", + "children": [], + "input": { + "document": "Refund requests are accepted within 30 days." + }, + "output": { + "keywords": [ + "refund", + "30 days" + ] + } + } + ], + "input": { + "document_count": 1 + }, + "output": { + "enriched_documents": 1 + }, + "metadata": { + "scenario": "langsmith-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.txt b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.txt new file mode 100644 index 000000000..aae5df2b8 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/__snapshots__/langsmith-v0-8-1-wrapped.span-tree.txt @@ -0,0 +1,163 @@ +span_tree: +├── support-assistant [task] +│ input: { +│ "question": "What is the refund window?" +│ } +│ output: { +│ "answer": "Refund requests are accepted within 30 days of purchase.", +│ "source": "Refund policy" +│ } +│ tags: [ +│ "support" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "scenario": "langsmith-instrumentation" +│ } +│ ├── retrieve-context [tool] +│ │ input: { +│ │ "query": "What is the refund window?" +│ │ } +│ │ output: { +│ │ "documents": [ +│ │ "Refund policy" +│ │ ] +│ │ } +│ │ tags: [ +│ │ "knowledge-base" +│ │ ] +│ │ metadata: { +│ │ "customer_id": "customer-123", +│ │ "data_source": "support-kb", +│ │ "scenario": "langsmith-instrumentation" +│ │ } +│ └── ChatOpenAI [llm] +│ input: { +│ "args": [ +│ { +│ "max_tokens": 64, +│ "messages": [ +│ { +│ "content": "Answer using only this context: Refund requests are accepted within 30 days of purchase.", +│ "role": "system" +│ }, +│ { +│ "content": "What is the refund window?", +│ "role": "user" +│ } +│ ], +│ "model": "gpt-4o-mini", +│ "temperature": 0 +│ }, +│ {} +│ ] +│ } +│ output: { +│ "choices": [ +│ { +│ "finish_reason": "stop", +│ "index": 0, +│ "message": { +│ "content": "Refund requests are accepted within 30 days of purchase.", +│ "role": "assistant" +│ } +│ } +│ ], +│ "created": 0, +│ "id": "", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "chat.completion" +│ } +│ tags: [ +│ "openai", +│ "chat-completion" +│ ] +│ metadata: { +│ "customer_id": "customer-123", +│ "customer_tier": "enterprise", +│ "deployment": "local-fixture", +│ "max_tokens": 64, +│ "model": "gpt-4o-mini", +│ "provider": "openai", +│ "scenario": "langsmith-instrumentation", +│ "temperature": 0 +│ } +│ metrics: { +│ "completion_tokens": 7, +│ "prompt_cached_tokens": 4, +│ "prompt_tokens": 18, +│ "tokens": 25 +│ } +├── lookup-customer-record [tool] +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ error: "Error: customer record unavailable" +├── manual-rag-pipeline [task] +│ input: { +│ "question": "Which policy applies?" +│ } +│ output: { +│ "matched_document": "Refund policy" +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +│ └── embed-query [llm] +│ input: { +│ "text": "Which policy applies?" +│ } +│ output: { +│ "embedding": [ +│ 0.1, +│ 0.2 +│ ] +│ } +│ tags: [ +│ "manual" +│ ] +│ metadata: { +│ "model": "manual-model", +│ "provider": "manual-provider", +│ "scenario": "langsmith-instrumentation" +│ } +│ metrics: { +│ "completion_tokens": 0, +│ "prompt_cached_tokens": 1, +│ "prompt_tokens": 3, +│ "time_to_first_token": 0, +│ "tokens": 3 +│ } +├── direct-search-index [tool] +│ input: { +│ "query": "refund window", +│ "top_k": 3 +│ } +│ output: { +│ "documents": [ +│ "Refund policy" +│ ] +│ } +│ metadata: { +│ "scenario": "langsmith-instrumentation" +│ } +└── offline-document-enrichment [task] + input: { + "document_count": 1 + } + output: { + "enriched_documents": 1 + } + metadata: { + "scenario": "langsmith-instrumentation" + } + └── extract-keywords [tool] + input: { + "document": "Refund requests are accepted within 30 days." + } + output: { + "keywords": [ + "refund", + "30 days" + ] + } diff --git a/e2e/scenarios/langsmith-instrumentation/assertions.ts b/e2e/scenarios/langsmith-instrumentation/assertions.ts new file mode 100644 index 000000000..9f2830164 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/assertions.ts @@ -0,0 +1,159 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import type { ScenarioRunContext } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { findLatestSpan } from "../../helpers/trace-selectors"; + +type RunLangSmithScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + env?: Record; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + env?: Record; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +const expectedNames = [ + "support-assistant", + "retrieve-context", + "ChatOpenAI", + "lookup-customer-record", + "manual-rag-pipeline", + "embed-query", + "direct-search-index", + "offline-document-enrichment", + "extract-keywords", +]; + +export function defineLangSmithInstrumentationAssertions(options: { + includeLangChain?: boolean; + name: string; + runScenario: RunLangSmithScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + const testConfig = { timeout: options.timeoutMs }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures the complete LangSmith lifecycle", testConfig, () => { + for (const name of expectedNames) { + expect(findLatestSpan(events, name), `missing ${name}`).toBeDefined(); + } + + const roots = [ + "support-assistant", + "lookup-customer-record", + "manual-rag-pipeline", + "direct-search-index", + "offline-document-enrichment", + ].map((name) => findLatestSpan(events, name)); + for (const root of roots) { + expect(root?.span.parentIds).toEqual([]); + expect(root?.span.rootId).toBe(root?.span.id); + } + + const assistant = findLatestSpan(events, "support-assistant"); + const retrieval = findLatestSpan(events, "retrieve-context"); + const llm = findLatestSpan(events, "ChatOpenAI"); + expect( + events.filter((event) => event.span.name === "ChatOpenAI"), + ).toHaveLength(1); + expect(retrieval?.span.parentIds).toEqual([assistant?.span.id ?? ""]); + expect(retrieval?.row).toMatchObject({ + input: { query: "What is the refund window?" }, + metadata: { + data_source: "support-kb", + scenario: "langsmith-instrumentation", + }, + output: { documents: ["Refund policy"] }, + tags: ["knowledge-base"], + }); + expect(llm?.span.parentIds).toEqual([assistant?.span.id ?? ""]); + expect(llm?.row).toMatchObject({ + metadata: { + customer_tier: "enterprise", + deployment: "local-fixture", + max_tokens: 64, + model: "gpt-4o-mini", + provider: "openai", + scenario: "langsmith-instrumentation", + temperature: 0, + }, + metrics: { + completion_tokens: 7, + prompt_cached_tokens: 4, + prompt_tokens: 18, + tokens: 25, + }, + tags: expect.arrayContaining(["chat-completion", "openai"]), + }); + expect(assistant?.output).toEqual({ + answer: "Refund requests are accepted within 30 days of purchase.", + source: "Refund policy", + }); + + const failed = findLatestSpan(events, "lookup-customer-record"); + expect(failed?.row.error).toContain("customer record unavailable"); + + const manualParent = findLatestSpan(events, "manual-rag-pipeline"); + const manualChild = findLatestSpan(events, "embed-query"); + expect(manualChild?.span.parentIds).toEqual([ + manualParent?.span.id ?? "", + ]); + expect(manualChild?.span.type).toBe("llm"); + expect(manualChild?.metrics).toMatchObject({ + prompt_cached_tokens: 1, + prompt_tokens: 3, + time_to_first_token: expect.any(Number), + tokens: 3, + }); + + const clientRun = findLatestSpan(events, "direct-search-index"); + expect(clientRun?.span.id).toBe("11111111-1111-4111-8111-111111111111"); + expect(clientRun?.span.type).toBe("tool"); + expect(clientRun?.output).toEqual({ documents: ["Refund policy"] }); + + const batchParent = findLatestSpan(events, "offline-document-enrichment"); + const batchChild = findLatestSpan(events, "extract-keywords"); + expect(batchChild?.span.parentIds).toEqual([batchParent?.span.id ?? ""]); + expect(batchChild?.span.type).toBe("tool"); + }); + + if (options.includeLangChain) { + test("does not duplicate LangChain callback spans", testConfig, () => { + expect( + events.filter((event) => event.span.name === "langchain-dedupe"), + ).toHaveLength(1); + }); + } + + test("matches span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/langsmith-instrumentation/package.json b/e2e/scenarios/langsmith-instrumentation/package.json new file mode 100644 index 000000000..216f96b3c --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/package.json @@ -0,0 +1,20 @@ +{ + "name": "@braintrust/e2e-langsmith-instrumentation", + "private": true, + "type": "module", + "braintrustScenario": { + "canary": { + "dependencies": { + "langsmith-v081": "langsmith@latest" + } + } + }, + "dependencies": { + "@langchain/core": "1.1.36", + "langsmith": "0.8.1", + "langsmith-v0330": "npm:langsmith@0.3.30", + "langsmith-v0511": "npm:langsmith@0.5.11", + "langsmith-v081": "npm:langsmith@0.8.1", + "openai": "4.104.0" + } +} diff --git a/e2e/scenarios/langsmith-instrumentation/pnpm-lock.yaml b/e2e/scenarios/langsmith-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..34a57e7d4 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/pnpm-lock.yaml @@ -0,0 +1,615 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@langchain/core': + specifier: 1.1.36 + version: 1.1.36(openai@4.104.0) + langsmith: + specifier: 0.8.1 + version: 0.8.1(openai@4.104.0) + langsmith-v0330: + specifier: npm:langsmith@0.3.30 + version: langsmith@0.3.30(openai@4.104.0) + langsmith-v0511: + specifier: npm:langsmith@0.5.11 + version: langsmith@0.5.11(openai@4.104.0) + langsmith-v081: + specifier: npm:langsmith@0.8.1 + version: langsmith@0.8.1(openai@4.104.0) + openai: + specifier: 4.104.0 + version: 4.104.0 + +packages: + + '@cfworker/json-schema@4.1.1': + resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + + '@langchain/core@1.1.36': + resolution: {integrity: sha512-9NWsdzU3uZD13lJwunXK0t6SIwew+UwcbHggW5yUdaiMmzKeNkDpp1lRD6p49N8+D0Vv4qmQBEKB4Ukh2jfnvw==} + engines: {node: '>=20'} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + console-table-printer@2.16.1: + resolution: {integrity: sha512-Sc9FRJ4O9xKGNrvulNdPfK5SyBcZ6lcaRnDE4AQ/uw6IDtjHhsqyzzqcnMikjyGaiOOF2tNOKoBhbVjRvFy9Lw==} + + decamelize@1.2.0: + resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} + engines: {node: '>=0.10.0'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + js-tiktoken@1.0.21: + resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==} + + langsmith@0.3.30: + resolution: {integrity: sha512-ZaiaOx9MysuSQlAkRw8mjm7iqhrlF7HI0LCTLxiNBEWBPywdkgI7UnN+s7KtlRiM0tP1cOLm+dQY++Fi33jkPQ==} + peerDependencies: + openai: '*' + peerDependenciesMeta: + openai: + optional: true + + langsmith@0.5.11: + resolution: {integrity: sha512-Yio502Ow2vbVt16P1sybNMNpMsr5BMqoeonoi4flrcDsP55No/aCe2zydtBNOv0+kjKQw4WSKAzTsNwenDeD5w==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + ws: '>=7' + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + ws: + optional: true + + langsmith@0.8.1: + resolution: {integrity: sha512-Y48oSj51aLvehAVwy+VU2i9COlCFmLi9T5REFyL4pWT2npcIKiBSiSlnyjxdkNXOdZ0YYz7Lud8aiZPjkjkXvA==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + ws: '>=7' + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + ws: + optional: true + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + simple-wcswidth@1.1.2: + resolution: {integrity: sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + +snapshots: + + '@cfworker/json-schema@4.1.1': {} + + '@langchain/core@1.1.36(openai@4.104.0)': + dependencies: + '@cfworker/json-schema': 4.1.1 + '@standard-schema/spec': 1.1.0 + ansi-styles: 5.2.0 + camelcase: 6.3.0 + decamelize: 1.2.0 + js-tiktoken: 1.0.21 + langsmith: 0.8.1(openai@4.104.0) + mustache: 4.2.0 + p-queue: 6.6.2 + uuid: 11.1.1 + zod: 4.4.3 + transitivePeerDependencies: + - '@opentelemetry/api' + - '@opentelemetry/exporter-trace-otlp-proto' + - '@opentelemetry/sdk-trace-base' + - openai + - ws + + '@standard-schema/spec@1.1.0': {} + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 18.19.130 + form-data: 4.0.6 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + + '@types/retry@0.12.0': {} + + '@types/uuid@10.0.0': {} + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@5.2.0: {} + + asynckit@0.4.0: {} + + base64-js@1.5.1: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + camelcase@6.3.0: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + chalk@5.6.2: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + console-table-printer@2.16.1: + dependencies: + simple-wcswidth: 1.1.2 + + decamelize@1.2.0: {} + + delayed-stream@1.0.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + + event-target-shim@5.0.1: {} + + eventemitter3@4.0.7: {} + + form-data-encoder@1.7.2: {} + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + gopd@1.2.0: {} + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + js-tiktoken@1.0.21: + dependencies: + base64-js: 1.5.1 + + langsmith@0.3.30(openai@4.104.0): + dependencies: + '@types/uuid': 10.0.0 + chalk: 4.1.2 + console-table-printer: 2.16.1 + p-queue: 6.6.2 + p-retry: 4.6.2 + semver: 7.8.5 + uuid: 10.0.0 + optionalDependencies: + openai: 4.104.0 + + langsmith@0.5.11(openai@4.104.0): + dependencies: + '@types/uuid': 10.0.0 + chalk: 5.6.2 + console-table-printer: 2.16.1 + p-queue: 6.6.2 + semver: 7.8.5 + uuid: 10.0.0 + optionalDependencies: + openai: 4.104.0 + + langsmith@0.8.1(openai@4.104.0): + dependencies: + p-queue: 6.6.2 + optionalDependencies: + openai: 4.104.0 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + ms@2.1.3: {} + + mustache@4.2.0: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + openai@4.104.0: + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + p-finally@1.0.0: {} + + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + + retry@0.13.1: {} + + semver@7.8.5: {} + + simple-wcswidth@1.1.2: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + tr46@0.0.3: {} + + undici-types@5.26.5: {} + + uuid@10.0.0: {} + + uuid@11.1.1: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + + zod@4.4.3: {} diff --git a/e2e/scenarios/langsmith-instrumentation/scenario.auto.cjs b/e2e/scenarios/langsmith-instrumentation/scenario.auto.cjs new file mode 100644 index 000000000..9df68cc55 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/scenario.auto.cjs @@ -0,0 +1,21 @@ +const root = require("langsmith-v081"); +const client = require("langsmith-v081/client"); +const openAI = require("openai"); +const openAIWrapper = require("langsmith-v081/wrappers"); +const runTrees = require("langsmith-v081/run_trees"); +const traceable = require("langsmith-v081/traceable"); + +void (async () => { + const { runMain } = await import("../../helpers/provider-runtime.mjs"); + const { runLangSmithScenario } = await import("./scenario.impl.mjs"); + + runMain(async () => { + await runLangSmithScenario({ + namespaces: { root, client, openAI, openAIWrapper, runTrees, traceable }, + wrapped: false, + }); + }); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/e2e/scenarios/langsmith-instrumentation/scenario.auto.mjs b/e2e/scenarios/langsmith-instrumentation/scenario.auto.mjs new file mode 100644 index 000000000..29083a72f --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/scenario.auto.mjs @@ -0,0 +1,17 @@ +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { + loadLangSmithNamespaces, + runLangSmithScenario, +} from "./scenario.impl.mjs"; + +runMain(async () => { + const dependencyName = process.env.LANGSMITH_PACKAGE_NAME; + if (!dependencyName) { + throw new Error("LANGSMITH_PACKAGE_NAME is required"); + } + await runLangSmithScenario({ + includeLangChain: process.env.LANGSMITH_INCLUDE_LANGCHAIN === "1", + namespaces: await loadLangSmithNamespaces(dependencyName), + wrapped: false, + }); +}); diff --git a/e2e/scenarios/langsmith-instrumentation/scenario.impl.mjs b/e2e/scenarios/langsmith-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..ae9735637 --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/scenario.impl.mjs @@ -0,0 +1,299 @@ +import { + initLogger, + wrapLangSmithClient, + wrapLangSmithRunTrees, + wrapLangSmithTraceable, +} from "braintrust"; +import { scopedName } from "../../helpers/provider-runtime.mjs"; + +export const LANGSMITH_SCENARIO_TIMEOUT_MS = 120_000; +export const LANGSMITH_SCENARIO_SPECS = [ + { + dependencyName: "langsmith-v0330", + snapshotName: "langsmith-v0-3-30", + }, + { + dependencyName: "langsmith-v0511", + snapshotName: "langsmith-v0-5-11", + }, + { + dependencyName: "langsmith-v081", + snapshotName: "langsmith-v0-8-1", + }, +]; + +const SCENARIO_NAME = "langsmith-instrumentation"; +const IDS = { + client: "11111111-1111-4111-8111-111111111111", + batchParent: "22222222-2222-4222-8222-222222222222", + batchChild: "33333333-3333-4333-8333-333333333333", +}; + +export async function loadLangSmithNamespaces(dependencyName) { + return { + root: await import(dependencyName), + client: await import(`${dependencyName}/client`), + openAI: await import("openai"), + openAIWrapper: await import(`${dependencyName}/wrappers`), + runTrees: await import(`${dependencyName}/run_trees`), + traceable: await import(`${dependencyName}/traceable`), + }; +} + +export async function runLangSmithScenario({ + includeLangChain = false, + namespaces, + wrapped, +}) { + const rootNamespace = namespaces.root; + const clientNamespace = wrapped + ? wrapLangSmithClient(namespaces.client) + : namespaces.client; + const runTreesNamespace = wrapped + ? wrapLangSmithRunTrees(namespaces.runTrees) + : namespaces.runTrees; + const traceableNamespace = wrapped + ? wrapLangSmithTraceable(namespaces.traceable) + : namespaces.traceable; + + rootNamespace.overrideFetchImplementation( + async () => + new Response(JSON.stringify({}), { + headers: { "content-type": "application/json" }, + status: 202, + }), + ); + + const logger = initLogger({ + projectName: scopedName("e2e-langsmith-instrumentation"), + }); + const client = new clientNamespace.Client({ + apiKey: "ls-test-key", + apiUrl: "http://langsmith.invalid", + autoBatchTracing: false, + timeout_ms: 5_000, + }); + + const retrieveContext = traceableNamespace.traceable( + async ({ question }) => [ + { + content: "Refund requests are accepted within 30 days of purchase.", + title: "Refund policy", + }, + ], + { + client, + metadata: { + data_source: "support-kb", + scenario: SCENARIO_NAME, + }, + name: "retrieve-context", + processInputs: (inputs) => ({ query: inputs.question }), + processOutputs: ({ outputs }) => ({ + documents: outputs.map((document) => document.title), + }), + run_type: "retriever", + tags: ["knowledge-base"], + }, + ); + + const openAI = namespaces.openAIWrapper.wrapOpenAI( + new namespaces.openAI.default({ + apiKey: "openai-test-key", + baseURL: "https://openai.invalid/v1", + fetch: async () => + new Response( + JSON.stringify({ + choices: [ + { + finish_reason: "stop", + index: 0, + message: { + content: + "Refund requests are accepted within 30 days of purchase.", + role: "assistant", + }, + }, + ], + created: 1_783_900_800, + id: "chatcmpl-langsmith-e2e", + model: "gpt-4o-mini-2024-07-18", + object: "chat.completion", + usage: { + completion_tokens: 7, + prompt_tokens: 18, + prompt_tokens_details: { cached_tokens: 4 }, + total_tokens: 25, + }, + }), + { + headers: { "content-type": "application/json" }, + status: 200, + }, + ), + }), + { + client, + metadata: { deployment: "local-fixture", scenario: SCENARIO_NAME }, + tags: ["openai"], + }, + ); + + const supportAssistant = traceableNamespace.traceable( + async ({ question }) => { + const documents = await retrieveContext({ question }); + const completion = await openAI.chat.completions.create( + { + max_tokens: 64, + messages: [ + { + content: `Answer using only this context: ${documents[0].content}`, + role: "system", + }, + { content: question, role: "user" }, + ], + model: "gpt-4o-mini", + temperature: 0, + }, + { + langsmithExtra: { + metadata: { customer_tier: "enterprise" }, + tags: ["chat-completion"], + }, + }, + ); + + return { + answer: completion.choices[0].message.content, + source: documents[0].title, + }; + }, + { + client, + metadata: { + customer_id: "customer-123", + scenario: SCENARIO_NAME, + }, + name: "support-assistant", + run_type: "chain", + tags: ["support"], + }, + ); + await supportAssistant({ question: "What is the refund window?" }); + + const failedTraceable = traceableNamespace.traceable( + async () => { + throw new Error("customer record unavailable"); + }, + { + client, + metadata: { scenario: SCENARIO_NAME }, + name: "lookup-customer-record", + run_type: "tool", + }, + ); + await failedTraceable().catch(() => undefined); + + const manualParent = new runTreesNamespace.RunTree({ + client, + extra: { metadata: { scenario: SCENARIO_NAME } }, + inputs: { question: "Which policy applies?" }, + name: "manual-rag-pipeline", + run_type: "chain", + }); + await manualParent.postRun(); + const manualChild = manualParent.createChild({ + extra: { + metadata: { + ls_model_name: "manual-model", + ls_provider: "manual-provider", + }, + }, + inputs: { text: "Which policy applies?" }, + name: "embed-query", + run_type: "embedding", + tags: ["manual"], + }); + await manualChild.postRun(); + manualChild.addEvent({ name: "new_token", time: new Date().toISOString() }); + await manualChild.end({ + embedding: [0.1, 0.2], + usage_metadata: { + input_tokens: 3, + output_tokens: 0, + total_tokens: 3, + input_token_details: { cache_read: 1 }, + }, + }); + await manualChild.patchRun(); + await manualParent.end({ matched_document: "Refund policy" }); + await manualParent.patchRun(); + + const now = Date.now(); + await client.createRun({ + dotted_order: `20260713T000000000000Z${IDS.client}`, + extra: { metadata: { scenario: SCENARIO_NAME } }, + id: IDS.client, + inputs: { query: "refund window", top_k: 3 }, + name: "direct-search-index", + run_type: "retriever", + start_time: now, + trace_id: IDS.client, + }); + await client.updateRun(IDS.client, { + end_time: now + 10, + outputs: { documents: ["Refund policy"] }, + trace_id: IDS.client, + }); + + await client.batchIngestRuns({ + runCreates: [ + { + dotted_order: `20260713T000000000000Z${IDS.batchParent}.20260713T000000000001Z${IDS.batchChild}`, + id: IDS.batchChild, + inputs: { document: "Refund requests are accepted within 30 days." }, + name: "extract-keywords", + parent_run_id: IDS.batchParent, + run_type: "tool", + start_time: now, + trace_id: IDS.batchParent, + }, + { + dotted_order: `20260713T000000000000Z${IDS.batchParent}`, + extra: { metadata: { scenario: SCENARIO_NAME } }, + id: IDS.batchParent, + inputs: { document_count: 1 }, + name: "offline-document-enrichment", + run_type: "prompt", + start_time: now, + trace_id: IDS.batchParent, + }, + ], + runUpdates: [ + { + end_time: now + 20, + id: IDS.batchChild, + outputs: { keywords: ["refund", "30 days"] }, + parent_run_id: IDS.batchParent, + trace_id: IDS.batchParent, + }, + { + end_time: now + 30, + id: IDS.batchParent, + outputs: { enriched_documents: 1 }, + trace_id: IDS.batchParent, + }, + ], + }); + + if (includeLangChain) { + const { RunnableLambda } = await import("@langchain/core/runnables"); + const runnable = RunnableLambda.from( + async (value) => `${value}!`, + ).withConfig({ runName: "langchain-dedupe" }); + await runnable.invoke("provider-free"); + } + + await client.flush?.(); + await logger.flush(); +} diff --git a/e2e/scenarios/langsmith-instrumentation/scenario.test.ts b/e2e/scenarios/langsmith-instrumentation/scenario.test.ts new file mode 100644 index 000000000..618a39bfe --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/scenario.test.ts @@ -0,0 +1,106 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineLangSmithInstrumentationAssertions } from "./assertions"; +import { + LANGSMITH_SCENARIO_SPECS, + LANGSMITH_SCENARIO_TIMEOUT_MS, +} from "./scenario.impl.mjs"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const scenarios = await Promise.all( + LANGSMITH_SCENARIO_SPECS.map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); +const baseEnv = { + BRAINTRUST_DISABLE_INSTRUMENTATION: "openai", + LANGCHAIN_TRACING_V2: "true", + LANGSMITH_API_KEY: "ls-test-key", + LANGSMITH_ENDPOINT: "http://langsmith.invalid", + LANGSMITH_TRACING: "true", +}; + +describe.concurrent("variants", () => { + for (const scenario of scenarios) { + describe.sequential(`langsmith sdk ${scenario.version}`, () => { + defineLangSmithInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.wrapper.ts", + env: { + ...baseEnv, + LANGSMITH_PACKAGE_NAME: scenario.dependencyName, + }, + runContext: { + originalScenarioDir, + variantKey: scenario.snapshotName, + }, + scenarioDir, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.snapshotName}-wrapped`, + testFileUrl: import.meta.url, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, + }); + + defineLangSmithInstrumentationAssertions({ + includeLangChain: scenario.dependencyName === "langsmith-v081", + name: "auto-hook instrumentation ESM", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.auto.mjs", + env: { + ...baseEnv, + LANGSMITH_INCLUDE_LANGCHAIN: + scenario.dependencyName === "langsmith-v081" ? "1" : "0", + LANGSMITH_PACKAGE_NAME: scenario.dependencyName, + }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + originalScenarioDir, + variantKey: scenario.snapshotName, + }, + scenarioDir, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.snapshotName}-auto-esm`, + testFileUrl: import.meta.url, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, + }); + }); + } +}); + +defineLangSmithInstrumentationAssertions({ + name: "latest auto-hook instrumentation CJS", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.auto.cjs", + env: baseEnv, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + originalScenarioDir, + variantKey: "langsmith-v0-8-1", + }, + scenarioDir, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: "langsmith-v0-8-1-auto-cjs", + testFileUrl: import.meta.url, + timeoutMs: LANGSMITH_SCENARIO_TIMEOUT_MS, +}); diff --git a/e2e/scenarios/langsmith-instrumentation/scenario.wrapper.ts b/e2e/scenarios/langsmith-instrumentation/scenario.wrapper.ts new file mode 100644 index 000000000..345d1198d --- /dev/null +++ b/e2e/scenarios/langsmith-instrumentation/scenario.wrapper.ts @@ -0,0 +1,16 @@ +import { runMain } from "../../helpers/scenario-runtime"; +import { + loadLangSmithNamespaces, + runLangSmithScenario, +} from "./scenario.impl.mjs"; + +runMain(async () => { + const dependencyName = process.env.LANGSMITH_PACKAGE_NAME; + if (!dependencyName) { + throw new Error("LANGSMITH_PACKAGE_NAME is required"); + } + await runLangSmithScenario({ + namespaces: await loadLangSmithNamespaces(dependencyName), + wrapped: true, + }); +}); diff --git a/js/README.md b/js/README.md index 5a104a833..da6da81a6 100644 --- a/js/README.md +++ b/js/README.md @@ -104,6 +104,49 @@ If you use TypeScript or other transpilation plugins, place the Braintrust plugi For deeper details, see the [auto-instrumentation architecture docs](src/auto-instrumentations/README.md). +### LangSmith tracing + +Braintrust supports LangSmith `>=0.3.30 <1.0.0`. LangSmith tracing remains authoritative: LangSmith must be enabled, and it continues exporting traces to LangSmith while Braintrust mirrors the same run lifecycle. This integration covers tracing only; LangSmith eval, Jest, and Vitest APIs are not instrumented. + +For automatic Node.js instrumentation, use the standard hook before importing LangSmith: + +```bash +node --import braintrust/hook.mjs app.js +``` + +The Vite, Webpack, esbuild, and Rollup plugins shown above apply the same automatic instrumentation in bundled applications. To instrument explicit namespaces instead, wrap the three LangSmith entrypoints you use: + +```typescript +import { + wrapLangSmithClient, + wrapLangSmithRunTrees, + wrapLangSmithTraceable, +} from "braintrust"; +import * as clientNamespace from "langsmith/client"; +import * as runTreesNamespace from "langsmith/run_trees"; +import * as traceableNamespace from "langsmith/traceable"; + +const { Client } = wrapLangSmithClient(clientNamespace); +const { RunTree } = wrapLangSmithRunTrees(runTreesNamespace); +const { traceable } = wrapLangSmithTraceable(traceableNamespace); +``` + +The wrappers are composable and idempotent. They preserve LangSmith behavior, including its network export and `on_end` callbacks. Automatic and explicit instrumentation can safely be used together. + +Disable LangSmith instrumentation in code or through the environment: + +```typescript +import { configureInstrumentation } from "braintrust"; + +configureInstrumentation({ integrations: { langsmith: false } }); +``` + +```bash +BRAINTRUST_DISABLE_INSTRUMENTATION=langsmith node --import braintrust/hook.mjs app.js +``` + +When Braintrust LangChain/LangGraph instrumentation is enabled, LangSmith runs serialized by LangChain are ignored to avoid duplicate spans. Set `langchain: false` (and use LangSmith instrumentation) when LangSmith should be the source for those runs instead. + ## Migration Guides ### Upgrading from 2.x to 3.x diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index b4642974a..b2d778960 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -18,6 +18,7 @@ import { googleGenAIConfigs } from "./google-genai"; import { groqConfigs } from "./groq"; import { huggingFaceConfigs } from "./huggingface"; import { langchainConfigs } from "./langchain"; +import { langSmithConfigs } from "./langsmith"; import { mistralConfigs } from "./mistral"; import { openAIAgentsCoreConfigs } from "./openai-agents"; import { openaiConfigs } from "./openai"; @@ -66,6 +67,7 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] integrations: ["langchain", "langgraph"], configs: langchainConfigs, }, + { integrations: ["langsmith"], configs: langSmithConfigs }, { integrations: ["openrouter"], configs: openRouterConfigs }, { integrations: ["openrouterAgent"], diff --git a/js/src/auto-instrumentations/configs/langsmith.ts b/js/src/auto-instrumentations/configs/langsmith.ts new file mode 100644 index 000000000..5ecc54f87 --- /dev/null +++ b/js/src/auto-instrumentations/configs/langsmith.ts @@ -0,0 +1,36 @@ +import type { InstrumentationConfig } from "../orchestrion-js"; +import { langSmithChannels } from "../../instrumentation/plugins/langsmith-channels"; + +const versionRange = ">=0.3.30 <1.0.0"; + +export const langSmithConfigs: InstrumentationConfig[] = [ + ...["dist/client.js", "dist/client.cjs"].flatMap((filePath) => [ + { + channelName: langSmithChannels.createRun.channelName, + module: { name: "langsmith", versionRange, filePath }, + functionQuery: { + className: "Client", + methodName: "createRun", + kind: "Async" as const, + }, + }, + { + channelName: langSmithChannels.updateRun.channelName, + module: { name: "langsmith", versionRange, filePath }, + functionQuery: { + className: "Client", + methodName: "updateRun", + kind: "Async" as const, + }, + }, + { + channelName: langSmithChannels.batchIngestRuns.channelName, + module: { name: "langsmith", versionRange, filePath }, + functionQuery: { + className: "Client", + methodName: "batchIngestRuns", + kind: "Async" as const, + }, + }, + ]), +]; diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 26a97fa0f..ea0533ac6 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -47,6 +47,7 @@ export { groqConfigs } from "./configs/groq"; export { genkitConfigs } from "./configs/genkit"; export { gitHubCopilotConfigs } from "./configs/github-copilot"; export { langchainConfigs } from "./configs/langchain"; +export { langSmithConfigs } from "./configs/langsmith"; export { piCodingAgentConfigs } from "./configs/pi-coding-agent"; // Re-export orchestrion configuration types from the internal fork. diff --git a/js/src/exports.ts b/js/src/exports.ts index 5e67fc1ba..a1f1035b7 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -218,6 +218,11 @@ export { wrapCohere } from "./wrappers/cohere"; export { wrapGroq } from "./wrappers/groq"; export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; export { wrapCopilotClient } from "./wrappers/github-copilot"; +export { + wrapLangSmithClient, + wrapLangSmithRunTrees, + wrapLangSmithTraceable, +} from "./wrappers/langsmith"; export { wrapVitest } from "./wrappers/vitest"; export { initNodeTestSuite } from "./wrappers/node-test"; export { diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 067c1f85f..fdce9dc99 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -15,6 +15,7 @@ import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; import { LangChainPlugin } from "./plugins/langchain-plugin"; +import { LangSmithPlugin } from "./plugins/langsmith-plugin"; import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; @@ -98,6 +99,10 @@ vi.mock("./plugins/langchain-plugin", () => ({ LangChainPlugin: createPluginClassMock(), })); +vi.mock("./plugins/langsmith-plugin", () => ({ + LangSmithPlugin: createPluginClassMock(), +})); + vi.mock("./plugins/pi-coding-agent-plugin", () => ({ PiCodingAgentPlugin: createPluginClassMock(), })); @@ -552,6 +557,33 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).toHaveBeenCalledTimes(1); }); + it("should create LangSmith with LangChain deduplication by default", () => { + const plugin = new BraintrustPlugin(); + plugin.enable(); + + expect(LangSmithPlugin).toHaveBeenCalledWith({ + skipLangChainRuns: true, + }); + const mockInstance = vi.mocked(LangSmithPlugin).mock.results[0].value; + expect(mockInstance.enable).toHaveBeenCalledTimes(1); + }); + + it("should disable LangSmith or let it capture LangChain runs", () => { + const disabled = new BraintrustPlugin({ + integrations: { langsmith: false }, + }); + disabled.enable(); + expect(LangSmithPlugin).not.toHaveBeenCalled(); + + const withoutLangChain = new BraintrustPlugin({ + integrations: { langchain: false }, + }); + withoutLangChain.enable(); + expect(LangSmithPlugin).toHaveBeenCalledWith({ + skipLangChainRuns: false, + }); + }); + it("should not create OpenRouter Agent plugin when openrouterAgent: false", () => { const plugin = new BraintrustPlugin({ integrations: { openrouterAgent: false }, @@ -580,6 +612,7 @@ describe("BraintrustPlugin", () => { groq: false, gitHubCopilot: false, langchain: false, + langsmith: false, piCodingAgent: false, strandsAgentSDK: false, }, @@ -601,6 +634,7 @@ describe("BraintrustPlugin", () => { expect(GroqPlugin).not.toHaveBeenCalled(); expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); expect(LangChainPlugin).not.toHaveBeenCalled(); + expect(LangSmithPlugin).not.toHaveBeenCalled(); expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); }); @@ -698,6 +732,9 @@ describe("BraintrustPlugin", () => { plugin.enable(); expect(LangChainPlugin).not.toHaveBeenCalled(); + expect(LangSmithPlugin).toHaveBeenCalledWith({ + skipLangChainRuns: true, + }); }); it("should not create AI SDK plugin when both aisdk and vercel are false", () => { @@ -925,6 +962,7 @@ describe("BraintrustPlugin", () => { expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(LangChainPlugin).toHaveBeenCalledTimes(1); + expect(LangSmithPlugin).toHaveBeenCalledTimes(1); }); it("should only disable plugins that were enabled", () => { diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index a1fe0e352..d5435f2b0 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -19,6 +19,7 @@ import { GenkitPlugin } from "./plugins/genkit-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; import { FluePlugin } from "./plugins/flue-plugin"; import { LangChainPlugin } from "./plugins/langchain-plugin"; +import { LangSmithPlugin } from "./plugins/langsmith-plugin"; import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; import type { InstrumentationIntegrationsConfig } from "./config"; @@ -66,6 +67,7 @@ export class BraintrustPlugin extends BasePlugin { private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null; private fluePlugin: FluePlugin | null = null; private langChainPlugin: LangChainPlugin | null = null; + private langSmithPlugin: LangSmithPlugin | null = null; private piCodingAgentPlugin: PiCodingAgentPlugin | null = null; private strandsAgentSDKPlugin: StrandsAgentSDKPlugin | null = null; @@ -200,6 +202,13 @@ export class BraintrustPlugin extends BasePlugin { this.langChainPlugin.enable(); } + if (integrations.langsmith !== false) { + this.langSmithPlugin = new LangSmithPlugin({ + skipLangChainRuns: integrations.langchain !== false, + }); + this.langSmithPlugin.enable(); + } + // Mastra is intentionally not wired here: `@mastra/core` ships its own // ObservabilityExporter contract, and `BraintrustObservabilityExporter` // (wrappers/mastra.ts) is auto-installed by the loader patch in @@ -317,6 +326,11 @@ export class BraintrustPlugin extends BasePlugin { this.langChainPlugin.disable(); this.langChainPlugin = null; } + + if (this.langSmithPlugin) { + this.langSmithPlugin.disable(); + this.langSmithPlugin = null; + } } } diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 31d9c6f93..8a2d43373 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -28,6 +28,7 @@ export interface InstrumentationIntegrationsConfig { strandsAgentSDK?: boolean; langchain?: boolean; langgraph?: boolean; + langsmith?: boolean; } export interface InstrumentationConfig { @@ -102,6 +103,7 @@ const envIntegrationAliases: Record< "langchain-js": "langchain", "@langchain": "langchain", langgraph: "langgraph", + langsmith: "langsmith", }; export function getDefaultInstrumentationIntegrations(): Record< @@ -136,6 +138,7 @@ export function getDefaultInstrumentationIntegrations(): Record< gitHubCopilot: true, langchain: true, langgraph: true, + langsmith: true, piCodingAgent: true, strandsAgentSDK: true, }; diff --git a/js/src/instrumentation/plugins/langsmith-channels.ts b/js/src/instrumentation/plugins/langsmith-channels.ts new file mode 100644 index 000000000..b066bf58b --- /dev/null +++ b/js/src/instrumentation/plugins/langsmith-channels.ts @@ -0,0 +1,30 @@ +import { channel, defineChannels } from "../core/channel-definitions"; +import type { + LangSmithBatchIngestRuns, + LangSmithClient, + LangSmithRun, +} from "../../vendor-sdk-types/langsmith"; + +export const langSmithChannels = defineChannels("langsmith", { + createRun: channel< + [run: LangSmithRun, options?: unknown], + Awaited>> + >({ + channelName: "Client.createRun", + kind: "async", + }), + updateRun: channel< + [runId: string, run: LangSmithRun, options?: unknown], + Awaited>> + >({ + channelName: "Client.updateRun", + kind: "async", + }), + batchIngestRuns: channel< + [runs: LangSmithBatchIngestRuns, options?: unknown], + Awaited>> + >({ + channelName: "Client.batchIngestRuns", + kind: "async", + }), +}); diff --git a/js/src/instrumentation/plugins/langsmith-plugin.test.ts b/js/src/instrumentation/plugins/langsmith-plugin.test.ts new file mode 100644 index 000000000..0d4808f16 --- /dev/null +++ b/js/src/instrumentation/plugins/langsmith-plugin.test.ts @@ -0,0 +1,249 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { configureNode } from "../../node/config"; +import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { langSmithChannels } from "./langsmith-channels"; +import { LangSmithPlugin } from "./langsmith-plugin"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +describe("LangSmithPlugin", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + let plugin: LangSmithPlugin; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "langsmith-plugin.test.ts", + projectId: "test-project-id", + }); + plugin = new LangSmithPlugin(); + plugin.enable(); + }); + + afterEach(() => { + plugin.disable(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("mirrors run lifecycle data with LangSmith IDs, timestamps, and metrics", async () => { + const rootId = "11111111-1111-4111-8111-111111111111"; + const childId = "22222222-2222-4222-8222-222222222222"; + const start = new Date(Date.now() - 1000); + const firstToken = new Date(start.getTime() + 250); + const end = new Date(start.getTime() + 500); + + await langSmithChannels.batchIngestRuns.tracePromise( + async () => undefined, + { + arguments: [ + { + runCreates: [ + { + id: childId, + trace_id: rootId, + parent_run_id: rootId, + dotted_order: `20260713T000000000000Z${rootId}.20260713T000000000001Z${childId}`, + name: "answer", + run_type: "llm", + start_time: start.toISOString(), + inputs: { messages: ["hello"] }, + extra: { + metadata: { + customer: "acme", + ls_provider: "openai", + ls_model_name: "gpt-test", + ls_temperature: 0.2, + usage_metadata: { ignored: true }, + }, + runtime: { hidden: true }, + }, + tags: ["unit", 1], + serialized: { secret: true }, + events: [{ name: "new_token", time: firstToken.toISOString() }], + }, + { + id: rootId, + trace_id: rootId, + name: "workflow", + run_type: "chain", + start_time: start.toISOString(), + inputs: { question: "hello" }, + }, + ], + runUpdates: [ + { + id: childId, + trace_id: rootId, + parent_run_id: rootId, + end_time: end.toISOString(), + outputs: { + generations: ["world"], + usage_metadata: { + input_tokens: 3, + output_tokens: 2, + total_tokens: 5, + input_token_details: { + cache_read: 1, + cache_creation: 2, + }, + }, + }, + }, + { + id: rootId, + trace_id: rootId, + end_time: end.toISOString(), + outputs: { answer: "world" }, + }, + ], + }, + ], + }, + ); + + const spans = (await backgroundLogger.drain()) as any[]; + const root = spans.find((span) => span.span_id === rootId); + const child = spans.find((span) => span.span_id === childId); + + expect(root).toMatchObject({ + id: rootId, + span_id: rootId, + root_span_id: rootId, + span_parents: [], + input: { question: "hello" }, + output: { answer: "world" }, + span_attributes: { name: "workflow", type: "task" }, + }); + expect(root?.metrics).toMatchObject({ + end: end.getTime() / 1000, + start: start.getTime() / 1000, + }); + expect(child).toMatchObject({ + id: childId, + span_id: childId, + root_span_id: rootId, + span_parents: [rootId], + input: { messages: ["hello"] }, + output: expect.objectContaining({ generations: ["world"] }), + tags: ["unit"], + metadata: { + customer: "acme", + provider: "openai", + model: "gpt-test", + temperature: 0.2, + }, + metrics: { + prompt_tokens: 3, + completion_tokens: 2, + tokens: 5, + prompt_cached_tokens: 1, + prompt_cache_creation_tokens: 2, + time_to_first_token: 0.25, + }, + span_attributes: { name: "answer", type: "llm" }, + }); + expect(child).not.toHaveProperty("serialized"); + expect(child.metadata).not.toHaveProperty("usage_metadata"); + expect(child.metadata).not.toHaveProperty("runtime"); + }); + + it("deduplicates mixed lifecycle sources and tolerates update-only runs", async () => { + const id = "33333333-3333-4333-8333-333333333333"; + const update = { + id, + trace_id: id, + name: "update only", + run_type: "retriever", + outputs: { documents: [] }, + end_time: new Date().toISOString(), + }; + + await langSmithChannels.updateRun.tracePromise(async () => undefined, { + arguments: [id, update], + }); + await langSmithChannels.createRun.tracePromise(async () => undefined, { + arguments: [update], + }); + await langSmithChannels.batchIngestRuns.tracePromise( + async () => undefined, + { arguments: [{ runUpdates: [update] }] }, + ); + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans.filter((span) => span.span_id === id)).toHaveLength(1); + expect(spans.find((span) => span.span_id === id)).toMatchObject({ + output: { documents: [] }, + span_attributes: { name: "update only", type: "tool" }, + }); + }); + + it("contains malformed and prototype-sensitive payloads", async () => { + let getterCalled = false; + const payload = Object.create({ id: "inherited-id", name: "inherited" }); + Object.defineProperty(payload, "outputs", { + get() { + getterCalled = true; + throw new Error("must not execute"); + }, + }); + + await expect( + langSmithChannels.createRun.tracePromise(async () => undefined, { + arguments: [payload], + }), + ).resolves.toBeUndefined(); + expect(getterCalled).toBe(false); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("suppresses LangChain-serialized runs only when configured", async () => { + const run = { + id: "44444444-4444-4444-8444-444444444444", + trace_id: "44444444-4444-4444-8444-444444444444", + name: "langchain runnable", + serialized: { lc: 1 }, + end_time: new Date().toISOString(), + }; + + await langSmithChannels.updateRun.tracePromise(async () => undefined, { + arguments: [run.id, run], + }); + expect(await backgroundLogger.drain()).toEqual([]); + + plugin.disable(); + plugin = new LangSmithPlugin({ skipLangChainRuns: false }); + plugin.enable(); + await langSmithChannels.updateRun.tracePromise(async () => undefined, { + arguments: [run.id, run], + }); + expect(await backgroundLogger.drain()).toHaveLength(1); + }); + + it("bounds the completed-run deduplication cache", () => { + const completedRuns = ( + plugin as unknown as { + completedRuns: { + get(key: string): true | undefined; + set(key: string, value: true): void; + }; + } + ).completedRuns; + + for (let index = 0; index <= 10_000; index++) { + completedRuns.set(`run-${index}`, true); + } + + expect(completedRuns.get("run-0")).toBeUndefined(); + expect(completedRuns.get("run-10000")).toBe(true); + }); +}); diff --git a/js/src/instrumentation/plugins/langsmith-plugin.ts b/js/src/instrumentation/plugins/langsmith-plugin.ts new file mode 100644 index 000000000..d6958e9ab --- /dev/null +++ b/js/src/instrumentation/plugins/langsmith-plugin.ts @@ -0,0 +1,557 @@ +import { SpanTypeAttribute } from "../../../util/index"; +import { debugLogger } from "../../debug-logger"; +import { startSpan } from "../../logger"; +import type { Span } from "../../logger"; +import { LRUCache } from "../../prompt-cache/lru-cache"; +import type { + LangSmithBatchIngestRuns, + LangSmithRun, +} from "../../vendor-sdk-types/langsmith"; +import { BasePlugin } from "../core"; +import { unsubscribeAll } from "../core/channel-tracing"; +import type { ChannelMessage } from "../core/channel-definitions"; +import { langSmithChannels } from "./langsmith-channels"; + +type ActiveRun = { + run: LangSmithRun; + span: Span; +}; + +type LangSmithPluginOptions = { + skipLangChainRuns?: boolean; +}; + +const MAX_COMPLETED_RUNS = 10_000; +const BLOCKED_METADATA_KEYS = new Set([ + "__proto__", + "constructor", + "prototype", + "usage_metadata", +]); +const BLOCKED_METADATA_PREFIXES = ["__pregel_", "langgraph_", "lc_"]; +const LLM_SETTING_KEYS = [ + "temperature", + "top_p", + "max_tokens", + "frequency_penalty", + "presence_penalty", + "stop", + "response_format", +] as const; + +export class LangSmithPlugin extends BasePlugin { + private readonly activeRuns = new Map(); + private readonly completedRuns = new LRUCache({ + max: MAX_COMPLETED_RUNS, + }); + private readonly skipLangChainRuns: boolean; + + constructor(options: LangSmithPluginOptions = {}) { + super(); + this.skipLangChainRuns = options.skipLangChainRuns ?? true; + } + + protected onEnable(): void { + const createChannel = langSmithChannels.createRun.tracingChannel(); + const createHandlers = { + start: ( + event: ChannelMessage, + ): void => { + this.containLifecycleFailure("createRun", () => { + this.processCreate(event.arguments[0]); + }); + }, + }; + createChannel.subscribe(createHandlers); + this.unsubscribers.push(() => createChannel.unsubscribe(createHandlers)); + + const updateChannel = langSmithChannels.updateRun.tracingChannel(); + const updateHandlers = { + start: ( + event: ChannelMessage, + ): void => { + this.containLifecycleFailure("updateRun", () => { + this.processUpdate(event.arguments[0], event.arguments[1]); + }); + }, + }; + updateChannel.subscribe(updateHandlers); + this.unsubscribers.push(() => updateChannel.unsubscribe(updateHandlers)); + + const batchChannel = langSmithChannels.batchIngestRuns.tracingChannel(); + const batchHandlers = { + start: ( + event: ChannelMessage, + ): void => { + this.containLifecycleFailure("batchIngestRuns", () => { + this.processBatch(event.arguments[0]); + }); + }, + }; + batchChannel.subscribe(batchHandlers); + this.unsubscribers.push(() => batchChannel.unsubscribe(batchHandlers)); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + for (const { span } of this.activeRuns.values()) { + span.end(); + } + this.activeRuns.clear(); + this.completedRuns.clear(); + } + + private processBatch(batch: LangSmithBatchIngestRuns): void { + if (!isRecord(batch)) { + return; + } + + const creates = ownValue(batch, "runCreates"); + if (Array.isArray(creates)) { + const parentFirst = [...creates].sort((left, right) => { + const leftId = stringValue(ownValue(left, "id")); + const rightId = stringValue(ownValue(right, "id")); + const leftParent = stringValue(ownValue(left, "parent_run_id")); + const rightParent = stringValue(ownValue(right, "parent_run_id")); + if (leftParent && leftParent === rightId) { + return 1; + } + if (rightParent && rightParent === leftId) { + return -1; + } + return dottedOrderDepth(left) - dottedOrderDepth(right); + }); + for (const run of parentFirst) { + this.containLifecycleFailure("batchIngestRuns create", () => { + this.processCreate(run); + }); + } + } + + const updates = ownValue(batch, "runUpdates"); + if (Array.isArray(updates)) { + for (const run of updates) { + this.containLifecycleFailure("batchIngestRuns update", () => { + this.processUpdate(stringValue(ownValue(run, "id")) ?? "", run); + }); + } + } + } + + private processCreate(run: LangSmithRun): void { + const id = stringValue(ownValue(run, "id")); + if (!id || this.completedRuns.get(id)) { + return; + } + if (this.shouldSkipLangChainRun(run)) { + this.completedRuns.set(id, true); + return; + } + + const active = this.activeRuns.get(id); + if (active) { + const previous = active.run; + active.run = mergeRuns(previous, run); + this.logRun(active.span, active.run, previous, false); + this.endIfComplete(id, active, active.run); + return; + } + + const span = this.startRunSpan(id, run); + const activeRun = { run, span }; + this.activeRuns.set(id, activeRun); + this.logRun(span, run, undefined, false); + this.endIfComplete(id, activeRun, run); + } + + private processUpdate(explicitId: string, run: LangSmithRun): void { + const id = stringValue(explicitId) ?? stringValue(ownValue(run, "id")); + if (!id || this.completedRuns.get(id)) { + return; + } + if (this.shouldSkipLangChainRun(run)) { + const active = this.activeRuns.get(id); + active?.span.end(); + this.activeRuns.delete(id); + this.completedRuns.set(id, true); + return; + } + + let active = this.activeRuns.get(id); + if (!active) { + const span = this.startRunSpan(id, run); + active = { run, span }; + this.activeRuns.set(id, active); + } + + const previous = active.run; + active.run = mergeRuns(previous, run); + this.logRun(active.span, active.run, previous, true); + this.endIfComplete(id, active, active.run); + } + + private startRunSpan(id: string, run: LangSmithRun): Span { + const traceId = stringValue(ownValue(run, "trace_id")) ?? id; + const parentId = + stringValue(ownValue(run, "parent_run_id")) ?? + stringValue(ownValue(ownValue(run, "parent_run"), "id")); + const startTime = timestampSeconds(ownValue(run, "start_time")); + + return startSpan({ + name: stringValue(ownValue(run, "name")) ?? "LangSmith run", + spanId: id, + parentSpanIds: { + parentSpanIds: parentId ? [parentId] : [], + rootSpanId: traceId, + }, + spanAttributes: { + type: mapRunType(ownValue(run, "run_type")), + }, + ...(startTime === undefined ? {} : { startTime }), + event: { id }, + }); + } + + private logRun( + span: Span, + run: LangSmithRun, + previous: LangSmithRun | undefined, + includeOutput: boolean, + ): void { + const inputs = preferOwnValue(run, previous, "inputs"); + const outputs = preferOwnValue(run, previous, "outputs"); + const error = errorMessage(preferOwnValue(run, previous, "error")); + const metadata = extractMetadata(run, previous); + const tags = extractTags(preferOwnValue(run, previous, "tags")); + const metrics = extractMetrics(run, previous); + + span.log({ + ...(inputs === undefined ? {} : { input: sanitizeLoggedValue(inputs) }), + ...(includeOutput && outputs !== undefined + ? { output: sanitizeLoggedValue(outputs) } + : {}), + ...(error === undefined ? {} : { error }), + ...(metadata === undefined ? {} : { metadata }), + ...(tags === undefined ? {} : { tags }), + ...(Object.keys(metrics).length === 0 ? {} : { metrics }), + }); + } + + private endIfComplete( + id: string, + active: ActiveRun, + run: LangSmithRun, + ): void { + const endTime = timestampSeconds(ownValue(run, "end_time")); + if ( + endTime === undefined && + errorMessage(ownValue(run, "error")) === undefined + ) { + return; + } + active.span.end(endTime === undefined ? undefined : { endTime }); + this.activeRuns.delete(id); + this.completedRuns.set(id, true); + } + + private shouldSkipLangChainRun(run: LangSmithRun): boolean { + if (!this.skipLangChainRuns) { + return false; + } + const serialized = ownValue(run, "serialized"); + return isRecord(serialized) && ownValue(serialized, "lc") === 1; + } + + private containLifecycleFailure(operation: string, fn: () => void): void { + try { + fn(); + } catch (error) { + debugLogger.error( + `Failed to process LangSmith ${operation} instrumentation:`, + error, + ); + } + } +} + +function ownValue(value: unknown, key: PropertyKey): unknown { + if (!isRecord(value)) { + return undefined; + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + return descriptor && "value" in descriptor ? descriptor.value : undefined; +} + +function preferOwnValue( + current: unknown, + previous: unknown, + key: PropertyKey, +): unknown { + const currentDescriptor = isRecord(current) + ? Object.getOwnPropertyDescriptor(current, key) + : undefined; + if (currentDescriptor && "value" in currentDescriptor) { + return currentDescriptor.value; + } + return ownValue(previous, key); +} + +function mergeRuns( + previous: LangSmithRun, + current: LangSmithRun, +): LangSmithRun { + const entries = new Map(); + for (const value of [previous, current]) { + if (!isRecord(value)) { + continue; + } + for (const [key, descriptor] of Object.entries( + Object.getOwnPropertyDescriptors(value), + )) { + if (descriptor.enumerable && "value" in descriptor) { + entries.set(key, descriptor.value); + } + } + } + return Object.fromEntries(entries); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function timestampSeconds(value: unknown): number | undefined { + if (value instanceof Date) { + const timestamp = value.getTime(); + return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; + } + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + return value > 10_000_000_000 ? value / 1000 : value; + } + if (typeof value === "string") { + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; + } + return undefined; +} + +function dottedOrderDepth(run: unknown): number { + const dottedOrder = stringValue(ownValue(run, "dotted_order")); + return dottedOrder ? dottedOrder.split(".").length : Number.MAX_SAFE_INTEGER; +} + +function mapRunType(runType: unknown): SpanTypeAttribute { + switch (runType) { + case "llm": + case "embedding": + return SpanTypeAttribute.LLM; + case "tool": + case "retriever": + return SpanTypeAttribute.TOOL; + default: + return SpanTypeAttribute.TASK; + } +} + +function extractMetadata( + run: LangSmithRun, + previous: LangSmithRun | undefined, +): Record | undefined { + const extra = preferOwnValue(run, previous, "extra"); + const rawMetadata = ownValue(extra, "metadata"); + if (!isRecord(rawMetadata)) { + return undefined; + } + + const metadata: Record = {}; + for (const [key, descriptor] of Object.entries( + Object.getOwnPropertyDescriptors(rawMetadata), + )) { + if ( + !("value" in descriptor) || + !descriptor.enumerable || + BLOCKED_METADATA_KEYS.has(key) || + BLOCKED_METADATA_PREFIXES.some((prefix) => key.startsWith(prefix)) || + (key.startsWith("ls_") && + key !== "ls_provider" && + key !== "ls_model_name" && + !LLM_SETTING_KEYS.some((setting) => key === `ls_${setting}`)) + ) { + continue; + } + + const normalizedKey = + key === "ls_provider" + ? "provider" + : key === "ls_model_name" + ? "model" + : key.startsWith("ls_") + ? key.slice(3) + : key; + const sanitized = sanitizeLoggedValue(descriptor.value); + if (sanitized !== undefined) { + metadata[normalizedKey] = sanitized; + } + } + + return Object.keys(metadata).length === 0 ? undefined : metadata; +} + +function extractTags(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const tags = value.filter( + (tag): tag is string => typeof tag === "string" && tag.length > 0, + ); + return tags.length > 0 ? tags : undefined; +} + +function extractMetrics( + run: LangSmithRun, + previous: LangSmithRun | undefined, +): Record { + const outputs = preferOwnValue(run, previous, "outputs"); + const extra = preferOwnValue(run, previous, "extra"); + const metadata = ownValue(extra, "metadata"); + const usage = + ownValue(outputs, "usage_metadata") ?? ownValue(metadata, "usage_metadata"); + const metrics: Record = {}; + + assignFirstMetric(metrics, "prompt_tokens", usage, [ + "input_tokens", + "prompt_tokens", + ]); + assignFirstMetric(metrics, "completion_tokens", usage, [ + "output_tokens", + "completion_tokens", + ]); + assignFirstMetric(metrics, "tokens", usage, ["total_tokens", "tokens"]); + + const inputTokenDetails = ownValue(usage, "input_token_details"); + assignFirstMetric(metrics, "prompt_cached_tokens", inputTokenDetails, [ + "cache_read", + "cached_tokens", + ]); + if (metrics.prompt_cached_tokens === undefined) { + assignFirstMetric(metrics, "prompt_cached_tokens", usage, [ + "cache_read_input_tokens", + "prompt_cached_tokens", + ]); + } + assignFirstMetric( + metrics, + "prompt_cache_creation_tokens", + inputTokenDetails, + ["cache_creation"], + ); + if (metrics.prompt_cache_creation_tokens === undefined) { + assignFirstMetric(metrics, "prompt_cache_creation_tokens", usage, [ + "cache_creation_input_tokens", + "prompt_cache_creation_tokens", + ]); + } + + if ( + metrics.tokens === undefined && + (metrics.prompt_tokens !== undefined || + metrics.completion_tokens !== undefined) + ) { + metrics.tokens = + (metrics.prompt_tokens ?? 0) + (metrics.completion_tokens ?? 0); + } + + const startTime = timestampSeconds( + preferOwnValue(run, previous, "start_time"), + ); + const events = preferOwnValue(run, previous, "events"); + if (startTime !== undefined && Array.isArray(events)) { + for (const event of events) { + if (ownValue(event, "name") !== "new_token") { + continue; + } + const eventTime = timestampSeconds(ownValue(event, "time")); + if (eventTime !== undefined && eventTime >= startTime) { + metrics.time_to_first_token = eventTime - startTime; + } + break; + } + } + + return metrics; +} + +function assignFirstMetric( + metrics: Record, + target: string, + source: unknown, + keys: string[], +): void { + for (const key of keys) { + const value = ownValue(source, key); + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + metrics[target] = value; + return; + } + } +} + +function errorMessage(value: unknown): string | undefined { + if (typeof value === "string") { + return value; + } + if (value instanceof Error) { + return value.message; + } + return undefined; +} + +function sanitizeLoggedValue( + value: unknown, + seen = new WeakSet(), + depth = 0, +): unknown { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + return Number.isFinite(value) ? value : undefined; + } + if (value instanceof Date) { + return value.toISOString(); + } + if (typeof value !== "object" || depth >= 20) { + return undefined; + } + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + + if (Array.isArray(value)) { + return value.map((item) => sanitizeLoggedValue(item, seen, depth + 1)); + } + + const entries: Array<[string, unknown]> = []; + for (const [key, descriptor] of Object.entries( + Object.getOwnPropertyDescriptors(value), + )) { + if ( + !descriptor.enumerable || + !("value" in descriptor) || + BLOCKED_METADATA_KEYS.has(key) + ) { + continue; + } + entries.push([key, sanitizeLoggedValue(descriptor.value, seen, depth + 1)]); + } + return Object.fromEntries(entries); +} diff --git a/js/src/vendor-sdk-types/langsmith.ts b/js/src/vendor-sdk-types/langsmith.ts new file mode 100644 index 000000000..4c7807657 --- /dev/null +++ b/js/src/vendor-sdk-types/langsmith.ts @@ -0,0 +1,77 @@ +export type LangSmithRunEvent = { + name?: unknown; + time?: unknown; +}; + +export type LangSmithRun = { + id?: unknown; + name?: unknown; + run_type?: unknown; + start_time?: unknown; + end_time?: unknown; + inputs?: unknown; + outputs?: unknown; + error?: unknown; + extra?: unknown; + tags?: unknown; + events?: unknown; + serialized?: unknown; + parent_run_id?: unknown; + parent_run?: LangSmithRun; + trace_id?: unknown; + dotted_order?: unknown; +}; + +export type LangSmithBatchIngestRuns = { + runCreates?: LangSmithRun[]; + runUpdates?: LangSmithRun[]; +}; + +export type LangSmithClient = { + createRun?: (run: LangSmithRun, options?: unknown) => Promise; + updateRun?: ( + runId: string, + run: LangSmithRun, + options?: unknown, + ) => Promise; + batchIngestRuns?: ( + runs: LangSmithBatchIngestRuns, + options?: unknown, + ) => Promise; +}; + +export type LangSmithClientConstructor = new ( + ...args: unknown[] +) => LangSmithClient; + +export type LangSmithClientModule = { + Client: LangSmithClientConstructor; +}; + +export type LangSmithRunTree = LangSmithRun & { + createChild?: (config: unknown) => LangSmithRunTree; + postRun?: (excludeChildRuns?: boolean) => Promise; + patchRun?: (options?: unknown) => Promise; +}; + +export type LangSmithRunTreeConstructor = new ( + ...args: unknown[] +) => LangSmithRunTree; + +export type LangSmithRunTreesModule = { + RunTree: LangSmithRunTreeConstructor; +}; + +export type LangSmithTraceableConfig = { + on_end?: (runTree?: LangSmithRunTree) => void; + [key: string]: unknown; +}; + +export type LangSmithTraceable = ( + fn: (...args: never[]) => unknown, + config?: LangSmithTraceableConfig, +) => (...args: never[]) => unknown; + +export type LangSmithTraceableModule = { + traceable: LangSmithTraceable; +}; diff --git a/js/src/wrappers/langsmith.test.ts b/js/src/wrappers/langsmith.test.ts new file mode 100644 index 000000000..4907cf6b2 --- /dev/null +++ b/js/src/wrappers/langsmith.test.ts @@ -0,0 +1,172 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const { tracePromise } = vi.hoisted(() => ({ + tracePromise: vi.fn((fn: () => Promise, _event?: unknown) => fn()), +})); + +vi.mock("../isomorph", () => ({ + default: { + newTracingChannel: vi.fn(() => ({ + subscribe: vi.fn(), + tracePromise, + unsubscribe: vi.fn(), + })), + }, +})); + +import { + wrapLangSmithClient, + wrapLangSmithRunTrees, + wrapLangSmithTraceable, +} from "./langsmith"; + +describe("LangSmith namespace wrappers", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("composes traceable on_end and preserves unrelated exports", async () => { + const existingOnEnd = vi.fn(); + const run = { + id: "run-1", + name: "traceable run", + run_type: "chain", + inputs: { input: "hello" }, + outputs: { output: "world" }, + start_time: 1, + end_time: 2, + trace_id: "run-1", + }; + const traceable = vi.fn( + (fn: (...args: unknown[]) => unknown, config?: Record) => + async (...args: unknown[]) => { + const result = await fn(...args); + (config?.on_end as ((run: unknown) => void) | undefined)?.(run); + return result; + }, + ); + const helper = () => "helper"; + const namespace = { helper, traceable }; + + const wrapped = wrapLangSmithTraceable(namespace); + const traced = wrapped.traceable(async (value: string) => `${value}!`, { + on_end: existingOnEnd, + }); + + await expect(traced("hello")).resolves.toBe("hello!"); + expect(existingOnEnd).toHaveBeenCalledWith(run); + expect(wrapped.helper).toBe(helper); + expect(tracePromise).toHaveBeenCalledTimes(1); + expect(tracePromise.mock.calls[0]?.[1]).toMatchObject({ + arguments: ["run-1", run], + }); + }); + + it("recursively wraps RunTree children without changing class identity", async () => { + class RunTree { + #secret = "run-tree-secret"; + + id = "run-tree"; + name = "run tree"; + run_type = "chain"; + inputs = {}; + start_time = 1; + trace_id = this.id; + + createChild() { + return new RunTree(); + } + + postRun() { + return Promise.resolve(this.#secret); + } + + patchRun() { + return Promise.resolve(this.#secret); + } + + readSecret() { + return this.#secret; + } + } + + const wrapped = wrapLangSmithRunTrees({ RunTree, untouched: true }); + const tree = new wrapped.RunTree({ name: "ignored" }); + const child = tree.createChild({ name: "child" }); + + expect(tree).toBeInstanceOf(RunTree); + expect(child).toBeInstanceOf(RunTree); + expect(tree.readSecret()).toBe("run-tree-secret"); + await expect(tree.postRun()).resolves.toBe("run-tree-secret"); + await expect(child.patchRun()).resolves.toBe("run-tree-secret"); + expect(tracePromise).toHaveBeenCalledTimes(2); + }); + + it("wraps Client lifecycle methods and safely binds other methods", async () => { + class Client { + #secret = "client-secret"; + + createRun(run: unknown) { + return Promise.resolve(run); + } + + updateRun(id: string, run: unknown) { + return Promise.resolve({ id, run }); + } + + batchIngestRuns(runs: unknown) { + return Promise.resolve(runs); + } + + readSecret() { + return this.#secret; + } + } + + const wrapped = wrapLangSmithClient({ Client }); + const client = new wrapped.Client(); + + expect(client).toBeInstanceOf(Client); + expect(client.readSecret()).toBe("client-secret"); + await client.createRun({ id: "create" }); + await client.updateRun("update", { output: true }); + await client.batchIngestRuns({ runCreates: [], runUpdates: [] }); + expect(tracePromise).toHaveBeenCalledTimes(3); + }); + + it("preserves Client method errors", async () => { + const expected = new Error("client failure"); + class Client { + async createRun(): Promise { + throw expected; + } + } + + const wrapped = wrapLangSmithClient({ Client }); + await expect(new wrapped.Client().createRun({})).rejects.toBe(expected); + }); + + it("does not double-wrap namespaces", async () => { + class Client { + createRun() { + return Promise.resolve(); + } + } + + const wrapped = wrapLangSmithClient(wrapLangSmithClient({ Client })); + await new wrapped.Client().createRun({ id: "one" }); + expect(tracePromise).toHaveBeenCalledTimes(1); + }); + + it("preserves namespace keys for module-shaped objects", () => { + const namespace = Object.freeze({ + Client: class Client {}, + helper: "preserved", + }); + + const wrapped = wrapLangSmithClient(namespace); + expect(Object.keys(wrapped)).toEqual(["Client", "helper"]); + expect("helper" in wrapped).toBe(true); + expect(wrapped.helper).toBe("preserved"); + }); +}); diff --git a/js/src/wrappers/langsmith.ts b/js/src/wrappers/langsmith.ts new file mode 100644 index 000000000..cc4a189a7 --- /dev/null +++ b/js/src/wrappers/langsmith.ts @@ -0,0 +1,352 @@ +import { debugLogger } from "../debug-logger"; +import { langSmithChannels } from "../instrumentation/plugins/langsmith-channels"; +import type { + LangSmithClient, + LangSmithClientConstructor, + LangSmithClientModule, + LangSmithRunTree, + LangSmithRunTreeConstructor, + LangSmithRunTreesModule, + LangSmithTraceable, + LangSmithTraceableConfig, + LangSmithTraceableModule, +} from "../vendor-sdk-types/langsmith"; + +const WRAPPED_CLIENT_CLASS = Symbol.for( + "braintrust.langsmith.wrapped-client-class", +); +const WRAPPED_CLIENT_INSTANCE = Symbol.for( + "braintrust.langsmith.wrapped-client-instance", +); +const WRAPPED_CLIENT_NAMESPACE = Symbol.for( + "braintrust.langsmith.wrapped-client-namespace", +); +const WRAPPED_RUN_TREE_CLASS = Symbol.for( + "braintrust.langsmith.wrapped-run-tree-class", +); +const WRAPPED_RUN_TREE_INSTANCE = Symbol.for( + "braintrust.langsmith.wrapped-run-tree-instance", +); +const WRAPPED_RUN_TREES_NAMESPACE = Symbol.for( + "braintrust.langsmith.wrapped-run-trees-namespace", +); +const WRAPPED_TRACEABLE = Symbol.for("braintrust.langsmith.wrapped-traceable"); +const WRAPPED_TRACEABLE_NAMESPACE = Symbol.for( + "braintrust.langsmith.wrapped-traceable-namespace", +); + +export function wrapLangSmithTraceable(namespace: T): T { + return wrapNamespaceExport( + namespace, + "traceable", + WRAPPED_TRACEABLE_NAMESPACE, + (value) => wrapTraceable(value as LangSmithTraceable), + ) as T & LangSmithTraceableModule; +} + +export function wrapLangSmithRunTrees(namespace: T): T { + return wrapNamespaceExport( + namespace, + "RunTree", + WRAPPED_RUN_TREES_NAMESPACE, + (value) => wrapRunTreeClass(value as LangSmithRunTreeConstructor), + ) as T & LangSmithRunTreesModule; +} + +export function wrapLangSmithClient(namespace: T): T { + return wrapNamespaceExport( + namespace, + "Client", + WRAPPED_CLIENT_NAMESPACE, + (value) => wrapClientClass(value as LangSmithClientConstructor), + ) as T & LangSmithClientModule; +} + +function wrapNamespaceExport( + namespace: T, + exportName: string, + marker: symbol, + wrap: (value: unknown) => unknown, +): T { + if (!namespace || typeof namespace !== "object") { + return namespace; + } + + const candidate = namespace as Record; + if (candidate[marker] === true) { + return namespace; + } + if (typeof candidate[exportName] !== "function") { + // eslint-disable-next-line no-restricted-properties -- public wrapper diagnostics follow existing wrapper behavior. + console.warn( + `Unsupported LangSmith ${exportName} namespace. Not wrapping.`, + ); + return namespace; + } + + const target = isModuleNamespace(namespace) + ? Object.setPrototypeOf({}, namespace) + : candidate; + const moduleNamespace = target !== candidate; + let wrappedExport: unknown; + + return new Proxy(target, { + get(target, prop, receiver) { + if (prop === marker) { + return true; + } + const value = Reflect.get(target, prop, receiver); + if (prop !== exportName || typeof value !== "function") { + return value; + } + wrappedExport ??= wrap(value); + return wrappedExport; + }, + getOwnPropertyDescriptor(target, prop) { + const descriptor = Reflect.getOwnPropertyDescriptor(target, prop); + if (descriptor || !moduleNamespace) { + return descriptor; + } + const namespaceDescriptor = Reflect.getOwnPropertyDescriptor( + candidate, + prop, + ); + return namespaceDescriptor + ? { ...namespaceDescriptor, configurable: true } + : undefined; + }, + has(target, prop) { + return ( + Reflect.has(target, prop) || (moduleNamespace && prop in candidate) + ); + }, + ownKeys(target) { + return moduleNamespace + ? Reflect.ownKeys(candidate) + : Reflect.ownKeys(target); + }, + }) as T; +} + +function isModuleNamespace(value: unknown): boolean { + if (!value || typeof value !== "object") { + return false; + } + if (value.constructor?.name === "Module") { + return true; + } + const firstKey = Object.keys(value)[0]; + if (!firstKey) { + return false; + } + const descriptor = Object.getOwnPropertyDescriptor(value, firstKey); + return descriptor ? !descriptor.configurable && !descriptor.writable : false; +} + +function wrapTraceable(traceable: LangSmithTraceable): LangSmithTraceable { + if ( + (traceable as unknown as Record)[WRAPPED_TRACEABLE] + ) { + return traceable; + } + + return new Proxy(traceable, { + get(target, prop, receiver) { + if (prop === WRAPPED_TRACEABLE) { + return true; + } + return Reflect.get(target, prop, receiver); + }, + apply(target, thisArg, argArray) { + const [fn, rawConfig] = argArray; + const config = + rawConfig && typeof rawConfig === "object" + ? (rawConfig as LangSmithTraceableConfig) + : undefined; + const originalOnEnd = config?.on_end; + const wrappedConfig: LangSmithTraceableConfig = { + ...config, + on_end(runTree) { + publishRunUpdate(runTree); + if (originalOnEnd) { + Reflect.apply(originalOnEnd, config, [runTree]); + } + }, + }; + return Reflect.apply(target, thisArg, [fn, wrappedConfig]); + }, + }); +} + +function wrapRunTreeClass( + RunTree: LangSmithRunTreeConstructor, +): LangSmithRunTreeConstructor { + if ( + (RunTree as unknown as Record)[WRAPPED_RUN_TREE_CLASS] + ) { + return RunTree; + } + + return new Proxy(RunTree, { + get(target, prop, receiver) { + if (prop === WRAPPED_RUN_TREE_CLASS) { + return true; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + construct(target, args, newTarget) { + return wrapRunTreeInstance(Reflect.construct(target, args, newTarget)); + }, + }); +} + +function wrapRunTreeInstance(runTree: LangSmithRunTree): LangSmithRunTree { + if ((runTree as Record)[WRAPPED_RUN_TREE_INSTANCE]) { + return runTree; + } + const cache = new Map(); + + return new Proxy(runTree, { + get(target, prop, receiver) { + if (prop === WRAPPED_RUN_TREE_INSTANCE) { + return true; + } + if (cache.has(prop)) { + return cache.get(prop); + } + + const value = Reflect.get(target, prop, receiver); + if (typeof value !== "function") { + return value; + } + + let wrapped: unknown; + if (prop === "createChild") { + wrapped = (...args: unknown[]) => + wrapRunTreeInstance(Reflect.apply(value, target, args)); + } else if (prop === "postRun") { + const method = value as (...args: unknown[]) => Promise; + wrapped = (...args: unknown[]) => + langSmithChannels.createRun.tracePromise( + () => Reflect.apply(method, target, args), + { arguments: [target] }, + ); + } else if (prop === "patchRun") { + const method = value as (...args: unknown[]) => Promise; + wrapped = (...args: unknown[]) => + langSmithChannels.updateRun.tracePromise( + () => Reflect.apply(method, target, args), + { + arguments: [ + typeof target.id === "string" ? target.id : "", + target, + ], + }, + ); + } else { + wrapped = value.bind(target); + } + cache.set(prop, wrapped); + return wrapped; + }, + }); +} + +function wrapClientClass( + Client: LangSmithClientConstructor, +): LangSmithClientConstructor { + if ( + (Client as unknown as Record)[WRAPPED_CLIENT_CLASS] + ) { + return Client; + } + + return new Proxy(Client, { + get(target, prop, receiver) { + if (prop === WRAPPED_CLIENT_CLASS) { + return true; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + construct(target, args, newTarget) { + return wrapClientInstance(Reflect.construct(target, args, newTarget)); + }, + }); +} + +function wrapClientInstance(client: LangSmithClient): LangSmithClient { + if ((client as Record)[WRAPPED_CLIENT_INSTANCE]) { + return client; + } + const cache = new Map(); + + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === WRAPPED_CLIENT_INSTANCE) { + return true; + } + if (cache.has(prop)) { + return cache.get(prop); + } + + const value = Reflect.get(target, prop, receiver); + if (typeof value !== "function") { + return value; + } + + let wrapped: unknown; + if (prop === "createRun") { + const method = value as (...args: unknown[]) => Promise; + wrapped = ( + ...args: Parameters> + ) => + langSmithChannels.createRun.tracePromise( + () => Reflect.apply(method, target, args), + { arguments: args }, + ); + } else if (prop === "updateRun") { + const method = value as (...args: unknown[]) => Promise; + wrapped = ( + ...args: Parameters> + ) => + langSmithChannels.updateRun.tracePromise( + () => Reflect.apply(method, target, args), + { arguments: args }, + ); + } else if (prop === "batchIngestRuns") { + const method = value as (...args: unknown[]) => Promise; + wrapped = ( + ...args: Parameters> + ) => + langSmithChannels.batchIngestRuns.tracePromise( + () => Reflect.apply(method, target, args), + { arguments: args }, + ); + } else { + wrapped = value.bind(target); + } + cache.set(prop, wrapped); + return wrapped; + }, + }); +} + +function publishRunUpdate(runTree: LangSmithRunTree | undefined): void { + if (!runTree || typeof runTree.id !== "string" || !runTree.id) { + return; + } + + try { + void langSmithChannels.updateRun + .tracePromise(() => Promise.resolve(undefined), { + arguments: [runTree.id, runTree], + }) + .catch((error) => { + debugLogger.error("LangSmith traceable instrumentation failed:", error); + }); + } catch (error) { + debugLogger.error("LangSmith traceable instrumentation failed:", error); + } +} From 77086b184e213ebe74713231330b9c38dd3a67f3 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 13 Jul 2026 16:12:53 +0200 Subject: [PATCH 2/3] fix --- js/src/vendor-sdk-types/langsmith.ts | 2 +- js/src/wrappers/langsmith.test.ts | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/js/src/vendor-sdk-types/langsmith.ts b/js/src/vendor-sdk-types/langsmith.ts index 4c7807657..4f2f7d7f4 100644 --- a/js/src/vendor-sdk-types/langsmith.ts +++ b/js/src/vendor-sdk-types/langsmith.ts @@ -14,7 +14,7 @@ export type LangSmithRun = { error?: unknown; extra?: unknown; tags?: unknown; - events?: unknown; + events?: LangSmithRunEvent[]; serialized?: unknown; parent_run_id?: unknown; parent_run?: LangSmithRun; diff --git a/js/src/wrappers/langsmith.test.ts b/js/src/wrappers/langsmith.test.ts index 4907cf6b2..f694f0d64 100644 --- a/js/src/wrappers/langsmith.test.ts +++ b/js/src/wrappers/langsmith.test.ts @@ -38,9 +38,12 @@ describe("LangSmith namespace wrappers", () => { trace_id: "run-1", }; const traceable = vi.fn( - (fn: (...args: unknown[]) => unknown, config?: Record) => - async (...args: unknown[]) => { - const result = await fn(...args); + ( + fn: (value: string) => Promise, + config?: Record, + ) => + async (value: string) => { + const result = await fn(value); (config?.on_end as ((run: unknown) => void) | undefined)?.(run); return result; }, @@ -73,7 +76,9 @@ describe("LangSmith namespace wrappers", () => { start_time = 1; trace_id = this.id; - createChild() { + constructor(_config?: unknown) {} + + createChild(_config?: unknown) { return new RunTree(); } @@ -137,7 +142,7 @@ describe("LangSmith namespace wrappers", () => { it("preserves Client method errors", async () => { const expected = new Error("client failure"); class Client { - async createRun(): Promise { + async createRun(_run: unknown): Promise { throw expected; } } @@ -148,7 +153,7 @@ describe("LangSmith namespace wrappers", () => { it("does not double-wrap namespaces", async () => { class Client { - createRun() { + createRun(_run: unknown) { return Promise.resolve(); } } From 8e3d35facc767c7a9532a96fbcc07b8c1704e966 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 14 Jul 2026 15:08:22 +0200 Subject: [PATCH 3/3] fixes and cleanup --- .../plugins/langsmith-plugin.test.ts | 63 +++++++++++++++++++ .../plugins/langsmith-plugin.ts | 18 +++++- js/src/vendor-sdk-types/langsmith.ts | 12 ---- js/src/wrappers/langsmith.test.ts | 16 ++++- js/src/wrappers/langsmith.ts | 21 +------ 5 files changed, 95 insertions(+), 35 deletions(-) diff --git a/js/src/instrumentation/plugins/langsmith-plugin.test.ts b/js/src/instrumentation/plugins/langsmith-plugin.test.ts index 0d4808f16..8e9b3aed4 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.test.ts +++ b/js/src/instrumentation/plugins/langsmith-plugin.test.ts @@ -187,6 +187,69 @@ describe("LangSmithPlugin", () => { }); }); + it("logs outputs from completed direct and batched creates", async () => { + const directId = "55555555-5555-4555-8555-555555555555"; + const batchId = "66666666-6666-4666-8666-666666666666"; + const endTime = new Date().toISOString(); + + await langSmithChannels.createRun.tracePromise(async () => undefined, { + arguments: [ + { + id: directId, + trace_id: directId, + name: "completed direct create", + outputs: { answer: "direct" }, + end_time: endTime, + }, + ], + }); + await langSmithChannels.batchIngestRuns.tracePromise( + async () => undefined, + { + arguments: [ + { + runCreates: [ + { + id: batchId, + trace_id: batchId, + name: "completed batch create", + outputs: { answer: "batch" }, + end_time: endTime, + }, + ], + }, + ], + }, + ); + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans.find((span) => span.span_id === directId)).toMatchObject({ + output: { answer: "direct" }, + }); + expect(spans.find((span) => span.span_id === batchId)).toMatchObject({ + output: { answer: "batch" }, + }); + }); + + it("completes runs containing invalid nested dates", async () => { + const id = "77777777-7777-4777-8777-777777777777"; + + await langSmithChannels.createRun.tracePromise(async () => undefined, { + arguments: [ + { + id, + trace_id: id, + name: "invalid nested date", + inputs: { value: new Date(Number.NaN) }, + end_time: new Date().toISOString(), + }, + ], + }); + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans.filter((span) => span.span_id === id)).toHaveLength(1); + }); + it("contains malformed and prototype-sensitive payloads", async () => { let getterCalled = false; const payload = Object.create({ id: "inherited-id", name: "inherited" }); diff --git a/js/src/instrumentation/plugins/langsmith-plugin.ts b/js/src/instrumentation/plugins/langsmith-plugin.ts index d6958e9ab..3c6e33918 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.ts +++ b/js/src/instrumentation/plugins/langsmith-plugin.ts @@ -152,7 +152,13 @@ export class LangSmithPlugin extends BasePlugin { if (active) { const previous = active.run; active.run = mergeRuns(previous, run); - this.logRun(active.span, active.run, previous, false); + this.logRun( + active.span, + active.run, + previous, + timestampSeconds(ownValue(active.run, "end_time")) !== undefined || + errorMessage(ownValue(active.run, "error")) !== undefined, + ); this.endIfComplete(id, active, active.run); return; } @@ -160,7 +166,13 @@ export class LangSmithPlugin extends BasePlugin { const span = this.startRunSpan(id, run); const activeRun = { run, span }; this.activeRuns.set(id, activeRun); - this.logRun(span, run, undefined, false); + this.logRun( + span, + run, + undefined, + timestampSeconds(ownValue(run, "end_time")) !== undefined || + errorMessage(ownValue(run, "error")) !== undefined, + ); this.endIfComplete(id, activeRun, run); } @@ -526,7 +538,7 @@ function sanitizeLoggedValue( return Number.isFinite(value) ? value : undefined; } if (value instanceof Date) { - return value.toISOString(); + return Number.isFinite(value.getTime()) ? value.toISOString() : undefined; } if (typeof value !== "object" || depth >= 20) { return undefined; diff --git a/js/src/vendor-sdk-types/langsmith.ts b/js/src/vendor-sdk-types/langsmith.ts index 4f2f7d7f4..9326bf403 100644 --- a/js/src/vendor-sdk-types/langsmith.ts +++ b/js/src/vendor-sdk-types/langsmith.ts @@ -44,10 +44,6 @@ export type LangSmithClientConstructor = new ( ...args: unknown[] ) => LangSmithClient; -export type LangSmithClientModule = { - Client: LangSmithClientConstructor; -}; - export type LangSmithRunTree = LangSmithRun & { createChild?: (config: unknown) => LangSmithRunTree; postRun?: (excludeChildRuns?: boolean) => Promise; @@ -58,10 +54,6 @@ export type LangSmithRunTreeConstructor = new ( ...args: unknown[] ) => LangSmithRunTree; -export type LangSmithRunTreesModule = { - RunTree: LangSmithRunTreeConstructor; -}; - export type LangSmithTraceableConfig = { on_end?: (runTree?: LangSmithRunTree) => void; [key: string]: unknown; @@ -71,7 +63,3 @@ export type LangSmithTraceable = ( fn: (...args: never[]) => unknown, config?: LangSmithTraceableConfig, ) => (...args: never[]) => unknown; - -export type LangSmithTraceableModule = { - traceable: LangSmithTraceable; -}; diff --git a/js/src/wrappers/langsmith.test.ts b/js/src/wrappers/langsmith.test.ts index f694f0d64..5c2588696 100644 --- a/js/src/wrappers/langsmith.test.ts +++ b/js/src/wrappers/langsmith.test.ts @@ -104,7 +104,12 @@ describe("LangSmith namespace wrappers", () => { expect(tree.readSecret()).toBe("run-tree-secret"); await expect(tree.postRun()).resolves.toBe("run-tree-secret"); await expect(child.patchRun()).resolves.toBe("run-tree-secret"); - expect(tracePromise).toHaveBeenCalledTimes(2); + + const replacement = vi.fn(() => Promise.resolve("replacement")); + tree.postRun = replacement; + await expect(tree.postRun()).resolves.toBe("replacement"); + expect(replacement).toHaveBeenCalledOnce(); + expect(tracePromise).toHaveBeenCalledTimes(3); }); it("wraps Client lifecycle methods and safely binds other methods", async () => { @@ -136,7 +141,14 @@ describe("LangSmith namespace wrappers", () => { await client.createRun({ id: "create" }); await client.updateRun("update", { output: true }); await client.batchIngestRuns({ runCreates: [], runUpdates: [] }); - expect(tracePromise).toHaveBeenCalledTimes(3); + + const replacement = vi.fn((run: unknown) => Promise.resolve(run)); + client.createRun = replacement; + await expect(client.createRun({ id: "replacement" })).resolves.toEqual({ + id: "replacement", + }); + expect(replacement).toHaveBeenCalledOnce(); + expect(tracePromise).toHaveBeenCalledTimes(4); }); it("preserves Client method errors", async () => { diff --git a/js/src/wrappers/langsmith.ts b/js/src/wrappers/langsmith.ts index cc4a189a7..a8486c573 100644 --- a/js/src/wrappers/langsmith.ts +++ b/js/src/wrappers/langsmith.ts @@ -3,13 +3,10 @@ import { langSmithChannels } from "../instrumentation/plugins/langsmith-channels import type { LangSmithClient, LangSmithClientConstructor, - LangSmithClientModule, LangSmithRunTree, LangSmithRunTreeConstructor, - LangSmithRunTreesModule, LangSmithTraceable, LangSmithTraceableConfig, - LangSmithTraceableModule, } from "../vendor-sdk-types/langsmith"; const WRAPPED_CLIENT_CLASS = Symbol.for( @@ -41,7 +38,7 @@ export function wrapLangSmithTraceable(namespace: T): T { "traceable", WRAPPED_TRACEABLE_NAMESPACE, (value) => wrapTraceable(value as LangSmithTraceable), - ) as T & LangSmithTraceableModule; + ); } export function wrapLangSmithRunTrees(namespace: T): T { @@ -50,7 +47,7 @@ export function wrapLangSmithRunTrees(namespace: T): T { "RunTree", WRAPPED_RUN_TREES_NAMESPACE, (value) => wrapRunTreeClass(value as LangSmithRunTreeConstructor), - ) as T & LangSmithRunTreesModule; + ); } export function wrapLangSmithClient(namespace: T): T { @@ -59,7 +56,7 @@ export function wrapLangSmithClient(namespace: T): T { "Client", WRAPPED_CLIENT_NAMESPACE, (value) => wrapClientClass(value as LangSmithClientConstructor), - ) as T & LangSmithClientModule; + ); } function wrapNamespaceExport( @@ -205,17 +202,12 @@ function wrapRunTreeInstance(runTree: LangSmithRunTree): LangSmithRunTree { if ((runTree as Record)[WRAPPED_RUN_TREE_INSTANCE]) { return runTree; } - const cache = new Map(); return new Proxy(runTree, { get(target, prop, receiver) { if (prop === WRAPPED_RUN_TREE_INSTANCE) { return true; } - if (cache.has(prop)) { - return cache.get(prop); - } - const value = Reflect.get(target, prop, receiver); if (typeof value !== "function") { return value; @@ -247,7 +239,6 @@ function wrapRunTreeInstance(runTree: LangSmithRunTree): LangSmithRunTree { } else { wrapped = value.bind(target); } - cache.set(prop, wrapped); return wrapped; }, }); @@ -280,17 +271,12 @@ function wrapClientInstance(client: LangSmithClient): LangSmithClient { if ((client as Record)[WRAPPED_CLIENT_INSTANCE]) { return client; } - const cache = new Map(); return new Proxy(client, { get(target, prop, receiver) { if (prop === WRAPPED_CLIENT_INSTANCE) { return true; } - if (cache.has(prop)) { - return cache.get(prop); - } - const value = Reflect.get(target, prop, receiver); if (typeof value !== "function") { return value; @@ -327,7 +313,6 @@ function wrapClientInstance(client: LangSmithClient): LangSmithClient { } else { wrapped = value.bind(target); } - cache.set(prop, wrapped); return wrapped; }, });